From 47e1c7edee6f91e2bd5c81306daba4d3856e0123 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sat, 15 Aug 2026 17:16:10 +0200 Subject: [PATCH 01/15] Release 4.0.0 Bump vsn to 4.0.0 and date the CHANGELOG entry. Major release: callback results returning an Erlang string now arrive in Python as a list of integers, matching call arguments, and nothing raises on upgrade. --- CHANGELOG.md | 11 ++++++++++- src/erlang_python.app.src | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f0d9e7..f8e1be0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,15 @@ # Changelog -## Unreleased +## 4.0.0 (2026-08-15) + +### Breaking Changes + +- **Callback results returning an Erlang string** - a string returned from an + Erlang callback (`"abc"`, a list of integers) now reaches Python as + `[97, 98, 99]` rather than `'abc'`, the same conversion call arguments have + always used. Return a binary (`<<"abc">>`) for a Python `str`. Nothing raises + on upgrade, so check what your registered callbacks return before deploying. + See the encoding change under Changed below. ### Added diff --git a/src/erlang_python.app.src b/src/erlang_python.app.src index 9bdcfd9..94abb6c 100644 --- a/src/erlang_python.app.src +++ b/src/erlang_python.app.src @@ -1,6 +1,6 @@ {application, erlang_python, [ {description, "Execute Python applications from Erlang using dirty NIFs"}, - {vsn, "3.1.1"}, + {vsn, "4.0.0"}, {registered, []}, {mod, {erlang_python_app, []}}, {applications, [ From e4f3b6136a1f71bc1a55e897478855d57221f651 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sat, 15 Aug 2026 21:29:23 +0200 Subject: [PATCH 02/15] Add worker loops: run and drive an ErlangEventLoop inside a context py_context:start_loop/stop_loop/loop_ref/submit/submit_await and the erlang.server helper let Erlang run N owngil workers that serve TCP/UDP on a socket Erlang bound (py:dup_fd per worker) or adopt accepted fds, and inject coroutines into the running loop. Fixes: owngil contexts had no ErlangEventLoop and returned the main loop from context_get_event_loop; owngil dispatch blocked on a dirty scheduler with a 30 s cap; transports closed fds still in the poll set; READ re-arm from the loop thread crashed in enif_select; recycled call_soon handles cancelled by their previous owner; task start failures dropped silently. --- CHANGELOG.md | 72 ++++ README.md | 1 + c_src/py_callback.c | 5 + c_src/py_event_loop.c | 433 +++++++++++++++++--- c_src/py_event_loop.h | 41 +- c_src/py_nif.c | 121 ++++-- c_src/py_nif.h | 4 + docs/asyncio.md | 1 + docs/interrupts.md | 6 + docs/owngil_internals.md | 9 +- docs/workers.md | 192 +++++++++ examples/bench_worker_loop.erl | 264 +++++++++++++ priv/_erlang_impl/__init__.py | 38 ++ priv/_erlang_impl/_loop.py | 144 ++++++- priv/_erlang_impl/_server.py | 82 ++++ priv/_erlang_impl/_transport.py | 57 ++- priv/tests/test_loop_helpers.py | 183 +++++++++ priv/tests/test_server.py | 168 ++++++++ priv/tests/test_transport_close.py | 236 +++++++++++ rebar.config | 2 + src/erlang_python.app.src | 2 +- src/py_context.erl | 325 ++++++++++++++- src/py_event_worker.erl | 5 + src/py_nif.erl | 8 + test/py_asyncio_compat_SUITE.erl | 46 ++- test/py_test_workerloop.py | 182 +++++++++ test/py_worker_loop_SUITE.erl | 572 +++++++++++++++++++++++++++ test/py_worker_loop_stress_SUITE.erl | 350 ++++++++++++++++ 28 files changed, 3434 insertions(+), 115 deletions(-) create mode 100644 docs/workers.md create mode 100644 examples/bench_worker_loop.erl create mode 100644 priv/_erlang_impl/_server.py create mode 100644 priv/tests/test_loop_helpers.py create mode 100644 priv/tests/test_server.py create mode 100644 priv/tests/test_transport_close.py create mode 100644 test/py_test_workerloop.py create mode 100644 test/py_worker_loop_SUITE.erl create mode 100644 test/py_worker_loop_stress_SUITE.erl diff --git a/CHANGELOG.md b/CHANGELOG.md index f8e1be0..b5ddd25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,77 @@ # Changelog +## 4.1.0 (2026-08-15) + +### Added + +- **Worker loops** - `py_context:start_loop/1,2` runs an `ErlangEventLoop` + forever on the context thread and returns at once; `py_context:submit/4,5` + and `submit_await/4,5,6` schedule a coroutine or function on it from Erlang + (results as `{async_result, TaskRef, _}`, `py_event_loop:await/1,2`); + `stop_loop/1,2` stops it cooperatively, then interrupts after a grace + period; `loop_ref/1` exposes the loop for `py_nif:submit_task/7`. The owner + receives `{py_loop_exit, Ctx, Result}` when the loop ends. While a loop + runs, `call/eval/exec/call_method` on that context return + `{error, loop_running}`. `py_context:new/1` takes `preload => Code`, run once + in the context before anything else. See `docs/workers.md`. +- **`erlang.server`** - `serve(listen_fd, protocol_factory, udp=False)`, + `adopt(fd, protocol_factory)` and `stop_serving(server)`: serve TCP or UDP + on a socket Erlang bound (`py:dup_fd/1` per worker) or take over one + accepted connection, from a coroutine scheduled with `submit`. This is the + gunicorn shape inside the VM: Erlang binds once, N owngil contexts accept + on their copy of the fd, Erlang supervises, scales and reloads. +- **Injection into subinterpreter loops** - `py_nif:process_ready_tasks/1` + attaches a thread state to the loop's subinterpreter, so `submit_task` works + for owngil loops, idle or running; scheduling into a running loop now wakes + it instead of waiting for the next poll timeout (about 25 us round trip + instead of up to 1 s). Tasks that fail to start (missing module or function, + argument conversion, the call itself raising) are reported to the caller as + `{async_result, Ref, {error, Reason}}` instead of being dropped. + +### Fixed + +- **owngil contexts had no event loop** - `owngil_context_thread_main` created + the `py_event_loop` module without a default loop, so `erlang.run()`, + `create_server` and channels raised "Erlang event loop not initialized" in + owngil contexts. Each owngil context now owns an `ErlangEventLoop` served by + its own `py_event_worker`. `py_nif:context_get_event_loop/1` returned the + main interpreter's loop for owngil contexts, which made every owngil start + re-point the main loop's worker to a process that died with the context. +- **owngil dispatch** - calls into owngil contexts went through a blocking + dispatch on a dirty CPU scheduler with a 30 s cap + (`OWNGIL_DISPATCH_TIMEOUT_SECS`); they now use the same async queue as + worker mode: no dirty scheduler held during the call, no cap, and a lower + round trip (about 11.6 us against 15.6 us before on the bench machine). +- **fd closed while still in the poll set** - transports closed their socket + right after `ERL_NIF_SELECT_STOP` was issued, which under connection churn + produced `Bad input fd in erts_poll()` and `enif_select ... stealing + control of fd` reports and could deliver events to the wrong resource once + the number was reused. Transports now detach the fd and hand it to the NIF + (`_release_fd_resource(fd_key, take_ownership)`), which closes it from the + select stop callback; the reselect path and the close path serialise on the + loop mutex. 10k connections across four workers now log nothing. +- **Queued tasks dropped in pairs** - `py_nif:process_ready_tasks/1` dequeued + one whole iovec element per task; when erts stored several small task + binaries in one element (tasks queued behind a busy worker), every task + after the first in that element was lost, seen as every second + `submit_task` never answering on slow machines. It now dequeues exactly the + bytes each term consumed. Tasks beyond the batch limit queued behind a + running loop were also left waiting for the next wakeup; the running-loop + path now returns `more` like the idle path. +- **Recycled handles cancelled by their previous owner** - `ErlangEventLoop` + handed pooled `Handle` objects out of `call_soon` (and out of `call_at` + when the delay rounded to zero, which `asyncio.sleep(0.001)` does depending + on the clock value). asyncio cancels such handles after they ran + (`sleep` does in its `finally`), which cancelled whatever callback had been + given the recycled handle since: every second sleeper never woke. Only the + fd event handles created inside `_dispatch` are pooled now; `call_soon` + and `call_at` return fresh handles. +- **Re-arming a read select from the Python thread** - re-selecting READ on + an fd the BEAM had moved into a scheduler poll set crashed inside + `enif_select` when done from the loop thread (transport `resume_reading`, + `add_reader` on an fd with an active writer). Read re-arms now go through + the loop's `py_event_worker` (`py_nif:fd_arm/2`). + ## 4.0.0 (2026-08-15) ### Breaking Changes diff --git a/README.md b/README.md index 6b7b201..d98c347 100644 --- a/README.md +++ b/README.md @@ -650,6 +650,7 @@ py:execution_mode(). %% => worker | owngil - [Threading](docs/threading.md) - [Logging and Tracing](docs/logging.md) - [Asyncio Event Loop](docs/asyncio.md) - Erlang-native asyncio with TCP/UDP support +- [Worker Loops](docs/workers.md) - Long-lived loops in owngil contexts, serving on sockets Erlang owns - [Reactor](docs/reactor.md) - FD-based protocol handling - [Security](docs/security.md) - Sandbox and blocked operations - [Changelog](https://github.com/benoitc/erlang-python/releases) diff --git a/c_src/py_callback.c b/c_src/py_callback.c index 69d822b..ba630a2 100644 --- a/c_src/py_callback.c +++ b/c_src/py_callback.c @@ -4138,11 +4138,16 @@ static int create_erlang_module(void) { " erlang.byte_channel = _erlang_impl.byte_channel\n" " erlang.ByteChannel = _erlang_impl.ByteChannel\n" " erlang.ByteChannelClosed = _erlang_impl.ByteChannelClosed\n" + " # Worker loops (py_context:start_loop/submit) and fd serving\n" + " erlang.server = _erlang_impl.server\n" + " erlang._run_loop_forever = _erlang_impl._run_loop_forever\n" + " erlang._stop_loop = _erlang_impl._stop_loop\n" " # Make erlang behave as a package for 'import erlang.reactor' syntax\n" " erlang.__path__ = [priv_dir]\n" " sys.modules['erlang.reactor'] = erlang.reactor\n" " sys.modules['erlang.channel'] = erlang.channel\n" " sys.modules['erlang.byte_channel'] = erlang.byte_channel\n" + " sys.modules['erlang.server'] = erlang.server\n" " return True\n" " except ImportError as e:\n" " import sys\n" diff --git a/c_src/py_event_loop.c b/c_src/py_event_loop.c index 5ed6573..198b048 100644 --- a/c_src/py_event_loop.c +++ b/c_src/py_event_loop.c @@ -35,6 +35,7 @@ */ #include "py_nif.h" +#include #include "py_event_loop.h" #include "py_reactor_buffer.h" @@ -338,6 +339,10 @@ static int set_interpreter_event_loop(erlang_event_loop_t *loop) { return 0; } +erlang_event_loop_t *get_current_interpreter_event_loop(void) { + return get_interpreter_event_loop(); +} + /* ============================================================================ * Resource Callbacks * ============================================================================ */ @@ -1950,11 +1955,6 @@ ERL_NIF_TERM nif_handle_fd_event_and_reselect(ErlNifEnv *env, int argc, return make_error(env, "invalid_fd_ref"); } - /* Check if FD is still open */ - if (atomic_load(&fd_res->closing_state) != FD_STATE_OPEN) { - return ATOM_OK; /* Silently ignore events on closing FDs */ - } - erlang_event_loop_t *loop = fd_res->loop; if (loop == NULL) { return make_error(env, "no_loop"); @@ -1965,6 +1965,18 @@ ERL_NIF_TERM nif_handle_fd_event_and_reselect(ErlNifEnv *env, int argc, uint64_t callback_id; bool is_active; + /* The state check and the reselect must be one step: the Python thread + * closes fds through py_release_fd_resource, which moves the state to + * CLOSING under this same mutex before it issues ERL_NIF_SELECT_STOP. + * Without the lock we could pass the check, lose the race, and reselect + * a closed (or already reused) fd number. */ + pthread_mutex_lock(&loop->mutex); + + if (atomic_load(&fd_res->closing_state) != FD_STATE_OPEN) { + pthread_mutex_unlock(&loop->mutex); + return ATOM_OK; /* Silently ignore events on closing FDs */ + } + if (is_read) { callback_id = fd_res->read_callback_id; is_active = fd_res->reader_active; @@ -1974,13 +1986,10 @@ ERL_NIF_TERM nif_handle_fd_event_and_reselect(ErlNifEnv *env, int argc, } if (!is_active || callback_id == 0) { + pthread_mutex_unlock(&loop->mutex); return ATOM_OK; /* Watcher was stopped, ignore */ } - /* Add to pending queue (has duplicate detection) */ - event_type_t event_type = is_read ? EVENT_TYPE_READ : EVENT_TYPE_WRITE; - event_loop_add_pending(loop, event_type, callback_id, fd_res->fd); - /* Immediately reselect for next event. * Use ATOM_UNDEFINED instead of enif_make_ref to avoid per-event allocation. * The ref is ignored by the worker anyway. */ @@ -1989,6 +1998,49 @@ ERL_NIF_TERM nif_handle_fd_event_and_reselect(ErlNifEnv *env, int argc, enif_select(env, (ErlNifEvent)fd_res->fd, select_flags, fd_res, target_pid, ATOM_UNDEFINED); + pthread_mutex_unlock(&loop->mutex); + + /* Add to pending queue (has duplicate detection; takes loop->mutex itself) */ + event_type_t event_type = is_read ? EVENT_TYPE_READ : EVENT_TYPE_WRITE; + event_loop_add_pending(loop, event_type, callback_id, fd_res->fd); + + return ATOM_OK; +} + +/** + * fd_arm(FdRef, read | write) -> ok + * + * Re-arm a read or write select for an fd whose resource already exists. + * Called by the loop's py_event_worker on behalf of ErlangEventLoop + * (_update_fd_read/_update_fd_write): re-selecting READ on an fd that a + * scheduler thread already polled must itself run on a scheduler thread, + * since erts keeps such fds in the scheduler's own poll set. Doing it from + * the Python thread crashes inside enif_select. Fresh selects, CANCEL and + * STOP are fine from any thread, and only re-arms go through here. + */ +ERL_NIF_TERM nif_fd_arm(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + (void)argc; + + fd_resource_t *fd_res; + if (!enif_get_resource(env, argv[0], FD_RESOURCE_TYPE, (void **)&fd_res)) { + return make_error(env, "invalid_fd_ref"); + } + erlang_event_loop_t *loop = fd_res->loop; + if (loop == NULL) { + return make_error(env, "no_loop"); + } + bool is_read = enif_compare(argv[1], ATOM_READ) == 0; + + pthread_mutex_lock(&loop->mutex); + if (atomic_load(&fd_res->closing_state) != FD_STATE_OPEN || + (is_read ? !fd_res->reader_active : !fd_res->writer_active)) { + pthread_mutex_unlock(&loop->mutex); + return ATOM_OK; /* Closed or disarmed again in the meantime */ + } + int select_flags = is_read ? ERL_NIF_SELECT_READ : ERL_NIF_SELECT_WRITE; + enif_select(env, (ErlNifEvent)fd_res->fd, select_flags, + fd_res, &loop->worker_pid, ATOM_UNDEFINED); + pthread_mutex_unlock(&loop->mutex); return ATOM_OK; } @@ -2753,6 +2805,102 @@ static inline void return_pooled_env(erlang_event_loop_t *loop, ErlNifEnv *term_ } } +/* ============================================================================ + * GIL handling for process_ready_tasks + * + * Main-interpreter loops use PyGILState_Ensure. Subinterpreter loops (OWN_GIL + * contexts) cannot: PyGILState_* only knows the main interpreter, so a fresh + * thread state is created for loop->interp and bound to this scheduler thread + * for the duration of the call. The attach count lets the interpreter thread + * wait for us before Py_EndInterpreter (event_loop_detach_interpreter). + * ============================================================================ */ + +typedef struct { + PyGILState_STATE gstate; + PyThreadState *tstate; /* non-NULL when attached to a subinterpreter */ +} loop_gil_t; + +static bool loop_gil_acquire(erlang_event_loop_t *loop, loop_gil_t *g) { + g->tstate = NULL; +#ifdef HAVE_SUBINTERPRETERS + if (loop->interp_id != 0) { + pthread_mutex_lock(&loop->mutex); + if (loop->interp == NULL) { + pthread_mutex_unlock(&loop->mutex); + return false; + } + g->tstate = PyThreadState_New(loop->interp); + if (g->tstate == NULL) { + pthread_mutex_unlock(&loop->mutex); + return false; + } + loop->external_attached++; + pthread_mutex_unlock(&loop->mutex); + /* Take the subinterpreter GIL outside loop->mutex: the loop thread + * may hold the GIL while waiting for loop->mutex. */ + PyEval_RestoreThread(g->tstate); + return true; + } +#endif + g->gstate = PyGILState_Ensure(); + return true; +} + +static void loop_gil_release(erlang_event_loop_t *loop, loop_gil_t *g) { +#ifdef HAVE_SUBINTERPRETERS + if (g->tstate != NULL) { + PyThreadState_Clear(g->tstate); + PyThreadState_DeleteCurrent(); /* releases the subinterpreter GIL */ + g->tstate = NULL; + pthread_mutex_lock(&loop->mutex); + loop->external_attached--; + pthread_mutex_unlock(&loop->mutex); + return; + } +#endif + (void)loop; + PyGILState_Release(g->gstate); +} + +void event_loop_detach_interpreter(erlang_event_loop_t *loop) { + if (loop == NULL) { + return; + } + pthread_mutex_lock(&loop->mutex); + loop->interp = NULL; + /* Teardown is rare: poll rather than add a condvar to the loop struct. */ + while (loop->external_attached > 0) { + pthread_mutex_unlock(&loop->mutex); + usleep(1000); + pthread_mutex_lock(&loop->mutex); + } + pthread_mutex_unlock(&loop->mutex); +} + +/** + * Report a task that could not be started to its caller as + * {async_result, Ref, {error, Reason}} instead of dropping it silently. + * Reason is the pending Python exception when one is set (cleared here), + * otherwise the given atom. Runs with the GIL held. + */ +static void send_task_failure(ErlNifEnv *term_env, ErlNifPid *caller_pid, + ERL_NIF_TERM ref, const char *reason) { + ErlNifEnv *msg_env = enif_alloc_env(); + if (msg_env == NULL) { + PyErr_Clear(); + return; + } + ERL_NIF_TERM err = PyErr_Occurred() ? make_py_error(msg_env) + : make_error(msg_env, reason); + ERL_NIF_TERM msg = enif_make_tuple3(msg_env, + enif_make_atom(msg_env, "async_result"), + enif_make_copy(msg_env, ref), + err); + (void)term_env; + enif_send(NULL, caller_pid, msg_env, msg); + enif_free_env(msg_env); +} + /** * process_ready_tasks(LoopRef) -> ok | {error, Reason} * @@ -2842,8 +2990,9 @@ ERL_NIF_TERM nif_process_ready_tasks(ErlNifEnv *env, int argc, } ERL_NIF_TERM task_term; - if (enif_binary_to_term(term_env, task_bin.data, task_bin.size, - &task_term, ERL_NIF_BIN2TERM_SAFE) == 0) { + size_t consumed = enif_binary_to_term(term_env, task_bin.data, task_bin.size, + &task_term, ERL_NIF_BIN2TERM_SAFE); + if (consumed == 0) { return_pooled_env(loop, term_env); /* Dequeue and skip this malformed task */ enif_ioq_deq(loop->task_queue, iov[0].iov_len, NULL); @@ -2856,8 +3005,11 @@ ERL_NIF_TERM nif_process_ready_tasks(ErlNifEnv *env, int argc, tasks[num_tasks].task_term = task_term; num_tasks++; - /* Dequeue (we've copied the data) */ - enif_ioq_deq(loop->task_queue, iov[0].iov_len, NULL); + /* Dequeue exactly the bytes of this term. The io queue merges small + * binaries enqueued back to back into one iovec element, so a slow + * consumer can find several tasks in iov[0]; dequeuing iov_len would + * silently drop the ones after the first. */ + enif_ioq_deq(loop->task_queue, consumed, NULL); atomic_fetch_sub(&loop->task_count, 1); } @@ -2866,13 +3018,28 @@ ERL_NIF_TERM nif_process_ready_tasks(ErlNifEnv *env, int argc, /* NOTE: We do NOT return early here even if num_tasks == 0. * We may have pending timer/FD events that need _run_once to process. * The first check (task_count == 0 && pending_count == 0) at the start - * of this function already handles the case where there's truly no work. */ + * of this function already handles the case where there's truly no work. + * + * Exception: a loop driven by run_forever on its own thread consumes + * pending events itself. With no tasks to schedule there is nothing for + * us to do under its GIL; wake it and leave. */ + if (num_tasks == 0 && atomic_load(&loop->py_running)) { + if (!loop->shutdown) { + pthread_mutex_lock(&loop->mutex); + pthread_cond_broadcast(&loop->event_cond); + pthread_mutex_unlock(&loop->mutex); + } + return ATOM_OK; + } /* ======================================================================== * PHASE 2: Process all tasks WITH GIL (Python operations) * ======================================================================== */ - PyGILState_STATE gstate = PyGILState_Ensure(); + loop_gil_t gil; + if (!loop_gil_acquire(loop, &gil)) { + return make_error(env, "interpreter_gone"); + } /* OPTIMIZATION: Use cached Python imports (uvloop-style) * Avoids PyImport_ImportModule on every call */ @@ -2897,7 +3064,7 @@ ERL_NIF_TERM nif_process_ready_tasks(ErlNifEnv *env, int argc, for (int i = 0; i < num_tasks; i++) { return_pooled_env(loop, tasks[i].term_env); } - PyGILState_Release(gstate); + loop_gil_release(loop, &gil); return make_error(env, "asyncio_import_failed"); } @@ -2914,7 +3081,7 @@ ERL_NIF_TERM nif_process_ready_tasks(ErlNifEnv *env, int argc, for (int i = 0; i < num_tasks; i++) { return_pooled_env(loop, tasks[i].term_env); } - PyGILState_Release(gstate); + loop_gil_release(loop, &gil); return make_error(env, "erlang_loop_import_failed"); } @@ -2925,7 +3092,7 @@ ERL_NIF_TERM nif_process_ready_tasks(ErlNifEnv *env, int argc, for (int i = 0; i < num_tasks; i++) { return_pooled_env(loop, tasks[i].term_env); } - PyGILState_Release(gstate); + loop_gil_release(loop, &gil); return make_error(env, "run_and_send_not_found"); } @@ -2938,7 +3105,7 @@ ERL_NIF_TERM nif_process_ready_tasks(ErlNifEnv *env, int argc, for (int i = 0; i < num_tasks; i++) { return_pooled_env(loop, tasks[i].term_env); } - PyGILState_Release(gstate); + loop_gil_release(loop, &gil); return make_error(env, "events_import_failed"); } @@ -2965,7 +3132,7 @@ ERL_NIF_TERM nif_process_ready_tasks(ErlNifEnv *env, int argc, for (int i = 0; i < num_tasks; i++) { return_pooled_env(loop, tasks[i].term_env); } - PyGILState_Release(gstate); + loop_gil_release(loop, &gil); return make_error(env, "loop_module_import_failed"); } @@ -2976,7 +3143,7 @@ ERL_NIF_TERM nif_process_ready_tasks(ErlNifEnv *env, int argc, for (int i = 0; i < num_tasks; i++) { return_pooled_env(loop, tasks[i].term_env); } - PyGILState_Release(gstate); + loop_gil_release(loop, &gil); return make_error(env, "loop_class_not_found"); } @@ -2987,7 +3154,7 @@ ERL_NIF_TERM nif_process_ready_tasks(ErlNifEnv *env, int argc, for (int i = 0; i < num_tasks; i++) { return_pooled_env(loop, tasks[i].term_env); } - PyGILState_Release(gstate); + loop_gil_release(loop, &gil); return make_error(env, "loop_creation_failed"); } @@ -3101,6 +3268,7 @@ ERL_NIF_TERM nif_process_ready_tasks(ErlNifEnv *env, int argc, } if (func == NULL) { + send_task_failure(term_env, &caller_pid, tuple_elems[1], "function_not_found"); return_pooled_env(loop, term_env); continue; } @@ -3135,6 +3303,7 @@ ERL_NIF_TERM nif_process_ready_tasks(ErlNifEnv *env, int argc, if (!args_ok) { Py_DECREF(args); Py_DECREF(func); + send_task_failure(term_env, &caller_pid, tuple_elems[1], "args_conversion_failed"); return_pooled_env(loop, term_env); continue; } @@ -3160,7 +3329,8 @@ ERL_NIF_TERM nif_process_ready_tasks(ErlNifEnv *env, int argc, Py_XDECREF(kwargs); if (coro == NULL) { - PyErr_Clear(); + /* The call itself raised: report the Python exception */ + send_task_failure(term_env, &caller_pid, tuple_elems[1], "call_failed"); return_pooled_env(loop, term_env); continue; } @@ -3248,13 +3418,25 @@ ERL_NIF_TERM nif_process_ready_tasks(ErlNifEnv *env, int argc, int running = PyObject_IsTrue(is_running); Py_DECREF(is_running); if (running) { - /* Loop is already running - just signal it and clean up. - * The pending events were already added by dispatch_timer/handle_fd_event, - * and the condition variable was signaled. The running loop will wake up - * and process them. + /* Loop is already running (run_forever on another thread): the + * coroutines were scheduled on its ready queue, wake it so they + * start now instead of at the next poll timeout. Same broadcast + * as _wakeup_for (call_soon_threadsafe). * Note: events_module is cached, so we don't DECREF it. */ Py_XDECREF(old_running_loop); - PyGILState_Release(gstate); + loop_gil_release(loop, &gil); + if (!loop->shutdown) { + pthread_mutex_lock(&loop->mutex); + pthread_cond_broadcast(&loop->event_cond); + pthread_mutex_unlock(&loop->mutex); + } + /* Same contract as the tail of this function: tasks beyond the + * batch limit are still queued, tell the worker to come back + * (submit_task will not send another wakeup while one is + * pending, so returning ok here would strand them). */ + if (atomic_load(&loop->task_count) > 0) { + return ATOM_MORE; + } return ATOM_OK; } } else { @@ -3312,7 +3494,7 @@ ERL_NIF_TERM nif_process_ready_tasks(ErlNifEnv *env, int argc, Py_XDECREF(restore); Py_XDECREF(old_running_loop); - PyGILState_Release(gstate); + loop_gil_release(loop, &gil); /* * Check if there are more tasks remaining (we hit MAX_TASK_BATCH limit). @@ -4265,7 +4447,7 @@ bool event_loop_add_pending(erlang_event_loop_t *loop, event_type_t type, * * Uses the same coalescing logic as submit_task to avoid message floods. */ - if (loop->has_worker) { + if (loop->has_worker && !atomic_load(&loop->py_running)) { if (!atomic_exchange(&loop->task_wake_pending, true)) { ErlNifEnv *msg_env = enif_alloc_env(); if (msg_env != NULL) { @@ -5317,6 +5499,16 @@ ERL_NIF_TERM nif_context_get_event_loop(ErlNifEnv *env, int argc, return make_error(env, "not_subinterp"); } + /* OWN_GIL contexts: the loop was created by the context thread inside its + * subinterpreter and recorded on the context. Read it without touching + * the main GIL, which would resolve the main interpreter's loop instead. */ + if (ctx->uses_own_gil) { + if (ctx->event_loop == NULL) { + return make_error(env, "no_event_loop"); + } + return enif_make_tuple2(env, ATOM_OK, enif_make_resource(env, ctx->event_loop)); + } + /* With shared-GIL pool model, event loop operations work on dirty schedulers. * py_context_acquire handles PyThreadState_Swap to the subinterpreter. */ @@ -7131,6 +7323,7 @@ static PyObject *py_loop_new(PyObject *self, PyObject *args) { } else { loop->interp_id = 0; /* Main interpreter */ } + loop->interp = current_interp; #else loop->interp_id = 0; /* Main interpreter */ #endif @@ -7643,15 +7836,42 @@ static PyObject *py_update_fd_read(PyObject *self, PyObject *args) { PyErr_SetString(PyExc_ValueError, "Invalid fd resource"); return NULL; } + if (fd_res->fd < 0 || atomic_load(&fd_res->closing_state) != FD_STATE_OPEN) { + enif_release_resource(fd_res); + PyErr_SetString(PyExc_ValueError, "fd resource is closed"); + return NULL; + } + + if (!event_loop_ensure_worker(fd_res->loop)) { + enif_release_resource(fd_res); + PyErr_SetString(PyExc_RuntimeError, "Event loop has no router or worker"); + return NULL; + } fd_res->read_callback_id = callback_id; fd_res->reader_active = true; - /* Re-register for read events (may already be registered, that's OK) */ - ErlNifPid *target_pid = fd_res->loop->has_worker ? - &fd_res->loop->worker_pid : &fd_res->loop->router_pid; - enif_select(fd_res->loop->msg_env, (ErlNifEvent)fd_res->fd, - ERL_NIF_SELECT_READ, fd_res, target_pid, ATOM_UNDEFINED); + /* Re-arm from the worker process, not from this Python thread: see + * nif_fd_arm for why a READ re-select must run on a scheduler thread. */ + ErlNifEnv *arm_env = enif_alloc_env(); + if (arm_env == NULL) { + fd_res->reader_active = false; + enif_release_resource(fd_res); + PyErr_SetString(PyExc_MemoryError, "Failed to allocate env"); + return NULL; + } + ERL_NIF_TERM arm_msg = enif_make_tuple3(arm_env, + enif_make_atom(arm_env, "fd_arm"), + enif_make_resource(arm_env, fd_res), + ATOM_READ); + if (!enif_send(NULL, &fd_res->loop->worker_pid, arm_env, arm_msg)) { + enif_free_env(arm_env); + fd_res->reader_active = false; + enif_release_resource(fd_res); + PyErr_SetString(PyExc_RuntimeError, "Event loop worker is gone"); + return NULL; + } + enif_free_env(arm_env); enif_release_resource(fd_res); Py_RETURN_NONE; @@ -7676,15 +7896,31 @@ static PyObject *py_update_fd_write(PyObject *self, PyObject *args) { PyErr_SetString(PyExc_ValueError, "Invalid fd resource"); return NULL; } + if (fd_res->fd < 0 || atomic_load(&fd_res->closing_state) != FD_STATE_OPEN) { + enif_release_resource(fd_res); + PyErr_SetString(PyExc_ValueError, "fd resource is closed"); + return NULL; + } + + if (!event_loop_ensure_worker(fd_res->loop)) { + enif_release_resource(fd_res); + PyErr_SetString(PyExc_RuntimeError, "Event loop has no router or worker"); + return NULL; + } fd_res->write_callback_id = callback_id; fd_res->writer_active = true; - /* Re-register for write events */ - ErlNifPid *target_pid = fd_res->loop->has_worker ? - &fd_res->loop->worker_pid : &fd_res->loop->router_pid; - enif_select(fd_res->loop->msg_env, (ErlNifEvent)fd_res->fd, - ERL_NIF_SELECT_WRITE, fd_res, target_pid, ATOM_UNDEFINED); + /* Same target as _add_writer_for: the loop's worker process. */ + ErlNifPid *target_pid = &fd_res->loop->worker_pid; + int ret = enif_select(fd_res->loop->msg_env, (ErlNifEvent)fd_res->fd, + ERL_NIF_SELECT_WRITE, fd_res, target_pid, ATOM_UNDEFINED); + if (ret < 0) { + fd_res->writer_active = false; + enif_release_resource(fd_res); + PyErr_SetString(PyExc_RuntimeError, "Failed to register fd for writing"); + return NULL; + } enif_release_resource(fd_res); Py_RETURN_NONE; @@ -7752,21 +7988,53 @@ static PyObject *py_clear_fd_write(PyObject *self, PyObject *args) { /** * Release fd_resource (stop all monitoring and release). - * Python function: _release_fd_resource(fd_key) -> None + * Python function: _release_fd_resource(fd_key, take_ownership=False) -> None + * + * With take_ownership the fd is closed by us, from the enif_select stop + * callback once the poll set has dropped it (or right here when it is not + * selected). Closing an fd from Python while ERL_NIF_SELECT_STOP is still + * scheduled leaves a stale entry in the poll set and lets the number be + * reused by the next accept(): that is the "Bad input fd in erts_poll" / + * "stealing control of fd" report class. Transports hand their socket over + * this way (see ErlangEventLoop._close_socket). */ static PyObject *py_release_fd_resource(PyObject *self, PyObject *args) { (void)self; unsigned long long fd_key; + int take_ownership = 0; - if (!PyArg_ParseTuple(args, "K", &fd_key)) { + if (!PyArg_ParseTuple(args, "K|p", &fd_key, &take_ownership)) { return NULL; } fd_resource_t *fd_res = fd_reg_take(fd_key); if (fd_res != NULL) { - if (fd_res->loop != NULL) { - enif_select(fd_res->loop->msg_env, (ErlNifEvent)fd_res->fd, - ERL_NIF_SELECT_STOP, fd_res, NULL, ATOM_UNDEFINED); + erlang_event_loop_t *loop = fd_res->loop; + if (take_ownership) { + fd_res->owns_fd = true; + /* Move to CLOSING under loop->mutex so a concurrent + * handle_fd_event_and_reselect cannot reselect this fd. */ + if (loop != NULL) pthread_mutex_lock(&loop->mutex); + int expected = FD_STATE_OPEN; + atomic_compare_exchange_strong(&fd_res->closing_state, + &expected, FD_STATE_CLOSING); + fd_res->reader_active = false; + fd_res->writer_active = false; + if (loop != NULL) pthread_mutex_unlock(&loop->mutex); + } + int rc = -1; + if (loop != NULL) { + rc = enif_select(loop->msg_env, (ErlNifEvent)fd_res->fd, + ERL_NIF_SELECT_STOP, fd_res, NULL, ATOM_UNDEFINED); + } + if (rc < 0 && take_ownership && fd_res->fd >= 0) { + /* Never selected (or no loop): nothing pending in the poll set */ + int expected = FD_STATE_CLOSING; + if (atomic_compare_exchange_strong(&fd_res->closing_state, + &expected, FD_STATE_CLOSED)) { + close(fd_res->fd); + fd_res->fd = -1; + } } enif_release_resource(fd_res); } @@ -7881,6 +8149,40 @@ static PyObject *py_cancel_timer_for(PyObject *self, PyObject *args) { Py_RETURN_NONE; } +/* Python function: _set_running_for(capsule, running) -> None + * + * Marks the loop as driven by run_forever on the calling thread (or not). + * See erlang_event_loop_t.py_running. */ +static PyObject *py_set_running_for(PyObject *self, PyObject *args) { + (void)self; + PyObject *capsule; + int running; + + if (!PyArg_ParseTuple(args, "Op", &capsule, &running)) { + return NULL; + } + erlang_event_loop_t *loop = loop_from_capsule(capsule); + if (loop == NULL) { + PyErr_Clear(); + Py_RETURN_NONE; + } + atomic_store(&loop->py_running, running ? true : false); + if (!running && loop->has_worker && !loop->shutdown) { + /* Events queued while we were running but not yet consumed must + * now be handled by the worker path again */ + if (atomic_load(&loop->pending_count) > 0 && + !atomic_exchange(&loop->task_wake_pending, true)) { + ErlNifEnv *msg_env = enif_alloc_env(); + if (msg_env != NULL) { + enif_send(NULL, &loop->worker_pid, msg_env, + enif_make_atom(msg_env, "task_ready")); + enif_free_env(msg_env); + } + } + } + Py_RETURN_NONE; +} + /* Python function: _wakeup_for(capsule) -> None */ static PyObject *py_wakeup_for(PyObject *self, PyObject *args) { (void)self; @@ -8019,6 +8321,7 @@ static PyMethodDef PyEventLoopMethods[] = { {"_set_global_loop_ref", py_set_global_loop_ref, METH_VARARGS, "Store Python loop reference in global loop"}, {"_run_once_native_for", py_run_once_for, METH_VARARGS, "Combined poll + get_pending for specific loop"}, {"_get_pending_for", py_get_pending_for, METH_VARARGS, "Get and clear pending events for specific loop"}, + {"_set_running_for", py_set_running_for, METH_VARARGS, "Mark loop as running under run_forever"}, {"_wakeup_for", py_wakeup_for, METH_VARARGS, "Wake up specific event loop"}, {"_is_initialized_for", py_is_initialized_for, METH_VARARGS, "Check if specific loop is initialized"}, {"_add_reader_for", py_add_reader_for, METH_VARARGS, "Register fd for read monitoring on specific loop"}, @@ -8149,6 +8452,45 @@ int create_default_event_loop(ErlNifEnv *env) { loop->has_router = false; loop->has_self = false; + /* Async task queue, env pool and namespace registry, as in + * nif_event_loop_new: a default loop must accept submit_task too, since + * OWN_GIL contexts only ever have this loop. */ + loop->task_queue = enif_ioq_create(ERL_NIF_IOQ_NORMAL); + if (loop->task_queue == NULL || + pthread_mutex_init(&loop->task_queue_mutex, NULL) != 0) { + if (loop->task_queue != NULL) { + enif_ioq_destroy(loop->task_queue); + loop->task_queue = NULL; + } + enif_free_env(loop->msg_env); + pthread_cond_destroy(&loop->event_cond); + pthread_mutex_destroy(&loop->mutex); + enif_release_resource(loop); + return -1; + } + loop->task_queue_initialized = true; + atomic_store(&loop->task_count, 0); + atomic_store(&loop->task_wake_pending, false); + loop->py_loop = NULL; + loop->py_loop_valid = false; + loop->py_cache_valid = false; + loop->callable_cache_count = 0; + loop->env_pool_count = 0; + if (pthread_mutex_init(&loop->env_pool_mutex, NULL) != 0 || + pthread_mutex_init(&loop->namespaces_mutex, NULL) != 0) { + pthread_mutex_destroy(&loop->task_queue_mutex); + loop->task_queue_initialized = false; + enif_ioq_destroy(loop->task_queue); + loop->task_queue = NULL; + enif_free_env(loop->msg_env); + pthread_cond_destroy(&loop->event_cond); + pthread_mutex_destroy(&loop->mutex); + enif_release_resource(loop); + return -1; + } + loop->namespaces_head = NULL; + loop->pid_env_head = NULL; + #ifdef HAVE_SUBINTERPRETERS /* Check if this is a subinterpreter by comparing to main interpreter */ PyInterpreterState *current_interp = PyInterpreterState_Get(); @@ -8159,6 +8501,7 @@ int create_default_event_loop(ErlNifEnv *env) { } else { loop->interp_id = 0; /* Main interpreter */ } + loop->interp = current_interp; #else loop->interp_id = 0; /* Main interpreter */ #endif diff --git a/c_src/py_event_loop.h b/c_src/py_event_loop.h index 419b777..2485e5b 100644 --- a/c_src/py_event_loop.h +++ b/c_src/py_event_loop.h @@ -39,8 +39,9 @@ #include #include -/* Forward declaration for Python object (avoids including Python.h in header) */ +/* Forward declarations for Python objects (avoids including Python.h in header) */ typedef struct _object PyObject; +typedef struct _is PyInterpreterState; /* ============================================================================ * Constants @@ -347,6 +348,20 @@ typedef struct erlang_event_loop { /** @brief Interpreter ID: 0 = main interpreter, >0 = subinterpreter */ uint32_t interp_id; + /** @brief Owning interpreter, needed to attach a thread state to a + * subinterpreter loop from an Erlang scheduler (see process_ready_tasks). + * NULL once the interpreter is being torn down. Guarded by mutex. */ + PyInterpreterState *interp; + + /** @brief Number of scheduler threads currently attached to interp + * through loop_gil_acquire(). Guarded by mutex. */ + int external_attached; + + /** @brief True while ErlangEventLoop.run_forever() drives this loop on + * its own thread. Pending events then need no task_ready round trip + * through the worker: the loop thread picks them up itself. */ + _Atomic bool py_running; + /* ========== Async Task Queue (uvloop-inspired) ========== */ /* * Future optimization: Replace serialized task queue with native MPSC @@ -864,6 +879,13 @@ ERL_NIF_TERM nif_handle_fd_event(ErlNifEnv *env, int argc, ERL_NIF_TERM nif_handle_fd_event_and_reselect(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +/** + * @brief Re-arm a read/write select from a scheduler thread + * + * NIF: fd_arm(FdRef, read | write) -> ok | {error, Reason} + */ +ERL_NIF_TERM nif_fd_arm(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + /** * @brief Stop read monitoring without closing the FD * @@ -1092,6 +1114,23 @@ int create_default_event_loop(ErlNifEnv *env); */ int init_subinterpreter_event_loop(ErlNifEnv *env); +/** + * @brief Event loop of the interpreter bound to the calling thread + * + * Must be called with that interpreter's GIL held. Returns NULL when no + * default loop exists yet. + */ +erlang_event_loop_t *get_current_interpreter_event_loop(void); + +/** + * @brief Detach a subinterpreter loop from its interpreter before teardown + * + * Call from the interpreter's own thread with its GIL released, right before + * Py_EndInterpreter. Blocks until every scheduler thread attached through + * process_ready_tasks has detached, and refuses new attachments. + */ +void event_loop_detach_interpreter(erlang_event_loop_t *loop); + /* ============================================================================ * Reactor NIF Functions (Erlang-as-Reactor architecture) * ============================================================================ */ diff --git a/c_src/py_nif.c b/c_src/py_nif.c index f7443ba..527b738 100644 --- a/c_src/py_nif.c +++ b/c_src/py_nif.c @@ -3542,6 +3542,20 @@ static ERL_NIF_TERM dispatch_to_worker_thread( * @param local_env Optional local environment (NULL for default) * @return {enqueued, RequestId} on success, {error, Reason} on failure */ +/** + * @brief Whether requests to this context go through the shared request queue + * of a dedicated thread (worker or OWN_GIL), which is what the async + * dispatch NIFs need. + */ +static inline bool ctx_uses_async_thread(const py_context_t *ctx) { +#ifdef HAVE_SUBINTERPRETERS + if (ctx->uses_own_gil) { + return true; + } +#endif + return ctx->uses_worker_thread; +} + static ERL_NIF_TERM dispatch_to_worker_thread_async( ErlNifEnv *env, py_context_t *ctx, @@ -3638,15 +3652,22 @@ static void *owngil_context_thread_main(void *arg) { return NULL; } - /* Register py_event_loop module for reactor support */ - if (create_py_event_loop_module() < 0) { - fprintf(stderr, "OWN_GIL: create_py_event_loop_module failed\n"); + /* Register py_event_loop module and create this interpreter's default + * ErlangEventLoop, so asyncio I/O (create_server, channels, timers) works + * inside the context. Keep a reference on the context so Erlang can wire a + * dedicated py_event_worker without acquiring the main GIL. */ + if (init_subinterpreter_event_loop(NULL) < 0) { + fprintf(stderr, "OWN_GIL: init_subinterpreter_event_loop failed\n"); PyErr_Print(); Py_EndInterpreter(ctx->own_gil_tstate); atomic_store(&ctx->init_error, true); atomic_store(&ctx->worker_running, false); return NULL; } + ctx->event_loop = get_current_interpreter_event_loop(); + if (ctx->event_loop != NULL) { + enif_keep_resource(ctx->event_loop); + } /* Create namespace dictionaries */ ctx->globals = PyDict_New(); @@ -3706,19 +3727,30 @@ static void *owngil_context_thread_main(void *arg) { /* Check if request was cancelled while queued */ if (atomic_load(&req->cancelled)) { - /* Request cancelled - signal completion without processing */ - req->result_env = enif_alloc_env(); - if (req->result_env) { - req->result = enif_make_tuple2(req->result_env, - enif_make_atom(req->result_env, "error"), - enif_make_atom(req->result_env, "cancelled")); - } - req->success = false; + /* Request cancelled - deliver error without processing */ + if (req->async_mode) { + enif_clear_env(ctx->msg_env); + ERL_NIF_TERM cancel_msg = enif_make_tuple3(ctx->msg_env, + enif_make_atom(ctx->msg_env, "py_result"), + enif_make_copy(ctx->msg_env, req->request_id), + enif_make_tuple2(ctx->msg_env, + enif_make_atom(ctx->msg_env, "error"), + enif_make_atom(ctx->msg_env, "cancelled"))); + enif_send(NULL, &req->caller_pid, ctx->msg_env, cancel_msg); + } else { + req->result_env = enif_alloc_env(); + if (req->result_env) { + req->result = enif_make_tuple2(req->result_env, + enif_make_atom(req->result_env, "error"), + enif_make_atom(req->result_env, "cancelled")); + } + req->success = false; - pthread_mutex_lock(&req->mutex); - atomic_store(&req->completed, true); - pthread_cond_signal(&req->cond); - pthread_mutex_unlock(&req->mutex); + pthread_mutex_lock(&req->mutex); + atomic_store(&req->completed, true); + pthread_cond_signal(&req->cond); + pthread_mutex_unlock(&req->mutex); + } ctx_request_release(req); continue; @@ -3760,16 +3792,35 @@ static void *owngil_context_thread_main(void *arg) { ctx->reactor_buffer_ptr = NULL; ctx->local_env_ptr = NULL; - /* Signal completion */ - pthread_mutex_lock(&req->mutex); - atomic_store(&req->completed, true); - pthread_cond_signal(&req->cond); - pthread_mutex_unlock(&req->mutex); + /* Deliver result - async (message to caller) or blocking (condvar) */ + if (req->async_mode) { + enif_clear_env(ctx->msg_env); + ERL_NIF_TERM result_msg = enif_make_tuple3(ctx->msg_env, + enif_make_atom(ctx->msg_env, "py_result"), + enif_make_copy(ctx->msg_env, req->request_id), + req->result_env ? enif_make_copy(ctx->msg_env, req->result) + : enif_make_tuple2(ctx->msg_env, + enif_make_atom(ctx->msg_env, "error"), + enif_make_atom(ctx->msg_env, "no_result"))); + enif_send(NULL, &req->caller_pid, ctx->msg_env, result_msg); + } else { + pthread_mutex_lock(&req->mutex); + atomic_store(&req->completed, true); + pthread_cond_signal(&req->cond); + pthread_mutex_unlock(&req->mutex); + } /* Release queue's reference to request */ ctx_request_release(req); } + /* Refuse new scheduler attachments to our event loop and wait for the + * ones in flight (process_ready_tasks). Must run with our GIL released, + * since an attached thread needs it to finish. */ + if (ctx->event_loop != NULL) { + event_loop_detach_interpreter((erlang_event_loop_t *)ctx->event_loop); + } + /* Cleanup: acquire our OWN_GIL and destroy interpreter */ PyEval_RestoreThread(ctx->own_gil_tstate); Py_XDECREF(ctx->module_cache); @@ -3779,6 +3830,15 @@ static void *owngil_context_thread_main(void *arg) { ctx->locals = NULL; ctx->module_cache = NULL; + /* Drop our reference on the interpreter's event loop before the + * interpreter goes away (the loop destructor skips Python cleanup for + * subinterpreter loops). Detaching was done above with the GIL released. */ + if (ctx->event_loop != NULL) { + void *loop = ctx->event_loop; + ctx->event_loop = NULL; + enif_release_resource(loop); + } + /* End interpreter - this releases our GIL and cleans up */ PyInterpreterState *ended_interp = ctx->own_gil_interp; Py_EndInterpreter(ctx->own_gil_tstate); @@ -4539,6 +4599,7 @@ static int owngil_context_init(py_context_t *ctx) { ctx->uses_own_gil = true; ctx->own_gil_tstate = NULL; ctx->own_gil_interp = NULL; + ctx->event_loop = NULL; /* Initialize worker thread state */ atomic_store(&ctx->worker_running, false); @@ -4777,6 +4838,7 @@ static ERL_NIF_TERM nif_context_create(ErlNifEnv *env, int argc, const ERL_NIF_T ctx->uses_own_gil = false; ctx->own_gil_tstate = NULL; ctx->own_gil_interp = NULL; + ctx->event_loop = NULL; if (use_owngil) { /* OWN_GIL mode: create dedicated pthread with OWN_GIL subinterpreter */ @@ -5347,8 +5409,8 @@ static ERL_NIF_TERM nif_context_call_async(ErlNifEnv *env, int argc, const ERL_N /* RequestId is argv[2] - can be any term */ ERL_NIF_TERM request_id = argv[2]; - /* Worker thread mode: dispatch async */ - if (ctx->uses_worker_thread) { + /* Dedicated thread (worker or OWN_GIL): dispatch async */ + if (ctx_uses_async_thread(ctx)) { /* Build request tuple: {Module, Func, Args, Kwargs} */ ERL_NIF_TERM kwargs = (argc > 6 && enif_is_map(env, argv[6])) ? argv[6] : enif_make_new_map(env); @@ -5397,8 +5459,8 @@ static ERL_NIF_TERM nif_context_eval_async(ErlNifEnv *env, int argc, const ERL_N /* RequestId is argv[2] - can be any term */ ERL_NIF_TERM request_id = argv[2]; - /* Worker thread mode: dispatch async */ - if (ctx->uses_worker_thread) { + /* Dedicated thread (worker or OWN_GIL): dispatch async */ + if (ctx_uses_async_thread(ctx)) { /* Build request tuple: {Code, Locals} */ ERL_NIF_TERM locals = (argc > 4 && enif_is_map(env, argv[4])) ? argv[4] : enif_make_new_map(env); @@ -5443,8 +5505,8 @@ static ERL_NIF_TERM nif_context_exec_async(ErlNifEnv *env, int argc, const ERL_N /* RequestId is argv[2] - can be any term */ ERL_NIF_TERM request_id = argv[2]; - /* Worker thread mode: dispatch async */ - if (ctx->uses_worker_thread) { + /* Dedicated thread (worker or OWN_GIL): dispatch async */ + if (ctx_uses_async_thread(ctx)) { return dispatch_to_worker_thread_async(env, ctx, CTX_REQ_EXEC, argv[3], caller_pid, request_id, NULL); } @@ -5487,7 +5549,7 @@ static ERL_NIF_TERM nif_context_call_with_env_async(ErlNifEnv *env, int argc, return make_error(env, "invalid_env"); } - if (!ctx->uses_worker_thread) { + if (!ctx_uses_async_thread(ctx)) { return make_error(env, "async_requires_worker_thread"); } @@ -5532,7 +5594,7 @@ static ERL_NIF_TERM nif_context_eval_with_env_async(ErlNifEnv *env, int argc, return make_error(env, "invalid_env"); } - if (!ctx->uses_worker_thread) { + if (!ctx_uses_async_thread(ctx)) { return make_error(env, "async_requires_worker_thread"); } @@ -5573,7 +5635,7 @@ static ERL_NIF_TERM nif_context_exec_with_env_async(ErlNifEnv *env, int argc, return make_error(env, "invalid_env"); } - if (!ctx->uses_worker_thread) { + if (!ctx_uses_async_thread(ctx)) { return make_error(env, "async_requires_worker_thread"); } @@ -8146,6 +8208,7 @@ static ErlNifFunc nif_funcs[] = { /* FD lifecycle management (uvloop-like API) */ {"handle_fd_event", 2, nif_handle_fd_event, 0}, {"handle_fd_event_and_reselect", 2, nif_handle_fd_event_and_reselect, 0}, + {"fd_arm", 2, nif_fd_arm, 0}, {"stop_reader", 1, nif_stop_reader, 0}, {"start_reader", 1, nif_start_reader, 0}, {"stop_writer", 1, nif_stop_writer, 0}, diff --git a/c_src/py_nif.h b/c_src/py_nif.h index 002418a..4f2e37c 100644 --- a/c_src/py_nif.h +++ b/c_src/py_nif.h @@ -1012,6 +1012,10 @@ struct py_context { /** @brief Interpreter state for OWN_GIL subinterpreter */ PyInterpreterState *own_gil_interp; + + /** @brief Default ErlangEventLoop of the subinterpreter (kept resource, + * set by the context thread, read by nif_context_get_event_loop) */ + void *event_loop; #else /** @brief Worker thread state (non-subinterp mode, kept for compatibility) */ PyThreadState *thread_state; diff --git a/docs/asyncio.md b/docs/asyncio.md index 28b30e2..b4f2990 100644 --- a/docs/asyncio.md +++ b/docs/asyncio.md @@ -1479,6 +1479,7 @@ process_batch(Items) -> ## See Also +- [Worker Loops](workers.md) - Run the loop forever in a context, inject coroutines from Erlang, serve on sockets Erlang owns - [Reactor](reactor.md) - Low-level FD-based protocol handling - [Security](security.md) - Sandbox and blocked operations - [Threading](threading.md) - For `erlang.async_call()` in asyncio contexts diff --git a/docs/interrupts.md b/docs/interrupts.md index 63ecb65..84d4465 100644 --- a/docs/interrupts.md +++ b/docs/interrupts.md @@ -75,3 +75,9 @@ ok = py_context:destroy(Ctx). context that just finished one call and started another stops the new one. - `py:call/3,4` and `py:eval/1,2` use `infinity` by default. Pass an explicit timeout, or use `py:interrupt/1`, if you need a bound. +- A context running a worker loop (`py_context:start_loop/1`) refuses + `call/eval/exec` with `{error, loop_running}` for this reason: a timed-out + call would interrupt the loop. `py_context:interrupt/1` on such a context + ends the loop with `{py_loop_exit, Ctx, {error, interrupted}}`; use + `py_context:stop_loop/1,2` for a cooperative stop. See [Worker + Loops](workers.md). diff --git a/docs/owngil_internals.md b/docs/owngil_internals.md index 03ee606..8d3b2e5 100644 --- a/docs/owngil_internals.md +++ b/docs/owngil_internals.md @@ -46,8 +46,9 @@ All major erlang_python features work with OWN_GIL mode: | PIDs (`erlang.Pid`) | Full | Round-trip serialization | | Send (`erlang.send`) | Full | Fire-and-forget messaging | | Reactor (`erlang.reactor`) | Full | FD-based protocols | -| Async Tasks | Full | `py_event_loop:create_task` | -| Asyncio | Full | `asyncio.sleep`, `gather`, etc. | +| Async Tasks | Full | `py_event_loop:create_task`, `py_context:submit` | +| Asyncio | Full | Own `ErlangEventLoop` per context: `erlang.run`, `create_server`, channels | +| Worker loops | Full | `py_context:start_loop`, see [Worker Loops](workers.md) | | Process-local envs | Full | Namespace isolation | ## Architecture @@ -302,7 +303,9 @@ py_env_resource_dtor(env, res) { ## Reactor / Event Loop Integration -OWN_GIL contexts support the reactor pattern for I/O-driven protocols. The `py_event_loop` module is registered in each OWN_GIL subinterpreter during startup. +OWN_GIL contexts support the reactor pattern for I/O-driven protocols. The `py_event_loop` module is registered in each OWN_GIL subinterpreter during startup, together with a default `ErlangEventLoop` for that interpreter. `py_context` starts a dedicated `py_event_worker` per OWN_GIL context and points that loop at it, so fd readiness and timers of one context never go through another context's process or through the main loop's worker. + +Requests to the OWN_GIL thread go through the same async queue as worker mode (`context_*_async` NIFs): the calling Erlang process waits in a `receive`, no dirty scheduler is held and there is no 30 s cap on a call. From an Erlang scheduler, `process_ready_tasks` attaches a temporary thread state to the subinterpreter to inject coroutines into its loop (`py_context:submit`); the context thread waits for those to detach before `Py_EndInterpreter`. ### Why Event Loop Registration Matters diff --git a/docs/workers.md b/docs/workers.md new file mode 100644 index 0000000..85fea63 --- /dev/null +++ b/docs/workers.md @@ -0,0 +1,192 @@ +# Worker Loops + +This guide covers running a long-lived asyncio event loop inside a Python +context and driving it from Erlang: starting and stopping the loop, injecting +coroutines into it, and serving TCP or UDP on sockets that Erlang owns. You +need it when you want Python servers or background async work to run as +supervised workers inside the VM, the way gunicorn runs worker processes, with +Erlang as the arbiter. + +## What a worker loop is + +An owngil `py_context` has its own interpreter, its own GIL, its own thread and +its own `ErlangEventLoop`. `py_context:start_loop/1` runs that loop forever on +the context thread. From then on: + +- `py_context:submit/4,5` schedules a coroutine or a plain function on the loop + and returns a task reference; the result arrives as `{async_result, TaskRef, + {ok, Value} | {error, Reason}}` (use `py_event_loop:await/1,2` or + `submit_await/4,5,6`). +- fds registered by the loop (servers, connections, channels) are served by + the loop thread; readiness comes through the context's own `py_event_worker` + process. +- `py_context:stop_loop/1,2` stops it from inside, or interrupts the thread if + it does not exit within the grace period. The owner receives + `{py_loop_exit, Ctx, Result}` when the loop ends, for any reason. + +Worker contexts get the same API on the shared main interpreter loop, which +allows one running `ErlangEventLoop` per interpreter: use owngil (Python +3.14+) for several workers. + +## Serve TCP on a socket Erlang owns + +Bind once in Erlang, duplicate the listen fd for each worker with +`py:dup_fd/1`, and let each worker accept on its copy. + +```python +# myapp.py, importable by the workers +import asyncio +import erlang + +class Echo(asyncio.Protocol): + def connection_made(self, transport): + self.transport = transport + + def data_received(self, data): + self.transport.write(data) + +async def serve(listen_fd): + server = await erlang.server.serve(listen_fd, Echo) + return 'serving' +``` + +```erlang +{ok, LSock} = gen_tcp:listen(8000, [binary, {reuseaddr, true}, {backlog, 1024}]), +{ok, LFd} = inet:getfd(LSock), + +Workers = [begin + {ok, W} = py_context:new(#{mode => owngil, preload => <<"import myapp">>}), + ok = py_context:start_loop(W), + {ok, Dup} = py:dup_fd(LFd), + {ok, <<"serving">>} = py_context:submit_await(W, myapp, serve, [Dup]), + W +end || _ <- lists:seq(1, 4)]. +``` + +Every worker now accepts on the same socket; the kernel hands each connection +to one of them. Erlang keeps the listen socket open across worker restarts, so +replacing a worker never closes the port. + +For UDP, open the socket with `gen_udp`, dup its fd the same way and call +`erlang.server.serve(fd, MyDatagramProtocol, udp=True)`; on Linux use one +`SO_REUSEPORT` socket per worker instead of one shared fd, so the kernel +spreads flows. + +## Hand over one connection at a time + +When Erlang wants to decide per connection (routing, tenancy, or keeping some +connections in Erlang), accept in Erlang and adopt the fd in the worker. + +```python +async def adopt(fd): + await erlang.server.adopt(fd, Echo) + return 'adopted' +``` + +```erlang +{ok, Conn} = gen_tcp:accept(LSock), +{ok, Fd} = inet:getfd(Conn), +{ok, Dup} = py:dup_fd(Fd), +{ok, <<"adopted">>} = py_context:submit_await(Worker, myapp, adopt, [Dup]), +gen_tcp:close(Conn). %% Python owns the dup, Erlang drops its copy +``` + +## Inject work into a running loop + +`submit` targets module level functions (`Module:Func`), imported in the +worker (put entry points in a module, or register one in `sys.modules` from +`preload`). Coroutine functions are awaited, plain functions are called. + +```erlang +{ok, Ref} = py_context:submit(W, myapp, refresh_cache, [Key]), +%% ... other work ... +{ok, Result} = py_event_loop:await(Ref, 5000). + +%% or in one step +{ok, Result} = py_context:submit_await(W, myapp, refresh_cache, [Key], #{}, 5000). +``` + +Task start failures come back as errors, not silence: `{error, +function_not_found}` for a missing module or function, `{error, +args_conversion_failed}`, or the Python exception if the call itself raised. + +## Control from Erlang without submit + +A `py_channel` awaited inside the loop delivers Erlang messages to the running +loop with no polling; use it as the control plane of a worker (adopt this fd, +drain, report stats): + +```python +async def control(channel_ref): + ch = erlang.Channel(channel_ref) + async for msg in ch: + match msg: + case ('adopt', fd): + await erlang.server.adopt(fd, Echo) + case ('stop',): + return 'stopped' +``` + +```erlang +{ok, Ch} = py_channel:new(), +{ok, _} = py_context:submit(W, myapp, control, [Ch]), +ok = py_channel:send(Ch, {adopt, Dup}). +``` + +## Stop, restart, supervise + +```erlang +ok = py_context:stop_loop(W, 5000), %% cooperative, interrupt after 5 s +receive {py_loop_exit, W, Result} -> Result end, +ok = py_context:start_loop(W). %% same context, fresh loop +``` + +- `stop_loop` returns `ok` once the loop has exited, `{error, no_loop}` when + none runs, `{error, timeout}` if it survived the interrupt. +- `py_context:interrupt/1` ends the loop at once with `{py_loop_exit, W, + {error, interrupted}}`; a loop blocked in a C call (`time.sleep`, a numpy + kernel) exits when that call returns. +- `py_context:stop/1` on a looping context interrupts the loop first, then + destroys the context. +- If the owner process (the caller of `start_loop`, or `#{owner => Pid}`) dies, + the loop is stopped. +- Put workers under your own supervisor. `py_context` processes are + `temporary` under `py_context_sup`; a restart is `py_context:new/1` again, + `start_loop`, and a new dup of the listen fd. Use `memory_limit` in + `py_context:new/1` to cap a worker, and `py_nif:context_memory_usage/1` to + watch it. + +## Rules + +- While a loop runs, `py_context:call/eval/exec/call_method` on that context + return `{error, loop_running}`. The thread is busy in the loop, and a call + that timed out would interrupt it. Use `submit`. +- Pass fds you may close: `py:dup_fd/1` copies, `serve` and `adopt` wrap the + fd in a socket that owns it and close it when done. Never hand the original + fd of a live `gen_tcp` socket. +- One running `ErlangEventLoop` per interpreter: worker mode supports one + worker loop, owngil one per context. +- Modules used by `submit` must be importable in the worker; the `exec` + namespace of the context is not searched. + +## Numbers + +`examples/bench_worker_loop.erl` (Apple M4 Pro, OTP 29, Python 3.14, gen_tcp +clients in the same VM): + +| Measure | Result | +|---|---| +| connect + echo + close, one worker | about 15 000 conn/s | +| keep-alive echo, one worker, 50 connections | about 70 000 req/s | +| connect + echo + close, 2 to 8 workers on one listen fd | about 23 000 conn/s (client bound) | +| `submit_await` coroutine, one caller | about 25 us round trip | +| `submit_await` coroutine, 100 callers | about 4 us per op, 250 000 ops/s | +| `py_context:call` on an idle owngil context | about 12 us | +| Erlang accept + adopt vs Python accept | 13 500 vs 14 500 conn/s | + +## See also + +- [Asyncio](asyncio.md) for the ErlangEventLoop itself +- [Channels](channel.md) for the control plane +- [Interrupts](interrupts.md) for what an interrupt does to a loop +- [OWN_GIL Internals](owngil_internals.md) for the thread and interpreter model diff --git a/examples/bench_worker_loop.erl b/examples/bench_worker_loop.erl new file mode 100644 index 0000000..ae51dda --- /dev/null +++ b/examples/bench_worker_loop.erl @@ -0,0 +1,264 @@ +#!/usr/bin/env escript +%% -*- erlang -*- +%%! -pa _build/default/lib/erlang_python/ebin + +%%% @doc Benchmark for worker loops (py_context:start_loop/submit, erlang.server). +%%% +%%% Measures, on OWN_GIL contexts: +%%% 1. connections/s and requests/s of an echo server on one worker loop +%%% 2. scaling with 1, 2, 4, 8 workers accepting on one listen fd +%%% 3. submit round trip latency into a running loop, 1/10/100 callers, +%%% against py_context:call on an idle context +%%% 4. adopt (Erlang accepts, hands the fd over) vs Python side accept +%%% +%%% Clients are plain gen_tcp sockets in this VM, so the numbers include the +%%% client cost; use them to compare shapes, not as absolute server figures. +%%% +%%% Run with: +%%% rebar3 compile && escript examples/bench_worker_loop.erl + +-mode(compile). + +-define(HOST, {127, 0, 0, 1}). + +-define(PY, <<" +import asyncio, erlang, sys, types +m = types.ModuleType('bench_wl'); sys.modules['bench_wl'] = m + +class Echo(asyncio.Protocol): + def connection_made(self, t): self.t = t + def data_received(self, d): self.t.write(d) + +class EchoClose(asyncio.Protocol): + def connection_made(self, t): self.t = t + def data_received(self, d): self.t.write(d); self.t.close() + +_servers = {} +async def serve(fd, keepalive): + srv = await erlang.server.serve(fd, Echo if keepalive else EchoClose) + _servers[fd] = srv + return 'ok' +async def adopt(fd): + await erlang.server.adopt(fd, EchoClose) + return 'ok' +async def noop(): + return 1 +def sync_noop(): + return 1 +m.serve = serve; m.adopt = adopt; m.noop = noop; m.sync_noop = sync_noop +">>). + +main(_Args) -> + io:format("~n"), + io:format("========================================================~n"), + io:format(" Worker loop benchmark~n"), + io:format("========================================================~n~n"), + {ok, _} = application:ensure_all_started(erlang_python), + print_system_info(), + case py_nif:owngil_supported() of + true -> + bench_single_worker(), + bench_scaling(), + bench_submit_latency(), + bench_adopt_vs_accept(); + false -> + io:format("~n[ERROR] worker loops need OWN_GIL (Python 3.14+)~n~n") + end, + halt(0). + +print_system_info() -> + io:format("System Information~n"), + io:format("------------------~n"), + io:format(" Erlang/OTP: ~s~n", [erlang:system_info(otp_release)]), + io:format(" Schedulers: ~p~n", [erlang:system_info(schedulers)]), + {ok, PyVer} = py:version(), + io:format(" Python: ~s~n", [PyVer]), + io:format("~n"). + +%% ============================================================================ +%% 1. Single worker: connections/s (connect, echo, close) and requests/s on +%% keep-alive connections +%% ============================================================================ + +bench_single_worker() -> + io:format("1. Single worker echo server~n"), + W = worker(), + {LSock, Port, LFd} = listen(), + {ok, Dup} = py:dup_fd(LFd), + {ok, <<"ok">>} = py_context:submit_await(W, bench_wl, serve, [Dup, false]), + N = 5000, + {Ms, Fails} = timed(fun() -> parallel_conns(Port, N, 50) end), + io:format(" connect+echo+close: ~p conns in ~p ms = ~p conn/s (failed ~p)~n", + [N, Ms, N * 1000 div max(1, Ms), Fails]), + ok = py_context:stop_loop(W), + py_context:stop(W), + gen_tcp:close(LSock), + + W2 = worker(), + {LSock2, Port2, LFd2} = listen(), + {ok, Dup2} = py:dup_fd(LFd2), + {ok, <<"ok">>} = py_context:submit_await(W2, bench_wl, serve, [Dup2, true]), + Conns = 50, + Reqs = 20000, + {Ms2, _} = timed(fun() -> keepalive_requests(Port2, Conns, Reqs div Conns) end), + io:format(" keep-alive echo: ~p reqs on ~p conns in ~p ms = ~p req/s~n", + [Reqs, Conns, Ms2, Reqs * 1000 div max(1, Ms2)]), + ok = py_context:stop_loop(W2), + py_context:stop(W2), + gen_tcp:close(LSock2), + io:format("~n"). + +%% ============================================================================ +%% 2. Scaling: N workers accepting on one listen fd +%% ============================================================================ + +bench_scaling() -> + io:format("2. Scaling: connect+echo+close, workers accepting on one listen fd~n"), + io:format(" ~-8s ~12s ~12s~n", ["workers", "conn/s", "ms"]), + N = 8000, + lists:foreach(fun(Workers) -> + Ws = [worker() || _ <- lists:seq(1, Workers)], + {LSock, Port, LFd} = listen(), + [begin + {ok, Dup} = py:dup_fd(LFd), + {ok, <<"ok">>} = py_context:submit_await(W, bench_wl, serve, [Dup, false]) + end || W <- Ws], + {Ms, _} = timed(fun() -> parallel_conns(Port, N, 100) end), + io:format(" ~-8w ~12w ~12w~n", [Workers, N * 1000 div max(1, Ms), Ms]), + [ok = py_context:stop_loop(W) || W <- Ws], + [py_context:stop(W) || W <- Ws], + gen_tcp:close(LSock) + end, [1, 2, 4, 8]), + io:format("~n"). + +%% ============================================================================ +%% 3. submit latency vs py_context:call +%% ============================================================================ + +bench_submit_latency() -> + io:format("3. Round trip latency~n"), + io:format(" ~-52s ~10s ~12s~n", ["path", "us/op", "ops/s"]), + {ok, Idle} = py_context:new(#{mode => owngil, preload => ?PY}), + N = 5000, + {MsCall, _} = timed(fun() -> + [{ok, 1} = py_context:call(Idle, bench_wl, sync_noop, []) || _ <- lists:seq(1, N)] + end), + row("py_context:call, idle owngil ctx", N, MsCall), + {MsSubIdle, _} = timed(fun() -> + [{ok, 1} = py_context:submit_await(Idle, bench_wl, sync_noop, []) || _ <- lists:seq(1, N)] + end), + row("submit_await sync fn, idle ctx", N, MsSubIdle), + py_context:stop(Idle), + + W = worker(), + lists:foreach(fun(Callers) -> + Per = N div Callers, + {Ms, _} = timed(fun() -> + Self = self(), + [spawn_link(fun() -> + [{ok, 1} = py_context:submit_await(W, bench_wl, noop, []) || _ <- lists:seq(1, Per)], + Self ! done + end) || _ <- lists:seq(1, Callers)], + [receive done -> ok end || _ <- lists:seq(1, Callers)] + end), + row(io_lib:format("submit_await coroutine, running loop, ~p callers", [Callers]), N, Ms) + end, [1, 10, 100]), + ok = py_context:stop_loop(W), + py_context:stop(W), + io:format("~n"). + +row(Label, N, Ms) -> + Us = Ms * 1000 / max(1, N), + io:format(" ~-52s ~10.1f ~12w~n", [Label, Us, N * 1000 div max(1, Ms)]). + +%% ============================================================================ +%% 4. adopt (Erlang accepts) vs Python accept +%% ============================================================================ + +bench_adopt_vs_accept() -> + io:format("4. Per connection: Erlang accept + adopt vs Python accept~n"), + N = 3000, + W = worker(), + {LSock, Port, LFd} = listen(), + {ok, Dup} = py:dup_fd(LFd), + {ok, <<"ok">>} = py_context:submit_await(W, bench_wl, serve, [Dup, false]), + {MsPy, _} = timed(fun() -> parallel_conns(Port, N, 50) end), + io:format(" Python accept: ~p conn/s~n", [N * 1000 div max(1, MsPy)]), + ok = py_context:stop_loop(W), + py_context:stop(W), + gen_tcp:close(LSock), + + W2 = worker(), + {LSock2, Port2, _} = listen(), + Acceptor = spawn_link(fun() -> acceptor(LSock2, W2) end), + {MsAd, _} = timed(fun() -> parallel_conns(Port2, N, 50) end), + io:format(" Erlang accept+adopt: ~p conn/s~n", [N * 1000 div max(1, MsAd)]), + unlink(Acceptor), exit(Acceptor, kill), + ok = py_context:stop_loop(W2), + py_context:stop(W2), + gen_tcp:close(LSock2), + io:format("~n"). + +acceptor(LSock, W) -> + case gen_tcp:accept(LSock) of + {ok, Conn} -> + {ok, Fd} = inet:getfd(Conn), + {ok, Dup} = py:dup_fd(Fd), + _ = py_context:submit(W, bench_wl, adopt, [Dup]), + gen_tcp:close(Conn), + acceptor(LSock, W); + _ -> + ok + end. + +%% ============================================================================ +%% Helpers +%% ============================================================================ + +worker() -> + {ok, W} = py_context:new(#{mode => owngil, preload => ?PY}), + ok = py_context:start_loop(W), + W. + +listen() -> + {ok, LSock} = gen_tcp:listen(0, [binary, {ip, ?HOST}, {active, false}, {backlog, 1024}]), + {ok, Port} = inet:port(LSock), + {ok, LFd} = inet:getfd(LSock), + {LSock, Port, LFd}. + +timed(Fun) -> + T0 = erlang:monotonic_time(millisecond), + R = Fun(), + {erlang:monotonic_time(millisecond) - T0, R}. + +parallel_conns(Port, N, Clients) -> + Self = self(), + Per = N div Clients, + [spawn_link(fun() -> + Fails = length([bad || _ <- lists:seq(1, Per), roundtrip(Port) =/= ok]), + Self ! {done, Fails} + end) || _ <- lists:seq(1, Clients)], + lists:sum([receive {done, F} -> F after 120000 -> Per end || _ <- lists:seq(1, Clients)]). + +roundtrip(Port) -> + case gen_tcp:connect(?HOST, Port, [binary, {active, false}], 5000) of + {ok, S} -> + ok = gen_tcp:send(S, <<"x">>), + R = gen_tcp:recv(S, 0, 5000), + gen_tcp:close(S), + case R of {ok, <<"x">>} -> ok; _ -> bad end; + _ -> + bad + end. + +keepalive_requests(Port, Conns, PerConn) -> + Self = self(), + [spawn_link(fun() -> + {ok, S} = gen_tcp:connect(?HOST, Port, [binary, {active, false}], 5000), + [begin ok = gen_tcp:send(S, <<"ping">>), {ok, <<"ping">>} = gen_tcp:recv(S, 4, 5000) end + || _ <- lists:seq(1, PerConn)], + gen_tcp:close(S), + Self ! done + end) || _ <- lists:seq(1, Conns)], + [receive done -> ok after 120000 -> ok end || _ <- lists:seq(1, Conns)], + ok. diff --git a/priv/_erlang_impl/__init__.py b/priv/_erlang_impl/__init__.py index 05abea5..0b69096 100644 --- a/priv/_erlang_impl/__init__.py +++ b/priv/_erlang_impl/__init__.py @@ -65,6 +65,7 @@ from . import _reactor as reactor from . import _channel as channel from . import _byte_channel as byte_channel +from . import _server as server from ._channel import Channel, reply, ChannelClosed from ._byte_channel import ByteChannel, ByteChannelClosed @@ -88,6 +89,7 @@ 'byte_channel', 'ByteChannel', 'ByteChannelClosed', + 'server', 'atom', ] @@ -165,6 +167,42 @@ def new_event_loop() -> ErlangEventLoop: return ErlangEventLoop() +def _run_loop_forever(notify_pid=None): + """Run an ErlangEventLoop on this thread until it is stopped. + + Entry point of py_context:start_loop/1: the context thread stays here + while Erlang injects coroutines with py_context:submit/4 (or the fd + events fire). notify_pid receives {py_loop_started} once the loop is + running. Returns 'stopped' when _stop_loop() ran, propagates + KeyboardInterrupt when py_context:interrupt/1 was used. + """ + loop = new_event_loop() + asyncio.set_event_loop(loop) + if notify_pid is not None: + import erlang as _erlang + + def _started(): + try: + _erlang.send(notify_pid, (atom('py_loop_started'),)) + except Exception: + pass + loop.call_soon(_started) + try: + loop.run_forever() + finally: + try: + asyncio.set_event_loop(None) + finally: + loop.close() + return 'stopped' + + +async def _stop_loop(): + """Stop the loop running _run_loop_forever() from inside it.""" + asyncio.get_running_loop().stop() + return 'stopping' + + def run(main, *, debug=None, **run_kwargs): """Run a coroutine using Erlang event loop. diff --git a/priv/_erlang_impl/_loop.py b/priv/_erlang_impl/_loop.py index 8be028b..e9f0b23 100644 --- a/priv/_erlang_impl/_loop.py +++ b/priv/_erlang_impl/_loop.py @@ -49,6 +49,15 @@ EVENT_TYPE_TIMER = 3 +class _PooledHandle(events.Handle): + """Handle recycled through ErlangEventLoop._handle_pool. + + Only fd event dispatch creates these; they never leave the loop, so no + one can cancel one after it ran and hit its next occupant. + """ + __slots__ = () + + class ErlangEventLoop(asyncio.AbstractEventLoop): """asyncio event loop backed by Erlang's scheduler. @@ -227,6 +236,14 @@ def run_forever(self): self._running = True # Don't reset _stopping here - honor stop() called before run_forever() + # Tell the NIF this loop consumes its own events from now on + set_running = getattr(self._pel, '_set_running_for', None) + if set_running is not None: + try: + set_running(self._loop_capsule, True) + except Exception: + set_running = None + # Register as the running loop old_running_loop = events._get_running_loop() events._set_running_loop(self) @@ -237,6 +254,11 @@ def run_forever(self): events._set_running_loop(old_running_loop) self._stopping = False self._running = False + if set_running is not None: + try: + set_running(self._loop_capsule, False) + except Exception: + pass self._thread_id = None self._set_coroutine_origin_tracking(False) @@ -314,11 +336,18 @@ def close(self): self._timer_refs.clear() self._handle_to_callback_id.clear() - # Remove all readers/writers + # Remove all readers/writers, then release fd resources kept alive by + # _stop_reading/_stop_writing for transports that were never closed for fd in list(self._readers.keys()): self.remove_reader(fd) for fd in list(self._writers.keys()): self.remove_writer(fd) + for fd, fd_key in list(self._fd_resources.items()): + try: + self._pel._release_fd_resource(fd_key) + except Exception: + pass + self._fd_resources.clear() # Clear signal handlers self._signal_handlers.clear() @@ -372,7 +401,14 @@ def call_soon(self, callback, *args, context=None): Uses handle pooling (uvloop-style) to reduce allocations. """ self._check_closed() - handle = self._get_handle(callback, args, context) + # A handle handed to the caller must not come from the pool: asyncio + # code keeps call_soon/call_later handles and cancels them after they + # ran (asyncio.sleep does), which would cancel whatever callback the + # recycled handle carries by then. Pooling stays for the fd event + # handles created in _dispatch, which never leave the loop. + if context is None: + context = contextvars.copy_context() + handle = events.Handle(callback, args, self, context) self._ready_append(handle) return handle @@ -400,10 +436,15 @@ def call_at(self, when, callback, *args, context=None): """Schedule a callback to be called at a specific time.""" self._check_closed() - # For zero or past times, schedule immediately via call_soon + # For zero or past times, run at the next iteration. Return a real + # TimerHandle (never pooled): callers cancel it after it ran. delay_ms = int((when - self.time()) * 1000) if delay_ms <= 0: - return self.call_soon(callback, *args, context=context) + if context is None: + context = contextvars.copy_context() + handle = events.TimerHandle(when, callback, args, self, context) + self._ready_append(handle) + return handle callback_id = self._next_id() @@ -590,6 +631,76 @@ def remove_writer(self, fd): return True + # ------------------------------------------------------------------------ + # Transport helpers: stop reading/writing without giving up the fd + # resource, and hand the socket to the NIF for closing. + # + # remove_reader/remove_writer release the fd resource as soon as neither + # side is active, which issues ERL_NIF_SELECT_STOP. If the socket is then + # closed from Python before the stop completes, the fd sits in the poll + # set while its number gets reused. Transports therefore only clear the + # callbacks here and let _close_socket transfer the fd to the NIF, which + # closes it from the stop callback. + # ------------------------------------------------------------------------ + + def _stop_reading(self, fd): + entry = self._readers.pop(fd, None) + if entry is None: + return False + self._callbacks_by_cid.pop(entry[2], None) + fd_key = self._fd_resources.get(fd) + if fd_key is not None: + try: + self._pel._clear_fd_read(fd_key) + except Exception: + pass + return True + + def _stop_writing(self, fd): + entry = self._writers.pop(fd, None) + if entry is None: + return False + self._callbacks_by_cid.pop(entry[2], None) + fd_key = self._fd_resources.get(fd) + if fd_key is not None: + try: + self._pel._clear_fd_write(fd_key) + except Exception: + pass + return True + + def _close_socket(self, sock): + """Close a socket that may still be registered with enif_select. + + Clears any reader/writer, detaches the fd from the socket object and + lets the NIF close it once the select stop has completed. Sockets the + loop never registered are closed directly. + """ + try: + fd = sock.fileno() + except (OSError, ValueError): + return + if fd is None or fd < 0: + return + self._stop_reading(fd) + self._stop_writing(fd) + fd_key = self._fd_resources.pop(fd, None) + if fd_key is None: + sock.close() + return + try: + sock.detach() + except OSError: + pass + try: + self._pel._release_fd_resource(fd_key, True) + except Exception: + # NIF unavailable (mock module or shutdown): close it ourselves + try: + os.close(fd) + except OSError: + pass + # ======================================================================== # Socket operations # ======================================================================== @@ -1154,7 +1265,7 @@ def _get_handle(self, callback, args, context=None): handle._context = context return handle except IndexError: - return events.Handle(callback, args, self, context) + return _PooledHandle(callback, args, self, context) def _return_handle(self, handle): """Return a Handle to the pool for reuse. @@ -1166,8 +1277,9 @@ def _return_handle(self, handle): If the TimerHandle is recycled and reused for another callback, the cancel() call will incorrectly cancel the new callback. """ - # Don't pool TimerHandle - asyncio.sleep holds a reference and cancels it - if isinstance(handle, events.TimerHandle): + # Only handles created by _dispatch are recycled (see _PooledHandle); + # call_soon/call_at handles are held by callers who may cancel them + if type(handle) is not _PooledHandle: return if len(self._handle_pool) < self._handle_pool_max: @@ -1230,6 +1342,9 @@ def __init__(self): class _MockNifModule: """Mock NIF module for testing without actual Erlang integration.""" + def __init__(self): + self._fd_by_key = {} + def _is_initialized(self): return True @@ -1260,6 +1375,7 @@ def _wakeup_for(self, capsule): def _add_reader_for(self, capsule, fd, callback_id): capsule._counter += 1 capsule.readers[fd] = (callback_id, capsule._counter) + self._fd_by_key[capsule._counter] = fd return capsule._counter def _remove_reader_for(self, capsule, fd_key): @@ -1271,6 +1387,7 @@ def _remove_reader_for(self, capsule, fd_key): def _add_writer_for(self, capsule, fd, callback_id): capsule._counter += 1 capsule.writers[fd] = (callback_id, capsule._counter) + self._fd_by_key[capsule._counter] = fd return capsule._counter def _remove_writer_for(self, capsule, fd_key): @@ -1295,9 +1412,16 @@ def _clear_fd_write(self, fd_key): """Clear write monitoring on fd_resource.""" pass - def _release_fd_resource(self, fd_key): - """Release fd_resource.""" - pass + def _release_fd_resource(self, fd_key, take_ownership=False): + """Release fd_resource. With take_ownership the fd is closed here, + since there is no NIF to close it from a select stop callback.""" + if take_ownership: + fd = self._fd_by_key.pop(fd_key, None) + if fd is not None: + try: + os.close(fd) + except OSError: + pass def _schedule_timer_for(self, capsule, delay_ms, callback_id): return callback_id diff --git a/priv/_erlang_impl/_server.py b/priv/_erlang_impl/_server.py new file mode 100644 index 0000000..2a6ccf5 --- /dev/null +++ b/priv/_erlang_impl/_server.py @@ -0,0 +1,82 @@ +# Copyright 2026 Benoit Chesneau +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Serve on fds handed over by Erlang. + +Erlang owns the listen socket (gen_tcp or the socket module), duplicates its +fd with py:dup_fd/1 for each worker context, and each worker calls serve() +on its copy from a coroutine scheduled with py_context:submit/4. Accepted +connections can also be handed over one by one with adopt(). + + async def main(listen_fd): + server = await erlang.server.serve(listen_fd, EchoProtocol) + await server.serve_forever() + +The fd passed in must be one this interpreter may close: serve() and adopt() +wrap it in a socket object that owns it. +""" + +import asyncio +import socket + +__all__ = ['serve', 'adopt', 'stop_serving'] + + +def _socket_from_fd(fd, *, udp=False): + """Wrap an fd owned by Python in a non-blocking socket object.""" + if not isinstance(fd, int) or fd < 0: + raise ValueError(f"invalid fd: {fd!r}") + kind = socket.SOCK_DGRAM if udp else socket.SOCK_STREAM + try: + sock = socket.socket(fileno=fd) + except OSError as exc: + raise OSError(exc.errno, f"cannot adopt fd {fd}: {exc.strerror}") from exc + if sock.type != kind: + sock.detach() + raise ValueError(f"fd {fd} is not a {'datagram' if udp else 'stream'} socket") + sock.setblocking(False) + return sock + + +async def serve(listen_fd, protocol_factory, *, udp=False, backlog=100): + """Serve on a listen fd (TCP/Unix) or a bound datagram fd (UDP). + + Returns an asyncio Server for stream sockets, or the datagram transport + for udp=True. The caller keeps the loop alive (serve_forever, or the + loop started with py_context:start_loop/1). + """ + loop = asyncio.get_running_loop() + sock = _socket_from_fd(listen_fd, udp=udp) + if udp: + transport, _protocol = await loop.create_datagram_endpoint( + protocol_factory, sock=sock) + return transport + return await loop.create_server(protocol_factory, sock=sock, backlog=backlog) + + +async def adopt(fd, protocol_factory): + """Take over an accepted connection whose fd Erlang handed to us. + + Returns (transport, protocol) like loop.create_connection. + """ + loop = asyncio.get_running_loop() + sock = _socket_from_fd(fd) + return await loop.create_connection(protocol_factory, sock=sock) + + +async def stop_serving(server, *, wait_closed=True): + """Stop accepting on a Server or close a datagram transport.""" + server.close() + if wait_closed and hasattr(server, 'wait_closed'): + await server.wait_closed() diff --git a/priv/_erlang_impl/_transport.py b/priv/_erlang_impl/_transport.py index 8f61000..6bac533 100644 --- a/priv/_erlang_impl/_transport.py +++ b/priv/_erlang_impl/_transport.py @@ -69,6 +69,16 @@ async def _start(self): self._loop.call_soon(self._protocol.connection_made, self) self._loop.add_reader(self._fileno, self._read_ready) + def __del__(self): + # An abandoned transport must still hand its fd back to the loop, so + # the enif_select registration is stopped before the number is reused. + sock = getattr(self, '_sock', None) + if sock is not None and sock.fileno() >= 0: + try: + self._loop._close_socket(sock) + except Exception: + pass + # Maximum reads per callback to avoid starving other events _max_reads_per_call = 16 @@ -105,7 +115,7 @@ def _read_ready(self): self._protocol.data_received(data) else: # Connection closed (EOF received) - self._loop.remove_reader(self._fileno) + self._loop._stop_reading(self._fileno) keep_open = self._protocol.eof_received() # If eof_received returns False/None, close the transport if not keep_open: @@ -159,7 +169,7 @@ def _write_ready_cb(self): for _ in range(self._max_writes_per_call): remaining = len(self._buffer) - self._buffer_offset if remaining <= 0: - self._loop.remove_writer(self._fileno) + self._loop._stop_writing(self._fileno) if self._closing: self._call_connection_lost(None) return @@ -176,11 +186,11 @@ def _write_ready_cb(self): if exc.errno == errno.EBADF: self._conn_lost += 1 return - self._loop.remove_writer(self._fileno) + self._loop._stop_writing(self._fileno) self._fatal_error(exc, 'Fatal write error') return except Exception as exc: - self._loop.remove_writer(self._fileno) + self._loop._stop_writing(self._fileno) self._fatal_error(exc, 'Fatal write error') return @@ -192,7 +202,7 @@ def _write_ready_cb(self): # Reset buffer when fully consumed self._buffer = self._buffer_factory() self._buffer_offset = 0 - self._loop.remove_writer(self._fileno) + self._loop._stop_writing(self._fileno) if self._closing: self._call_connection_lost(None) @@ -203,7 +213,7 @@ def write_eof(self): self._closing = True # Check if no pending data (buffer fully consumed) if self._buffer_offset >= len(self._buffer): - self._loop.remove_reader(self._fileno) + self._loop._stop_reading(self._fileno) self._call_connection_lost(None) def can_write_eof(self): @@ -214,7 +224,7 @@ def close(self): if self._closing: return self._closing = True - self._loop.remove_reader(self._fileno) + self._loop._stop_reading(self._fileno) # Check if no pending data (buffer fully consumed) if self._buffer_offset >= len(self._buffer): self._conn_lost += 1 @@ -229,8 +239,10 @@ def _call_connection_lost(self, exc): try: self._protocol.connection_lost(exc) finally: + # The fd may still be in the BEAM poll set: let the loop hand it + # to the NIF, which closes it once the select stop completes. try: - self._sock.close() + self._loop._close_socket(self._sock) except OSError: pass @@ -274,8 +286,8 @@ def abort(self): """Close immediately.""" self._closing = True self._conn_lost += 1 - self._loop.remove_reader(self._fileno) - self._loop.remove_writer(self._fileno) + self._loop._stop_reading(self._fileno) + self._loop._stop_writing(self._fileno) self._call_connection_lost(None) def pause_reading(self): @@ -283,7 +295,7 @@ def pause_reading(self): if self._closing or self._paused: return self._paused = True - self._loop.remove_reader(self._fileno) + self._loop._stop_reading(self._fileno) def resume_reading(self): """Resume reading from the transport.""" @@ -333,6 +345,14 @@ async def _start(self): self._protocol.connection_made(self) self._loop.add_reader(self._fileno, self._read_ready) + def __del__(self): + sock = getattr(self, '_sock', None) + if sock is not None and sock.fileno() >= 0: + try: + self._loop._close_socket(sock) + except Exception: + pass + def _read_ready(self): """Called when data is available to read. @@ -432,7 +452,7 @@ def _write_ready(self): self._buffer.popleft() - self._loop.remove_writer(self._fileno) + self._loop._stop_writing(self._fileno) if self._closing: self._call_connection_lost(None) @@ -441,7 +461,7 @@ def close(self): if self._closing: return self._closing = True - self._loop.remove_reader(self._fileno) + self._loop._stop_reading(self._fileno) if not self._buffer: self._conn_lost += 1 self._call_connection_lost(None) @@ -455,8 +475,10 @@ def _call_connection_lost(self, exc): try: self._protocol.connection_lost(exc) finally: + # The fd may still be in the BEAM poll set: let the loop hand it + # to the NIF, which closes it once the select stop completes. try: - self._sock.close() + self._loop._close_socket(self._sock) except OSError: pass @@ -487,8 +509,8 @@ def abort(self): """Close immediately.""" self._closing = True self._conn_lost += 1 - self._loop.remove_reader(self._fileno) - self._loop.remove_writer(self._fileno) + self._loop._stop_reading(self._fileno) + self._loop._stop_writing(self._fileno) self._buffer.clear() self._call_connection_lost(None) @@ -540,8 +562,7 @@ def close(self): return self._serving = False for sock in self._sockets: - self._loop.remove_reader(sock.fileno()) - sock.close() + self._loop._close_socket(sock) self._sockets.clear() # Wake up waiters diff --git a/priv/tests/test_loop_helpers.py b/priv/tests/test_loop_helpers.py new file mode 100644 index 0000000..74e6150 --- /dev/null +++ b/priv/tests/test_loop_helpers.py @@ -0,0 +1,183 @@ +# Copyright 2026 Benoit Chesneau +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the worker loop entry points behind py_context:start_loop/1. + +_run_loop_forever() runs an ErlangEventLoop on the calling thread until +_stop_loop() (a coroutine scheduled on it) stops it. Runs inside the BEAM +through tests.ct_runner in an owngil context (its own interpreter, no other +ErlangEventLoop alive), so it exercises the real py_event_loop module. +""" + +import asyncio +import threading +import unittest + + +def _impl(): + # _run_loop_forever looks up new_event_loop in the _erlang_impl namespace, + # so that is the module to hook + import _erlang_impl + return _erlang_impl + + +class TestLoopHelpers(unittest.TestCase): + + def test_run_forever_returns_after_stop(self): + impl = _impl() + started = threading.Event() + result = {} + + def runner(): + # The loop must be created on the running thread; grab it once + # it is running to schedule the stop + orig = impl.new_event_loop + + def new_loop_hook(): + loop = orig() + result['loop'] = loop + started.set() + return loop + impl.new_event_loop = new_loop_hook + try: + result['ret'] = impl._run_loop_forever() + finally: + impl.new_event_loop = orig + + t = threading.Thread(target=runner) + t.start() + self.assertTrue(started.wait(5)) + loop = result['loop'] + # give run_forever a moment to start + for _ in range(100): + if loop.is_running(): + break + threading.Event().wait(0.01) + self.assertTrue(loop.is_running()) + loop.call_soon_threadsafe(loop.create_task, impl._stop_loop()) + t.join(5) + self.assertFalse(t.is_alive()) + self.assertEqual(result['ret'], 'stopped') + self.assertTrue(loop.is_closed()) + # the current event loop is cleared, a new one can be created + with self.assertRaises(RuntimeError): + asyncio.get_running_loop() + loop2 = impl.new_event_loop() + loop2.close() + + def test_stop_loop_outside_loop_raises(self): + impl = _impl() + coro = impl._stop_loop() + with self.assertRaises(RuntimeError): + coro.send(None) # no running loop + coro.close() + + def test_second_loop_while_running_fails_cleanly(self): + """One running ErlangEventLoop per interpreter: a second creation + raises while the first runs, and works again once it has stopped.""" + impl = _impl() + loop = impl.new_event_loop() + stopped = threading.Event() + + def runner(): + try: + loop.run_forever() + finally: + stopped.set() + + t = threading.Thread(target=runner) + t.start() + for _ in range(100): + if loop.is_running(): + break + threading.Event().wait(0.01) + self.assertTrue(loop.is_running()) + try: + with self.assertRaises(RuntimeError): + impl.new_event_loop() + finally: + loop.call_soon_threadsafe(loop.stop) + t.join(5) + loop.close() + self.assertTrue(stopped.is_set()) + again = impl.new_event_loop() + again.close() + + +if __name__ == '__main__': + unittest.main() + + +class TestHandleReuse(unittest.TestCase): + """call_soon/call_at handles are the caller's: cancelling one after it + ran must not touch a later callback (the handle pool used to recycle + them, so asyncio.sleep's cancel in its finally block killed whichever + callback had been given the recycled handle).""" + + def _loop(self): + impl = _impl() + return impl.new_event_loop() + + def test_cancel_after_run_does_not_kill_next_callback(self): + loop = self._loop() + seen = [] + try: + h1 = loop.call_soon(seen.append, 1) + loop.call_soon(loop.stop) + loop.run_forever() + self.assertEqual(seen, [1]) + # h1 ran; a stale cancel must be a no-op + h1.cancel() + loop.call_soon(seen.append, 2) + loop.call_soon(loop.stop) + loop.run_forever() + self.assertEqual(seen, [1, 2]) + finally: + loop.close() + + def test_zero_delay_call_later_returns_timer_handle(self): + loop = self._loop() + seen = [] + try: + h = loop.call_later(0, seen.append, 'x') + self.assertIsInstance(h, asyncio.TimerHandle) + loop.call_soon(loop.stop) + loop.run_forever() + self.assertEqual(seen, ['x']) + h.cancel() # after it ran, no effect on anything else + loop.call_soon(seen.append, 'y') + loop.call_soon(loop.stop) + loop.run_forever() + self.assertEqual(seen, ['x', 'y']) + finally: + loop.close() + + def test_many_sleeps_all_wake(self): + """500 concurrent sleep(0.001): every one comes back, whether the + delay rounds to a timer or to the next iteration.""" + loop = self._loop() + woke = [] + + async def one(i): + await asyncio.sleep(0.001) + woke.append(i) + + async def main(): + await asyncio.gather(*(one(i) for i in range(500))) + + try: + loop.run_until_complete(main()) + finally: + loop.close() + self.assertEqual(sorted(woke), list(range(500))) diff --git a/priv/tests/test_server.py b/priv/tests/test_server.py new file mode 100644 index 0000000..00d7ede --- /dev/null +++ b/priv/tests/test_server.py @@ -0,0 +1,168 @@ +# Copyright 2026 Benoit Chesneau +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for erlang.server (serve / adopt / stop_serving on handed-over fds). + +The fds come from sockets created here and detached, which is what +py:dup_fd/1 produces on the Erlang side: a descriptor Python may own. +""" + +import asyncio +import os +import socket +import unittest + +from . import _testbase as tb + + +def _server_module(): + try: + import erlang + if hasattr(erlang, 'server'): + return erlang.server + except ImportError: + pass + from _erlang_impl import _server + return _server + + +class Echo(asyncio.Protocol): + def connection_made(self, transport): + self.transport = transport + + def data_received(self, data): + self.transport.write(b'echo:' + data) + self.transport.close() + + +class _TestServe: + + def test_serve_tcp_on_listen_fd(self): + server_mod = _server_module() + lsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + lsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + lsock.bind(('127.0.0.1', 0)) + lsock.listen(16) + port = lsock.getsockname()[1] + fd = lsock.detach() # what py:dup_fd hands over + + async def main(): + server = await server_mod.serve(fd, Echo) + self.assertTrue(server.is_serving()) + client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client.setblocking(False) + await self.loop.sock_connect(client, ('127.0.0.1', port)) + await self.loop.sock_sendall(client, b'hi') + data = await self.loop.sock_recv(client, 1024) + client.close() + await server_mod.stop_serving(server) + self.assertFalse(server.is_serving()) + return data + + self.assertEqual(self.loop.run_until_complete(main()), b'echo:hi') + + def test_serve_udp_on_bound_fd(self): + server_mod = _server_module() + usock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + usock.bind(('127.0.0.1', 0)) + port = usock.getsockname()[1] + fd = usock.detach() + got = [] + + class UDP(asyncio.DatagramProtocol): + def connection_made(self, transport): + self.transport = transport + + def datagram_received(self, data, addr): + got.append(data) + self.transport.sendto(b'udp:' + data, addr) + + async def main(): + transport = await server_mod.serve(fd, UDP, udp=True) + client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + client.setblocking(False) + client.sendto(b'ping', ('127.0.0.1', port)) + fut = self.loop.create_future() + + def on_ready(): + try: + fut.set_result(client.recv(1024)) + except BlockingIOError: + return + self.loop.remove_reader(client.fileno()) + + self.loop.add_reader(client.fileno(), on_ready) + data = await asyncio.wait_for(fut, 5) + client.close() + await server_mod.stop_serving(transport, wait_closed=False) + return data + + self.assertEqual(self.loop.run_until_complete(main()), b'udp:ping') + self.assertEqual(got, [b'ping']) + + def test_adopt_connected_fd(self): + server_mod = _server_module() + a, b = socket.socketpair() + fd = a.detach() + b.setblocking(False) + + async def main(): + transport, protocol = await server_mod.adopt(fd, Echo) + self.assertIsInstance(protocol, Echo) + await self.loop.sock_sendall(b, b'x') + data = await self.loop.sock_recv(b, 1024) + b.close() + return data + + self.assertEqual(self.loop.run_until_complete(main()), b'echo:x') + + def test_bad_fd_rejected(self): + server_mod = _server_module() + + async def main(): + with self.assertRaises(ValueError): + await server_mod.serve(-1, Echo) + with self.assertRaises(ValueError): + await server_mod.serve('nope', Echo) + with self.assertRaises(OSError): + await server_mod.serve(99999, Echo) + with self.assertRaises(ValueError): + await server_mod.adopt(-5, Echo) + return 'ok' + + self.assertEqual(self.loop.run_until_complete(main()), 'ok') + + def test_wrong_socket_type_rejected(self): + server_mod = _server_module() + usock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + usock.bind(('127.0.0.1', 0)) + fd = usock.detach() + + async def main(): + with self.assertRaises(ValueError): + await server_mod.serve(fd, Echo) # datagram fd, stream expected + return 'ok' + + try: + self.assertEqual(self.loop.run_until_complete(main()), 'ok') + finally: + os.close(fd) + + +class TestErlangServe(_TestServe, tb.ErlangTestCase): + """erlang.server on ErlangEventLoop.""" + + +class TestAIOServe(_TestServe, tb.AIOTestCase): + """erlang.server on the stdlib loop (portable helpers).""" diff --git a/priv/tests/test_transport_close.py b/priv/tests/test_transport_close.py new file mode 100644 index 0000000..471501a --- /dev/null +++ b/priv/tests/test_transport_close.py @@ -0,0 +1,236 @@ +# Copyright 2026 Benoit Chesneau +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the transport close path of ErlangEventLoop. + +A transport must not close its socket while the fd may still be in the BEAM +poll set: it detaches the fd and hands it to the loop (_close_socket), which +releases the fd resource with ownership so the NIF closes it after the +select stop. These tests drive the loop against a recording NIF stub, so +they check the contract without a running BEAM. +""" + +import asyncio +import os +import socket +import unittest + +from _erlang_impl import _loop as loop_mod +from _erlang_impl._transport import ErlangSocketTransport, ErlangDatagramTransport + + +class _RecordingNif(loop_mod._MockNifModule): + """Mock NIF that records fd resource calls.""" + + def __init__(self): + super().__init__() + self.calls = [] + + def _add_reader_for(self, capsule, fd, callback_id): + self.calls.append(('add_reader', fd)) + return super()._add_reader_for(capsule, fd, callback_id) + + def _add_writer_for(self, capsule, fd, callback_id): + self.calls.append(('add_writer', fd)) + return super()._add_writer_for(capsule, fd, callback_id) + + def _clear_fd_read(self, fd_key): + self.calls.append(('clear_read', self._fd_by_key.get(fd_key))) + + def _clear_fd_write(self, fd_key): + self.calls.append(('clear_write', self._fd_by_key.get(fd_key))) + + def _release_fd_resource(self, fd_key, take_ownership=False): + self.calls.append(('release', self._fd_by_key.get(fd_key), take_ownership)) + return super()._release_fd_resource(fd_key, take_ownership) + + +def _make_loop(): + """ErlangEventLoop over the recording stub (no py_event_loop C module).""" + loop = loop_mod.ErlangEventLoop.__new__(loop_mod.ErlangEventLoop) + nif = _RecordingNif() + # Mirror the parts of __init__ the transports touch + loop._pel = nif + loop._loop_capsule = nif._loop_new() + loop._uses_global_capsule = False + loop._readers = {} + loop._writers = {} + loop._callbacks_by_cid = {} + loop._fd_resources = {} + loop._timers = {} + loop._timer_refs = {} + loop._handle_to_callback_id = {} + loop._ready = __import__('collections').deque() + loop._ready_append = loop._ready.append + loop._ready_popleft = loop._ready.popleft + loop._handle_pool = [] + loop._handle_pool_max = 150 + loop._cached_time = 0.0 + loop._wake_pending = False + loop._running = False + loop._stopping = False + loop._closed = False + loop._thread_id = None + loop._clock_resolution = 1e-9 + loop._exception_handler = None + loop._current_handle = None + loop._debug = False + loop._task_factory = None + loop._default_executor = None + loop._signal_handlers = {} + loop._execution_mode = None + loop._callback_id = 0 + return loop, nif + + +class _Proto(asyncio.Protocol): + def __init__(self): + self.lost = [] + + def connection_made(self, transport): + pass + + def connection_lost(self, exc): + self.lost.append(exc) + + +class TestTransportClose(unittest.TestCase): + + def setUp(self): + self.loop, self.nif = _make_loop() + self.a, self.b = socket.socketpair() + self.a.setblocking(False) + + def tearDown(self): + for s in (self.a, self.b): + try: + s.close() + except OSError: + pass + + def _fd_is_open(self, fd): + try: + os.fstat(fd) + return True + except OSError: + return False + + def test_close_hands_fd_to_nif(self): + fd = self.a.fileno() + proto = _Proto() + transport = ErlangSocketTransport(self.loop, self.a, proto) + self.loop.add_reader(fd, transport._read_ready) + self.assertEqual(self.nif.calls, [('add_reader', fd)]) + + transport.close() + # No pending writes: connection_lost ran and the socket was detached, + # the fd itself is closed by the (mock) NIF with ownership + self.assertEqual(proto.lost, [None]) + self.assertEqual(self.a.fileno(), -1) + self.assertIn(('release', fd, True), self.nif.calls) + self.assertFalse(self._fd_is_open(fd)) + # nothing left registered for that fd + self.assertNotIn(fd, self.loop._fd_resources) + self.assertNotIn(fd, self.loop._readers) + + def test_stop_reading_keeps_resource(self): + fd = self.a.fileno() + transport = ErlangSocketTransport(self.loop, self.a, _Proto()) + self.loop.add_reader(fd, transport._read_ready) + transport.pause_reading() + self.assertIn(('clear_read', fd), self.nif.calls) + # resource kept for resume, no release issued + self.assertNotIn(('release', fd, False), self.nif.calls) + self.assertIn(fd, self.loop._fd_resources) + transport.resume_reading() + self.assertIn(fd, self.loop._readers) + + def test_abort_closes_once(self): + fd = self.a.fileno() + proto = _Proto() + transport = ErlangSocketTransport(self.loop, self.a, proto) + self.loop.add_reader(fd, transport._read_ready) + transport.abort() + transport.abort() + transport.close() + self.assertEqual(proto.lost, [None]) + releases = [c for c in self.nif.calls if c[0] == 'release'] + self.assertEqual(releases, [('release', fd, True)]) + + def test_pending_write_defers_close(self): + fd = self.a.fileno() + proto = _Proto() + transport = ErlangSocketTransport(self.loop, self.a, proto) + self.loop.add_reader(fd, transport._read_ready) + # Fill the buffer so the write cannot complete synchronously + transport._buffer = bytearray(b'pending') + transport._buffer_offset = 0 + transport.close() + # reading stopped, connection not lost yet, socket still open + self.assertIn(('clear_read', fd), self.nif.calls) + self.assertEqual(proto.lost, []) + self.assertNotEqual(self.a.fileno(), -1) + # drain: the write callback finishes and closes + transport._write_ready_cb() + self.assertEqual(proto.lost, [None]) + self.assertIn(('release', fd, True), self.nif.calls) + + def test_datagram_close(self): + u = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + u.bind(('127.0.0.1', 0)) + u.setblocking(False) + fd = u.fileno() + proto = _Proto() + transport = ErlangDatagramTransport(self.loop, u, proto) + self.loop.add_reader(fd, transport._read_ready) + transport.close() + self.assertEqual(proto.lost, [None]) + self.assertIn(('release', fd, True), self.nif.calls) + self.assertFalse(self._fd_is_open(fd)) + + def test_close_socket_unregistered(self): + # A socket the loop never registered is closed directly + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + fd = s.fileno() + self.loop._close_socket(s) + self.assertFalse(self._fd_is_open(fd)) + self.assertEqual([c for c in self.nif.calls if c[0] == 'release'], []) + + def test_loop_close_releases_kept_resources(self): + fd = self.a.fileno() + transport = ErlangSocketTransport(self.loop, self.a, _Proto()) + self.loop.add_reader(fd, transport._read_ready) + transport.pause_reading() # resource kept without reader + self.loop.close() + self.assertIn(('release', fd, False), self.nif.calls) + self.assertEqual(self.loop._fd_resources, {}) + + def test_del_abandoned_transport(self): + # A paused transport is not referenced by the loop's readers; when it + # is dropped its socket still goes through the loop close path + fd = self.a.fileno() + transport = ErlangSocketTransport(self.loop, self.a, _Proto()) + self.loop.add_reader(fd, transport._read_ready) + transport.pause_reading() + sock = self.a + self.a = socket.socket() # keep tearDown happy + del transport + import gc + gc.collect() + self.assertEqual(sock.fileno(), -1) + self.assertIn(('release', fd, True), self.nif.calls) + + +if __name__ == '__main__': + unittest.main() diff --git a/rebar.config b/rebar.config index a93fbe8..c36b4fc 100644 --- a/rebar.config +++ b/rebar.config @@ -62,6 +62,7 @@ <<"docs/scalability.md">>, <<"docs/threading.md">>, <<"docs/asyncio.md">>, + <<"docs/workers.md">>, <<"docs/reactor.md">>, <<"docs/process-bound-envs.md">>, <<"docs/security.md">>, @@ -93,6 +94,7 @@ <<"docs/scalability.md">>, <<"docs/threading.md">>, <<"docs/asyncio.md">>, + <<"docs/workers.md">>, <<"docs/reactor.md">>, <<"docs/process-bound-envs.md">>, <<"docs/security.md">>, diff --git a/src/erlang_python.app.src b/src/erlang_python.app.src index 94abb6c..251cf8f 100644 --- a/src/erlang_python.app.src +++ b/src/erlang_python.app.src @@ -1,6 +1,6 @@ {application, erlang_python, [ {description, "Execute Python applications from Erlang using dirty NIFs"}, - {vsn, "4.0.0"}, + {vsn, "4.1.0"}, {registered, []}, {mod, {erlang_python_app, []}}, {applications, [ diff --git a/src/py_context.erl b/src/py_context.erl index b79da74..fa1a72f 100644 --- a/src/py_context.erl +++ b/src/py_context.erl @@ -55,7 +55,17 @@ is_subinterp/1, create_local_env/1, get_nif_ref/1, - interrupt/1 + interrupt/1, + start_loop/1, + start_loop/2, + stop_loop/1, + stop_loop/2, + loop_ref/1, + submit/4, + submit/5, + submit_await/4, + submit_await/5, + submit_await/6 ]). %% Internal exports @@ -83,9 +93,19 @@ id :: pos_integer(), interp_id :: non_neg_integer(), event_state = #{} :: map(), %% #{loop_ref => ref(), worker_pid => pid()} - callback_handler :: pid() | undefined %% For thread-model callback handling + callback_handler :: pid() | undefined, %% For thread-model callback handling + %% Worker loop (start_loop/1): request id of the run_forever exec, the + %% owner that gets {py_loop_exit, Ctx, Result}, its monitor, and the + %% callers waiting in stop_loop/2 + loop_req :: reference() | undefined, + loop_owner :: pid() | undefined, + loop_owner_mon :: reference() | undefined, + loop_stop_waiters = [] :: [{pid(), reference()}] }). +%% Time given to a running loop to exit after py_context:interrupt/1 +-define(LOOP_INTERRUPT_GRACE_MS, 3000). + %% ============================================================================ %% API %% ============================================================================ @@ -422,6 +442,108 @@ init_ref_tab() -> ok end. +%% @doc Run an ErlangEventLoop forever on the context's thread. +%% +%% Returns as soon as the loop is started. The loop keeps running until +%% stop_loop/1,2 or interrupt/1; the owner (the caller by default) receives +%% `{py_loop_exit, Ctx, Result}' when it ends, where Result is the return of +%% the exec that ran it (`ok', `{error, interrupted}', or a Python error). +%% +%% While the loop runs, call/eval/exec/call_method on this context return +%% `{error, loop_running}': the thread is busy in the loop, and a timed-out +%% call would interrupt it. Use submit/4,5 and submit_await/4,5,6 instead; +%% they inject coroutines into the running loop. +%% +%% Options: +%% - `owner' - pid that receives `{py_loop_exit, Ctx, Result}' (default: caller). +%% If the owner dies the loop is stopped. +-spec start_loop(context()) -> ok | {error, term()}. +start_loop(Ctx) -> + start_loop(Ctx, #{}). + +-spec start_loop(context(), map()) -> ok | {error, term()}. +start_loop(Ctx, Opts) when is_pid(Ctx), is_map(Opts) -> + Owner = maps:get(owner, Opts, self()), + MRef = erlang:monitor(process, Ctx), + Ctx ! {start_loop, self(), MRef, Owner}, + await_ctrl_reply(Ctx, MRef, 15000). + +%% @doc Stop a loop started with start_loop/1,2. +%% +%% Asks the loop to stop from inside (a coroutine calling `loop.stop()'), +%% then interrupts the thread if it has not exited after Grace ms (default +%% 5000). Returns `ok' once the loop has exited, `{error, no_loop}' when none +%% is running, `{error, timeout}' if it survived the interrupt too. +-spec stop_loop(context()) -> ok | {error, term()}. +stop_loop(Ctx) -> + stop_loop(Ctx, 5000). + +-spec stop_loop(context(), non_neg_integer()) -> ok | {error, term()}. +stop_loop(Ctx, GraceMs) when is_pid(Ctx), is_integer(GraceMs), GraceMs >= 0 -> + MRef = erlang:monitor(process, Ctx), + Ctx ! {stop_loop, self(), MRef, GraceMs}, + await_ctrl_reply(Ctx, MRef, GraceMs + ?LOOP_INTERRUPT_GRACE_MS + 2000). + +%% @doc Event loop reference of this context, usable with py_nif:submit_task/7 +%% and py_event_loop:create_task/4. +%% +%% owngil contexts have their own loop; worker contexts share the main +%% interpreter's loop (py_event_loop:get_loop/0). +-spec loop_ref(context()) -> {ok, reference()} | {error, term()}. +loop_ref(Ctx) when is_pid(Ctx) -> + MRef = erlang:monitor(process, Ctx), + Ctx ! {loop_ref, self(), MRef}, + await_ctrl_reply(Ctx, MRef, 5000). + +%% @doc Schedule `Module:Func(Args...)' on the context's event loop and +%% return at once with `{ok, TaskRef}'. +%% +%% Works whether or not start_loop/1 is active: with a running loop the +%% coroutine is injected into it, otherwise the event worker steps the loop. +%% The result arrives as `{async_result, TaskRef, {ok, Value} | {error, R}}'; +%% use py_event_loop:await/1,2 or submit_await/4,5,6. Coroutine functions +%% are awaited, plain functions are called and their value returned. +%% Module must be importable in the context (sys.modules), so put entry +%% points in a module rather than in the exec namespace. +-spec submit(context(), atom() | binary(), atom() | binary(), list()) -> + {ok, reference()} | {error, term()}. +submit(Ctx, Module, Func, Args) -> + submit(Ctx, Module, Func, Args, #{}). + +-spec submit(context(), atom() | binary(), atom() | binary(), list(), map()) -> + {ok, reference()} | {error, term()}. +submit(Ctx, Module, Func, Args, Kwargs) when is_pid(Ctx), is_list(Args), is_map(Kwargs) -> + case loop_ref(Ctx) of + {ok, LoopRef} -> + TaskRef = make_ref(), + case py_nif:submit_task(LoopRef, self(), TaskRef, + to_binary(Module), to_binary(Func), Args, Kwargs) of + ok -> {ok, TaskRef}; + {error, _} = Error -> Error + end; + {error, _} = Error -> + Error + end. + +%% @doc submit/4 followed by py_event_loop:await/2 (default timeout 5000 ms). +-spec submit_await(context(), atom() | binary(), atom() | binary(), list()) -> + {ok, term()} | {error, term()}. +submit_await(Ctx, Module, Func, Args) -> + submit_await(Ctx, Module, Func, Args, #{}, 5000). + +-spec submit_await(context(), atom() | binary(), atom() | binary(), list(), map()) -> + {ok, term()} | {error, term()}. +submit_await(Ctx, Module, Func, Args, Kwargs) -> + submit_await(Ctx, Module, Func, Args, Kwargs, 5000). + +-spec submit_await(context(), atom() | binary(), atom() | binary(), list(), map(), + timeout()) -> {ok, term()} | {error, term()}. +submit_await(Ctx, Module, Func, Args, Kwargs, Timeout) -> + case submit(Ctx, Module, Func, Args, Kwargs) of + {ok, TaskRef} -> py_event_loop:await(TaskRef, Timeout); + {error, _} = Error -> Error + end. + %% ============================================================================ %% Internal functions %% ============================================================================ @@ -453,6 +575,21 @@ await_reply(Ctx, MRef, Timeout) -> {error, timeout} end. +%% @private +%% Reply wait for loop control messages: unlike await_reply/3 a timeout here +%% must not interrupt the context (it would kill the loop we are managing). +await_ctrl_reply(Ctx, MRef, Timeout) -> + receive + {MRef, Result} -> + erlang:demonitor(MRef, [flush]), + Result; + {'DOWN', MRef, process, Ctx, Reason} -> + {error, {context_died, Reason}} + after Timeout -> + erlang:demonitor(MRef, [flush]), + {error, timeout} + end. + %% @private register_nif_ref(Ref) -> try @@ -494,7 +631,7 @@ init(Parent, Id, Mode, Opts) -> register_nif_ref(Ref), case apply_memory_limit(Ref, Opts) of ok -> - init_started(Parent, Id, Ref, InterpId); + init_started(Parent, Id, Ref, InterpId, Opts); {error, LimitError} -> unregister_nif_ref(), try py_nif:context_destroy(Ref) catch _:_ -> ok end, @@ -516,12 +653,23 @@ apply_memory_limit(Ref, Opts) -> end. %% @private -init_started(Parent, Id, Ref, InterpId) -> +init_started(Parent, Id, Ref, InterpId, Opts) -> %% Apply all registered imports and paths to this interpreter apply_registered_imports(Ref), apply_registered_paths(Ref), %% Apply preload code (populates globals for process-local envs) apply_preload(Ref), + %% Per-context preload from new/1 (imports the app once per worker) + case maps:get(preload, Opts, undefined) of + undefined -> ok; + PreCode when is_binary(PreCode); is_list(PreCode) -> + case handle_exec_with_async(Ref, iolist_to_binary(PreCode)) of + ok -> ok; + {error, PreErr} -> + error_logger:warning_msg( + "py_context ~p: preload failed: ~p~n", [InterpId, PreErr]) + end + end, %% For subinterpreters, create a dedicated event worker EventState = setup_event_worker(Ref, InterpId), %% For thread-model subinterpreters, spawn a dedicated callback handler @@ -635,8 +783,86 @@ create_context(owngil) -> %% @private %% Main context loop. Handles requests and uses suspension-based callback support. -loop(#state{ref = Ref, interp_id = InterpId} = State) -> +loop(#state{ref = Ref, interp_id = InterpId, loop_req = LoopReq} = State) -> receive + %% ---- worker loop management (start_loop/stop_loop/loop_ref) ---- + {start_loop, From, MRef, _Owner} when LoopReq =/= undefined -> + From ! {MRef, {error, already_running}}, + loop(State); + + {start_loop, From, MRef, Owner} -> + {Reply, NewState} = do_start_loop(Owner, State), + From ! {MRef, Reply}, + loop(NewState); + + {stop_loop, From, MRef, _GraceMs} when LoopReq =:= undefined -> + From ! {MRef, {error, no_loop}}, + loop(State); + + {stop_loop, From, MRef, GraceMs} -> + loop(begin_stop_loop(From, MRef, GraceMs, State)); + + {loop_ref, From, MRef} -> + From ! {MRef, context_loop_ref(State)}, + loop(State); + + {py_result, LoopReq, Result} when LoopReq =/= undefined -> + loop(loop_exited(Result, State)); + + {loop_stop_deadline, LoopReq} when LoopReq =/= undefined -> + %% Cooperative stop did not land: interrupt the thread + _ = py_nif:context_interrupt(Ref), + erlang:send_after(?LOOP_INTERRUPT_GRACE_MS, self(), + {loop_interrupt_deadline, LoopReq}), + loop(State); + + {loop_interrupt_deadline, LoopReq} when LoopReq =/= undefined -> + [W ! {M, {error, timeout}} || {W, M} <- State#state.loop_stop_waiters], + loop(State#state{loop_stop_waiters = []}); + + {loop_stop_deadline, _} -> + loop(State); + {loop_interrupt_deadline, _} -> + loop(State); + + {'DOWN', Mon, process, _Owner, _Reason} + when Mon =:= State#state.loop_owner_mon, LoopReq =/= undefined -> + %% Owner is gone: nobody will hear the exit, stop the loop + loop(begin_stop_loop(undefined, undefined, 5000, + State#state{loop_owner_mon = undefined})); + + {async_result, _TaskRef, _} -> + %% Result of a coroutine this process submitted (loop stop) - drop + loop(State); + + %% ---- while a worker loop runs, the thread is not available ---- + {call, From, MRef, _, _, _, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + {call, From, MRef, _, _, _, _, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + {eval, From, MRef, _, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + {eval, From, MRef, _, _, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + {exec, From, MRef, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + {exec, From, MRef, _, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + {call_method, From, MRef, _, _, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + + {stop, From, MRef} when LoopReq =/= undefined -> + %% Get the thread out of the loop before destroying the context, + %% otherwise context_destroy waits for a thread that never returns + terminate(normal, stop_running_loop(State)), + From ! {MRef, ok}; + + {'EXIT', _Pid, Reason} = Exit when LoopReq =/= undefined, + (Reason =:= shutdown orelse Reason =:= kill orelse + (is_tuple(Reason) andalso element(1, Reason) =:= shutdown)) -> + self() ! Exit, + loop(stop_running_loop(State)); + {call, From, MRef, Module, Func, Args, Kwargs} -> Result = handle_call_with_suspension(Ref, Module, Func, Args, Kwargs), From ! {MRef, Result}, @@ -736,6 +962,95 @@ loop(#state{ref = Ref, interp_id = InterpId} = State) -> end end. +%% ============================================================================ +%% Worker loop helpers +%% ============================================================================ + +%% @private Loop reference: the context's own loop (owngil) or the shared +%% main-interpreter loop (worker mode) +context_loop_ref(#state{event_state = #{loop_ref := LoopRef}}) -> + {ok, LoopRef}; +context_loop_ref(_State) -> + py_event_loop:get_loop(). + +%% @private Start run_forever on the context thread through the async exec +%% path, so this process stays free to serve loop_ref/stop_loop and the +%% dirty schedulers are not held. +do_start_loop(Owner, #state{ref = Ref} = State) -> + case context_loop_ref(State) of + {ok, _} -> + LoopReq = make_ref(), + case py_nif:context_call_async(Ref, self(), LoopReq, <<"erlang">>, + <<"_run_loop_forever">>, [self()], #{}) of + {enqueued, LoopReq} -> + %% Wait for the loop to actually run before answering, so + %% a submit right after start_loop finds it + receive + {py_loop_started} -> + Mon = case is_pid(Owner) of + true -> erlang:monitor(process, Owner); + false -> undefined + end, + {ok, State#state{loop_req = LoopReq, loop_owner = Owner, + loop_owner_mon = Mon, loop_stop_waiters = []}}; + {py_result, LoopReq, {error, Reason}} -> + {{error, Reason}, State}; + {py_result, LoopReq, Other} -> + {{error, {loop_exited, Other}}, State} + after 10000 -> + {{error, loop_start_timeout}, State} + end; + {error, Reason} -> + {{error, Reason}, State} + end; + {error, Reason} -> + {{error, Reason}, State} + end. + +%% @private Ask the running loop to stop from inside, arm the interrupt +%% deadline, and remember who to answer once it has exited. +begin_stop_loop(From, MRef, GraceMs, #state{loop_req = LoopReq} = State) -> + Waiters = case From of + undefined -> State#state.loop_stop_waiters; + _ -> [{From, MRef} | State#state.loop_stop_waiters] + end, + case context_loop_ref(State) of + {ok, LoopRef} -> + _ = py_nif:submit_task(LoopRef, self(), make_ref(), + <<"erlang">>, <<"_stop_loop">>, [], #{}); + _ -> + ok + end, + erlang:send_after(GraceMs, self(), {loop_stop_deadline, LoopReq}), + State#state{loop_stop_waiters = Waiters}. + +%% @private The exec running the loop returned: tell the owner and the +%% stop_loop callers, clear the loop state. +loop_exited(Result, #state{loop_owner = Owner, loop_owner_mon = Mon, + loop_stop_waiters = Waiters} = State) -> + case Mon of + undefined -> ok; + _ -> erlang:demonitor(Mon, [flush]) + end, + case is_pid(Owner) of + true -> Owner ! {py_loop_exit, self(), Result}; + false -> ok + end, + [W ! {M, ok} || {W, M} <- Waiters], + State#state{loop_req = undefined, loop_owner = undefined, + loop_owner_mon = undefined, loop_stop_waiters = []}. + +%% @private Synchronous stop used before terminate: interrupt and wait a +%% bounded time for the exec to return. +stop_running_loop(#state{ref = Ref, loop_req = LoopReq} = State) -> + _ = py_nif:context_interrupt(Ref), + receive + {py_result, LoopReq, Result} -> + loop_exited(Result, State) + after ?LOOP_INTERRUPT_GRACE_MS -> + loop_exited({error, timeout}, State) + end. + %% @private Clean up resources on termination terminate(_Reason, #state{ref = Ref, event_state = EventState, callback_handler = CallbackHandler}) -> unregister_nif_ref(), diff --git a/src/py_event_worker.erl b/src/py_event_worker.erl index 884465c..da8b98e 100644 --- a/src/py_event_worker.erl +++ b/src/py_event_worker.erl @@ -92,6 +92,11 @@ handle_info({timeout, TimerRef}, State) -> handle_info({select, _FdRes, _Ref, cancelled}, State) -> {noreply, State}; +%% Re-arm request from the Python loop (see py_nif:fd_arm/2) +handle_info({fd_arm, FdRes, Mode}, State) -> + _ = py_nif:fd_arm(FdRes, Mode), + {noreply, State}; + %% Handle task_ready wakeup from submit_task NIF. %% This is sent via enif_send when a new async task is submitted. %% Uses a drain-until-empty loop to handle tasks submitted during processing. diff --git a/src/py_nif.erl b/src/py_nif.erl index d59bde9..0357bc6 100644 --- a/src/py_nif.erl +++ b/src/py_nif.erl @@ -121,6 +121,7 @@ %% FD lifecycle management (uvloop-like API) handle_fd_event/2, handle_fd_event_and_reselect/2, + fd_arm/2, stop_reader/1, start_reader/1, stop_writer/1, @@ -925,6 +926,13 @@ handle_fd_event(_FdRef, _Type) -> handle_fd_event_and_reselect(_FdRef, _Type) -> ?NIF_STUB. +%% @doc Re-arm a read or write select for an existing fd resource. +%% Sent to the event worker by the Python loop, which cannot re-select a +%% scheduler-polled fd from its own thread. +-spec fd_arm(reference(), read | write) -> ok | {error, term()}. +fd_arm(_FdRef, _Type) -> + ?NIF_STUB. + %% @doc Stop/pause read monitoring without closing the FD. %% The watcher still exists and can be restarted with start_reader. -spec stop_reader(reference()) -> ok | {error, term()}. diff --git a/test/py_asyncio_compat_SUITE.erl b/test/py_asyncio_compat_SUITE.erl index 40d73a6..6de5c7e 100644 --- a/test/py_asyncio_compat_SUITE.erl +++ b/test/py_asyncio_compat_SUITE.erl @@ -50,7 +50,11 @@ test_executors_erlang/1, test_context_erlang/1, test_process_erlang/1, - test_erlang_api/1 + test_erlang_api/1, + test_server_erlang/1, + test_server_asyncio/1, + test_transport_close/1, + test_loop_helpers/1 ]). %% Asyncio comparison tests (standard asyncio) @@ -84,7 +88,10 @@ groups() -> test_executors_erlang, test_context_erlang, test_process_erlang, - test_erlang_api + test_erlang_api, + test_server_erlang, + test_transport_close, + test_loop_helpers ]}, {comparison_tests, [sequence], [ test_base_asyncio, @@ -94,7 +101,8 @@ groups() -> test_unix_asyncio, test_dns_asyncio, test_executors_asyncio, - test_context_asyncio + test_context_asyncio, + test_server_asyncio ]} ]. @@ -176,6 +184,35 @@ test_erlang_api(Config) -> %% test_erlang_api has only Erlang-specific tests, run all run_python_tests("tests.test_erlang_api", <<"*">>, Config). +%% erlang.server (serve/adopt on handed-over fds) on the Erlang loop +test_server_erlang(Config) -> + run_erlang_tests("tests.test_server", Config). + +%% Transport close path against a recording NIF stub (no loop needed) +test_transport_close(Config) -> + run_python_tests("tests.test_transport_close", <<"*">>, Config). + +%% _run_loop_forever/_stop_loop, the entry points of py_context:start_loop. +%% Needs an interpreter with no other ErlangEventLoop, so an owngil context. +test_loop_helpers(Config) -> + case py_nif:owngil_supported() of + false -> + {skip, "needs an owngil context (Python 3.14+)"}; + true -> + PrivDir = ?config(priv_dir, Config), + {ok, Ctx} = py_context:new(#{mode => owngil}), + ok = py_context:exec(Ctx, iolist_to_binary(io_lib:format( + "import sys\nif '~s' not in sys.path:\n sys.path.insert(0, '~s')\n", + [PrivDir, PrivDir]))), + Result = py_context:call(Ctx, 'tests.ct_runner', run_tests, + [<<"tests.test_loop_helpers">>, <<"*">>], #{}, 120000), + py_context:stop(Ctx), + case Result of + {ok, Results} -> handle_test_results("tests.test_loop_helpers", <<"*">>, Results); + {error, Reason} -> ct:fail({python_error, Reason}) + end + end. + %% ============================================================================ %% Asyncio Comparison Tests (standard asyncio) %% ============================================================================ @@ -209,6 +246,9 @@ test_executors_asyncio(Config) -> test_context_asyncio(Config) -> run_asyncio_tests("tests.test_context", Config). +test_server_asyncio(Config) -> + run_asyncio_tests("tests.test_server", Config). + %% ============================================================================ %% Internal Functions %% ============================================================================ diff --git a/test/py_test_workerloop.py b/test/py_test_workerloop.py new file mode 100644 index 0000000..a233ef5 --- /dev/null +++ b/test/py_test_workerloop.py @@ -0,0 +1,182 @@ +# Helpers for py_worker_loop_SUITE: protocols and coroutines that +# py_context:submit/4 schedules on a context's worker loop. +import asyncio +import erlang + +served = 0 +_servers = {} +_datagram = {} + + +class Echo(asyncio.Protocol): + """Reply with 'ok:' + data, tagged with the worker id, then close.""" + + tag = b'' + + def connection_made(self, transport): + self.transport = transport + + def data_received(self, data): + global served + served += 1 + self.transport.write(self.tag + b'ok:' + data) + self.transport.close() + + +class KeepAlive(asyncio.Protocol): + """Answer every 'ping' with 'pong', keep the connection open.""" + + def connection_made(self, transport): + self.transport = transport + self.buf = b'' + + def data_received(self, data): + global served + self.buf += data + while len(self.buf) >= 4: + self.buf = self.buf[4:] + served += 1 + self.transport.write(b'pong') + + +class Greedy(asyncio.Protocol): + """'hog' allocates past any sane memory cap; anything else echoes.""" + + def connection_made(self, transport): + self.transport = transport + + def data_received(self, data): + if data == b'hog': + try: + hog = [[] for _ in range(3000000)] # about 170 MB of empty lists + self.transport.write(b'no-cap:%d' % len(hog)) + except MemoryError: + self.transport.write(b'memoryerror') + else: + self.transport.write(b'ok:' + data) + self.transport.close() + + +class EchoUDP(asyncio.DatagramProtocol): + def connection_made(self, transport): + self.transport = transport + + def datagram_received(self, data, addr): + global served + served += 1 + self.transport.sendto(b'udp:' + data, addr) + + +async def serve(fd, tag=b''): + """Serve TCP on a listen fd handed over by Erlang.""" + if isinstance(tag, str): + tag = tag.encode() + proto = type('TaggedEcho', (Echo,), {'tag': tag}) + server = await erlang.server.serve(fd, proto) + _servers[fd] = server + return 'serving' + + +async def serve_keepalive(fd): + server = await erlang.server.serve(fd, KeepAlive) + _servers[fd] = server + return 'serving' + + +async def serve_greedy(fd): + server = await erlang.server.serve(fd, Greedy) + _servers[fd] = server + return 'serving' + + +async def block_loop(seconds): + """Wedge the loop in a blocking C call (time.sleep).""" + import time + time.sleep(seconds) + return 'unblocked' + + +async def serve_udp(fd): + transport = await erlang.server.serve(fd, EchoUDP, udp=True) + _datagram[fd] = transport + return 'serving' + + +async def stop(fd): + server = _servers.pop(fd, None) + if server is not None: + await erlang.server.stop_serving(server) + transport = _datagram.pop(fd, None) + if transport is not None: + transport.close() + return 'stopped' + + +async def adopt(fd): + """Take over an accepted connection fd.""" + await erlang.server.adopt(fd, Echo) + return 'adopted' + + +_started = [] + + +async def add(a, b): + _started.append(a) + await asyncio.sleep(0.001) + return a + b + + +def started(): + """Which add() calls actually started (diagnostics).""" + return list(_started) + + +_woke = [] + + +async def add_traced(a, b): + """add() that records wake up: which sleeps came back.""" + _started.append(a) + await asyncio.sleep(0.001) + _woke.append(a) + return a + b + + +async def loop_state(): + """Loop side view for diagnostics.""" + loop = asyncio.get_running_loop() + timers = getattr(loop, '_timers', {}) + return { + 'started': len(_started), + 'woke': len(_woke), + 'woke_head': _woke[:12], + 'timers_left': len(timers), + 'timer_ids': sorted(timers)[:12], + 'ready': len(getattr(loop, '_ready', [])), + 'tasks': len(asyncio.all_tasks()), + } + + +def sync_add(a, b): + return a + b + + +async def sleep_then(value, seconds): + await asyncio.sleep(seconds) + return value + + +async def raise_error(): + raise ValueError('boom') + + +async def wait_channel(ref): + """Await one message on a py_channel from inside the loop.""" + ch = erlang.Channel(ref) + msg = await ch.async_receive() + return msg + + +def served_count(): + return served diff --git a/test/py_worker_loop_SUITE.erl b/test/py_worker_loop_SUITE.erl new file mode 100644 index 0000000..db41475 --- /dev/null +++ b/test/py_worker_loop_SUITE.erl @@ -0,0 +1,572 @@ +%%% @doc Common Test suite for worker loops. +%%% +%%% Covers py_context:start_loop/1,2, stop_loop/1,2, loop_ref/1, submit/4,5, +%%% submit_await/4,5,6, the `preload' option, the erlang.server helper, and +%%% the owngil fixes behind them (per-context event loop, async dispatch, +%%% coroutine injection, fd close after select stop). +%%% +%%% Most cases need owngil (one loop per interpreter, and worker contexts +%%% share the main interpreter); those skip on Python < 3.14. +-module(py_worker_loop_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([ + all/0, + groups/0, + init_per_suite/1, + end_per_suite/1, + init_per_group/2, + end_per_group/2, + init_per_testcase/2, + end_per_testcase/2 +]). + +-export([ + %% owngil + test_loop_ref_per_context/1, + test_start_stop_loop/1, + test_start_twice/1, + test_stop_idle/1, + test_calls_rejected_while_running/1, + test_submit_idle_and_running/1, + test_submit_errors_reported/1, + test_submit_ordering/1, + test_tcp_serve_on_dup_fd/1, + test_udp_serve_on_dup_fd/1, + test_adopt_accepted_fd/1, + test_three_workers_one_listen_fd/1, + test_channel_awaited_in_loop/1, + test_owner_death_stops_loop/1, + test_stop_context_while_looping/1, + test_interrupt_ends_loop/1, + test_long_call_no_30s_cap/1, + test_main_pool_unaffected/1, + test_churn_no_poll_reports/1, + test_preload_option/1, + test_bad_fd_rejected/1, + %% worker mode + test_worker_mode_single_loop/1 +]). + +%% logger handler callback used by test_churn_no_poll_reports +-export([log/2]). + +-define(HOST, {127, 0, 0, 1}). + +all() -> + [{group, owngil}, {group, worker}]. + +groups() -> + [{owngil, [], [ + test_loop_ref_per_context, + test_start_stop_loop, + test_start_twice, + test_stop_idle, + test_calls_rejected_while_running, + test_submit_idle_and_running, + test_submit_errors_reported, + test_submit_ordering, + test_tcp_serve_on_dup_fd, + test_udp_serve_on_dup_fd, + test_adopt_accepted_fd, + test_three_workers_one_listen_fd, + test_channel_awaited_in_loop, + test_owner_death_stops_loop, + test_stop_context_while_looping, + test_interrupt_ends_loop, + test_long_call_no_30s_cap, + test_main_pool_unaffected, + test_churn_no_poll_reports, + test_preload_option, + test_bad_fd_rejected + ]}, + {worker, [], [test_worker_mode_single_loop]}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(erlang_python), + TestDir = filename:dirname(code:which(?MODULE)), + [{test_dir, TestDir} | Config]. + +end_per_suite(_Config) -> + ok = application:stop(erlang_python), + ok. + +init_per_group(owngil, Config) -> + case py_nif:owngil_supported() of + true -> [{mode, owngil} | Config]; + false -> {skip, "worker loops need OWN_GIL (Python 3.14+)"} + end; +init_per_group(worker, Config) -> + [{mode, worker} | Config]. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + flush(), + Config. + +end_per_testcase(_TestCase, _Config) -> + flush(), + ok. + +%%% ============================================================================ +%%% Loop lifecycle +%%% ============================================================================ + +%% @doc Every owngil context has its own loop, distinct from the main one. +test_loop_ref_per_context(Config) -> + C1 = new_ctx(Config), + C2 = new_ctx(Config), + {ok, L1} = py_context:loop_ref(C1), + {ok, L2} = py_context:loop_ref(C2), + {ok, LMain} = py_event_loop:get_loop(), + true = L1 =/= L2, + true = L1 =/= LMain, + true = L2 =/= LMain, + stop_ctx(C1), stop_ctx(C2), + ok. + +%% @doc start_loop returns once running; stop_loop returns once exited and +%% the owner hears about it; the context is usable again after. +test_start_stop_loop(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {ok, 3} = py_context:submit_await(C, py_test_workerloop, add, [1, 2]), + ok = py_context:stop_loop(C), + receive {py_loop_exit, C, {ok, <<"stopped">>}} -> ok + after 2000 -> ct:fail(no_loop_exit) + end, + {ok, 4} = py_context:eval(C, <<"2+2">>, #{}, 5000), + %% and again + ok = py_context:start_loop(C), + {ok, 7} = py_context:submit_await(C, py_test_workerloop, add, [3, 4]), + ok = py_context:stop_loop(C), + stop_ctx(C), + ok. + +test_start_twice(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {error, already_running} = py_context:start_loop(C), + ok = py_context:stop_loop(C), + stop_ctx(C), + ok. + +test_stop_idle(Config) -> + C = new_ctx(Config), + {error, no_loop} = py_context:stop_loop(C), + stop_ctx(C), + ok. + +%% @doc call/eval/exec/call_method are refused while the loop runs (a timed +%% out call would interrupt the loop) and the loop keeps working. +test_calls_rejected_while_running(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {error, loop_running} = py_context:eval(C, <<"1">>), + {error, loop_running} = py_context:exec(C, <<"x = 1">>), + {error, loop_running} = py_context:call(C, math, sqrt, [4.0]), + {ok, 3} = py_context:submit_await(C, py_test_workerloop, add, [1, 2]), + ok = py_context:stop_loop(C), + stop_ctx(C), + ok. + +%%% ============================================================================ +%%% submit +%%% ============================================================================ + +%% @doc submit works on an idle context (event worker steps the loop) and +%% into a running loop, for coroutines and plain functions. +test_submit_idle_and_running(Config) -> + C = new_ctx(Config), + {ok, 3} = py_context:submit_await(C, py_test_workerloop, add, [1, 2]), + {ok, 5} = py_context:submit_await(C, py_test_workerloop, sync_add, [2, 3]), + ok = py_context:start_loop(C), + T0 = erlang:monotonic_time(millisecond), + {ok, 9} = py_context:submit_await(C, py_test_workerloop, add, [4, 5]), + Latency = erlang:monotonic_time(millisecond) - T0, + %% injected coroutines wake the loop, no wait for the poll timeout + true = Latency < 500, + {ok, 6} = py_context:submit_await(C, py_test_workerloop, sync_add, [1, 5]), + {ok, TaskRef} = py_context:submit(C, py_test_workerloop, sleep_then, [<<"late">>, 0.05]), + {ok, <<"late">>} = py_event_loop:await(TaskRef, 2000), + ok = py_context:stop_loop(C), + stop_ctx(C), + ok. + +%% @doc Failures to start or run a task are reported, not dropped. +test_submit_errors_reported(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {error, function_not_found} = py_context:submit_await(C, py_test_workerloop, nope, []), + {error, function_not_found} = py_context:submit_await(C, no_such_module, f, []), + {error, {'TypeError', _}} = py_context:submit_await(C, math, sqrt, [<<"x">>]), + {error, _} = py_context:submit_await(C, py_test_workerloop, raise_error, []), + ok = py_context:stop_loop(C), + stop_ctx(C), + ok. + +%% @doc A burst of submits from one caller completes and comes back in order. +%% More than MAX_TASK_BATCH (64) tasks are queued before the worker gets to +%% them, so this also covers the running-loop branch returning `more'. +test_submit_ordering(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {ok, LoopRef} = py_context:loop_ref(C), + Refs = [begin + R = make_ref(), + ok = py_nif:submit_task(LoopRef, self(), R, <<"py_test_workerloop">>, <<"add_traced">>, [I, 0], #{}), + {I, R} + end || I <- lists:seq(1, 500)], + %% Collect within one overall deadline so a strand shows up as a count, + %% not as a timetrap + Deadline = erlang:monotonic_time(millisecond) + 30000, + Results = collect_results(Refs, Deadline, #{}), + Bad = [{I, maps:get(R, Results, missing)} || {I, R} <- Refs, + maps:get(R, Results, missing) =/= {ok, I}], + case Bad of + [] -> ok; + _ -> ct:log("~p of 500 tasks did not complete: ~p", [length(Bad), lists:sublist(Bad, 10)]), + ct:log("loop alive (sync): ~p", [py_context:submit_await(C, py_test_workerloop, sync_add, [1, 1])]), + ct:log("loop alive (coro): ~p", [py_context:submit_await(C, py_test_workerloop, add, [1, 1])]), + ct:log("mailbox: ~p", [erlang:process_info(self(), message_queue_len)]), + Alive = py_context:submit_await(C, py_test_workerloop, add, [1, 1]), + LoopState = py_context:submit_await(C, py_test_workerloop, loop_state, []), + ct:fail({tasks_incomplete, length(Bad), lists:sublist(Bad, 6), + {first_bad_index, element(1, hd(Bad))}, {loop_alive, Alive}, + {loop_state, LoopState}}) + end, + ok = py_context:stop_loop(C), + stop_ctx(C), + ok. + +%%% ============================================================================ +%%% Serving on fds handed over by Erlang +%%% ============================================================================ + +test_tcp_serve_on_dup_fd(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {LSock, Port, Dup} = listen_dup(), + {ok, <<"serving">>} = py_context:submit_await(C, py_test_workerloop, serve, [Dup]), + [<<"ok:x">> = roundtrip(Port, <<"x">>) || _ <- lists:seq(1, 100)], + {ok, 100} = py_context:submit_await(C, py_test_workerloop, served_count, []), + {ok, <<"stopped">>} = py_context:submit_await(C, py_test_workerloop, stop, [Dup]), + ok = py_context:stop_loop(C), + gen_tcp:close(LSock), + stop_ctx(C), + ok. + +test_udp_serve_on_dup_fd(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {ok, USock} = gen_udp:open(0, [binary, {ip, ?HOST}, {active, false}]), + {ok, Port} = inet:port(USock), + {ok, Fd} = inet:getfd(USock), + {ok, Dup} = py:dup_fd(Fd), + {ok, <<"serving">>} = py_context:submit_await(C, py_test_workerloop, serve_udp, [Dup]), + {ok, Client} = gen_udp:open(0, [binary, {ip, ?HOST}, {active, false}]), + ok = gen_udp:send(Client, ?HOST, Port, <<"ping">>), + {ok, {_, _, <<"udp:ping">>}} = gen_udp:recv(Client, 0, 2000), + {ok, <<"stopped">>} = py_context:submit_await(C, py_test_workerloop, stop, [Dup]), + gen_udp:close(Client), + gen_udp:close(USock), + ok = py_context:stop_loop(C), + stop_ctx(C), + ok. + +%% @doc Erlang accepts, then hands the connection fd to the loop. +test_adopt_accepted_fd(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {ok, LSock} = gen_tcp:listen(0, [binary, {ip, ?HOST}, {active, false}]), + {ok, Port} = inet:port(LSock), + Self = self(), + spawn_link(fun() -> + {ok, S} = gen_tcp:connect(?HOST, Port, [binary, {active, false}], 2000), + ok = gen_tcp:send(S, <<"adopted?">>), + Self ! {client, gen_tcp:recv(S, 0, 3000)}, + gen_tcp:close(S) + end), + {ok, Conn} = gen_tcp:accept(LSock, 2000), + {ok, ConnFd} = inet:getfd(Conn), + {ok, Dup} = py:dup_fd(ConnFd), + {ok, <<"adopted">>} = py_context:submit_await(C, py_test_workerloop, adopt, [Dup]), + %% Erlang gives up its copy; Python owns the dup + gen_tcp:close(Conn), + receive {client, {ok, <<"ok:adopted?">>}} -> ok + after 3000 -> ct:fail(no_reply_through_adopted_fd) + end, + gen_tcp:close(LSock), + ok = py_context:stop_loop(C), + stop_ctx(C), + ok. + +%% @doc gunicorn shape: one listen socket, three workers accepting on dups. +test_three_workers_one_listen_fd(Config) -> + Ctxs = [new_ctx(Config) || _ <- lists:seq(1, 3)], + {ok, LSock} = gen_tcp:listen(0, [binary, {ip, ?HOST}, {active, false}, {backlog, 512}]), + {ok, Port} = inet:port(LSock), + {ok, LFd} = inet:getfd(LSock), + lists:foreach(fun({I, C}) -> + ok = py_context:start_loop(C), + {ok, Dup} = py:dup_fd(LFd), + Tag = list_to_binary("w" ++ integer_to_list(I) ++ ":"), + {ok, <<"serving">>} = py_context:submit_await(C, py_test_workerloop, serve, [Dup, Tag]) + end, lists:zip(lists:seq(1, 3), Ctxs)), + Replies = [roundtrip(Port, <<"x">>) || _ <- lists:seq(1, 300)], + 300 = length([R || R <- Replies, binary:part(R, byte_size(R) - 4, 4) =:= <<"ok:x">>]), + Tags = lists:usort([binary:part(R, 0, 3) || R <- Replies]), + ct:log("workers that served: ~p", [Tags]), + %% All three workers accept on the same socket + 3 = length(Tags), + [ok = py_context:stop_loop(C) || C <- Ctxs], + [stop_ctx(C) || C <- Ctxs], + gen_tcp:close(LSock), + ok. + +%% @doc A py_channel awaited inside the loop is the Erlang to loop control +%% plane: no polling, message delivered into the running loop. +test_channel_awaited_in_loop(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {ok, Ch} = py_channel:new(), + {ok, TaskRef} = py_context:submit(C, py_test_workerloop, wait_channel, [Ch]), + timer:sleep(100), + ok = py_channel:send(Ch, {adopt, 42}), + {ok, Msg} = py_event_loop:await(TaskRef, 3000), + ct:log("channel message seen by the loop: ~p", [Msg]), + ok = py_channel:close(Ch), + ok = py_context:stop_loop(C), + stop_ctx(C), + ok. + +%%% ============================================================================ +%%% Failure and shutdown paths +%%% ============================================================================ + +test_owner_death_stops_loop(Config) -> + C = new_ctx(Config), + Owner = spawn(fun() -> receive never -> ok end end), + ok = py_context:start_loop(C, #{owner => Owner}), + {ok, 3} = py_context:submit_await(C, py_test_workerloop, add, [1, 2]), + exit(Owner, kill), + ok = wait_until(fun() -> py_context:eval(C, <<"1">>, #{}, 1000) =:= {ok, 1} end, 5000), + stop_ctx(C), + ok. + +%% @doc Stopping the context while its loop runs interrupts the loop first, +%% so context_destroy does not wait 30 s for the thread. +test_stop_context_while_looping(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + T0 = erlang:monotonic_time(millisecond), + ok = py_context:stop(C), + Elapsed = erlang:monotonic_time(millisecond) - T0, + ct:log("stop while looping took ~p ms", [Elapsed]), + true = Elapsed < 10000, + receive {py_loop_exit, C, {error, interrupted}} -> ok + after 1000 -> ct:fail(no_interrupted_exit) + end, + false = is_process_alive(C), + ok. + +test_interrupt_ends_loop(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + ok = py_context:interrupt(C), + receive {py_loop_exit, C, {error, interrupted}} -> ok + after 3000 -> ct:fail(no_interrupted_exit) + end, + {ok, 4} = py_context:eval(C, <<"2+2">>, #{}, 5000), + stop_ctx(C), + ok. + +%% @doc owngil calls no longer go through the 30 s blocking dispatch, and do +%% not hold a dirty scheduler while they run. +test_long_call_no_30s_cap(Config) -> + C = new_ctx(Config), + Other = new_ctx(Config), + Self = self(), + spawn_link(fun() -> + Self ! {long, py_context:eval(C, <<"__import__('time').sleep(32) or 7">>, #{}, infinity)} + end), + timer:sleep(200), + T0 = erlang:monotonic_time(millisecond), + {ok, 2} = py_context:eval(Other, <<"1+1">>, #{}, 5000), + true = erlang:monotonic_time(millisecond) - T0 < 1000, + receive {long, {ok, 7}} -> ok + after 40000 -> ct:fail(long_call_did_not_return) + end, + stop_ctx(C), stop_ctx(Other), + ok. + +%% @doc Starting and stopping owngil contexts must not touch the main loop's +%% worker (they used to re-point it and leave it dead). +test_main_pool_unaffected(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + ok = py_context:stop_loop(C), + stop_ctx(C), + {ok, 4.0} = py_event_loop:run(math, sqrt, [16.0]), + ok. + +%% @doc Connection churn leaves no fd in the BEAM poll set behind: no +%% "Bad input fd in erts_poll()" or "stealing control" reports. +test_churn_no_poll_reports(Config) -> + ok = logger:add_handler(?MODULE, ?MODULE, #{config => self()}), + C = new_ctx(Config), + ok = py_context:start_loop(C), + {LSock, Port, Dup} = listen_dup(), + {ok, <<"serving">>} = py_context:submit_await(C, py_test_workerloop, serve, [Dup]), + N = 2000, + Self = self(), + Workers = 20, + [spawn_link(fun() -> + Fails = length([bad || _ <- lists:seq(1, N div Workers), + roundtrip(Port, <<"x">>) =/= <<"ok:x">>]), + Self ! {churn_done, Fails} + end) || _ <- lists:seq(1, Workers)], + Fails = lists:sum([receive {churn_done, F} -> F after 60000 -> N end + || _ <- lists:seq(1, Workers)]), + 0 = Fails, + timer:sleep(300), + ok = py_context:stop_loop(C), + gen_tcp:close(LSock), + stop_ctx(C), + ok = logger:remove_handler(?MODULE), + Reports = collect_reports(), + ct:log("erts_poll reports: ~p", [Reports]), + [] = Reports, + ok. + +test_preload_option(Config) -> + Pre = <<"import sys, types\n" + "_m = types.ModuleType('preloaded_mod')\n" + "async def hello():\n" + " return 'hi'\n" + "_m.hello = hello\n" + "sys.modules['preloaded_mod'] = _m\n">>, + {ok, C} = py_context:new(#{mode => ?config(mode, Config), preload => Pre}), + ok = py_context:start_loop(C), + {ok, <<"hi">>} = py_context:submit_await(C, preloaded_mod, hello, []), + ok = py_context:stop_loop(C), + stop_ctx(C), + ok. + +test_bad_fd_rejected(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {error, _} = py_context:submit_await(C, py_test_workerloop, serve, [-1]), + {error, _} = py_context:submit_await(C, py_test_workerloop, serve, [99999]), + {error, _} = py_context:submit_await(C, py_test_workerloop, adopt, [<<"x">>]), + %% loop still fine + {ok, 3} = py_context:submit_await(C, py_test_workerloop, add, [1, 2]), + ok = py_context:stop_loop(C), + stop_ctx(C), + ok. + +%%% ============================================================================ +%%% Worker mode +%%% ============================================================================ + +%% @doc Worker contexts share the main interpreter, which allows one +%% ErlangEventLoop: the first start_loop works, a second context cannot. +test_worker_mode_single_loop(Config) -> + W1 = new_ctx(Config), + W2 = new_ctx(Config), + ok = py_context:start_loop(W1), + {ok, 4.0} = py_context:submit_await(W1, math, sqrt, [16.0]), + {ok, 3} = py_context:submit_await(W1, py_test_workerloop, add, [1, 2]), + {error, _} = py_context:start_loop(W2), + ok = py_context:stop_loop(W1), + stop_ctx(W1), stop_ctx(W2), + ok. + +%%% ============================================================================ +%%% Logger handler (churn test) +%%% ============================================================================ + +log(#{msg := Msg}, #{config := Pid}) -> + Text = case Msg of + {string, S} -> unicode:characters_to_binary(S); + {report, R} -> unicode:characters_to_binary(io_lib:format("~p", [R])); + {Fmt, Args} -> unicode:characters_to_binary(io_lib:format(Fmt, Args)) + end, + case binary:match(Text, [<<"erts_poll">>, <<"stealing control">>]) of + nomatch -> ok; + _ -> Pid ! {poll_report, Text} + end, + ok. + +collect_reports() -> + receive {poll_report, T} -> [T | collect_reports()] + after 200 -> [] + end. + +%%% ============================================================================ +%%% Helpers +%%% ============================================================================ + +new_ctx(Config) -> + {ok, C} = py_context:new(#{mode => ?config(mode, Config)}), + TestDir = ?config(test_dir, Config), + Code = iolist_to_binary(io_lib:format( + "import sys\nif '~s' not in sys.path:\n sys.path.insert(0, '~s')\n" + "import py_test_workerloop\n", [TestDir, TestDir])), + ok = py_context:exec(C, Code), + C. + +stop_ctx(C) -> + try py_context:stop(C) catch _:_ -> ok end, + ok. + +listen_dup() -> + {ok, LSock} = gen_tcp:listen(0, [binary, {ip, ?HOST}, {active, false}, {backlog, 512}]), + {ok, Port} = inet:port(LSock), + {ok, LFd} = inet:getfd(LSock), + {ok, Dup} = py:dup_fd(LFd), + {LSock, Port, Dup}. + +roundtrip(Port, Data) -> + {ok, S} = gen_tcp:connect(?HOST, Port, [binary, {active, false}], 5000), + ok = gen_tcp:send(S, Data), + R = case gen_tcp:recv(S, 0, 5000) of + {ok, Bin} -> Bin; + Other -> Other + end, + gen_tcp:close(S), + R. + +collect_results([], _Deadline, Acc) -> + Acc; +collect_results(Refs, Deadline, Acc) -> + Wait = max(0, Deadline - erlang:monotonic_time(millisecond)), + receive + {async_result, R, Res} -> + case lists:keytake(R, 2, Refs) of + {value, _, Rest} -> collect_results(Rest, Deadline, Acc#{R => Res}); + false -> collect_results(Refs, Deadline, Acc) + end + after Wait -> + Acc + end. + +wait_until(Fun, TimeoutMs) -> + Deadline = erlang:monotonic_time(millisecond) + TimeoutMs, + wait_until_loop(Fun, Deadline). + +wait_until_loop(Fun, Deadline) -> + case Fun() of + true -> ok; + false -> + case erlang:monotonic_time(millisecond) > Deadline of + true -> {error, timeout}; + false -> timer:sleep(50), wait_until_loop(Fun, Deadline) + end + end. + +flush() -> + receive _ -> flush() after 0 -> ok end. diff --git a/test/py_worker_loop_stress_SUITE.erl b/test/py_worker_loop_stress_SUITE.erl new file mode 100644 index 0000000..b20cff6 --- /dev/null +++ b/test/py_worker_loop_stress_SUITE.erl @@ -0,0 +1,350 @@ +%%% @doc Stress tests for worker loops. +%%% +%%% Long running, high volume checks on py_context:start_loop/submit and +%%% erlang.server: connection churn on several workers with no fd left in the +%%% BEAM poll set, held keep-alive connections, start/stop cycling without +%%% leaks, submit storms during traffic, and the kill paths (memory cap, +%%% interrupt of a wedged loop). +%%% +%%% Skipped unless the environment variable STRESS is set (`STRESS=1 rebar3 +%%% ct --suite py_worker_loop_stress_SUITE`), and needs owngil (Python 3.14+). +-module(py_worker_loop_stress_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([ + all/0, + init_per_suite/1, + end_per_suite/1, + init_per_testcase/2, + end_per_testcase/2 +]). + +-export([ + test_churn_four_workers/1, + test_keepalive_connections_held/1, + test_start_stop_cycles/1, + test_submit_storm_under_traffic/1, + test_memory_cap_in_handler/1, + test_interrupt_wedged_loop/1 +]). + +%% logger handler callback +-export([log/2]). + +-define(HOST, {127, 0, 0, 1}). + +all() -> [ + test_churn_four_workers, + test_keepalive_connections_held, + test_start_stop_cycles, + test_submit_storm_under_traffic, + test_memory_cap_in_handler, + test_interrupt_wedged_loop +]. + +init_per_suite(Config) -> + case os:getenv("STRESS") of + false -> + {skip, "set STRESS=1 to run the worker loop stress suite"}; + _ -> + {ok, _} = application:ensure_all_started(erlang_python), + case py_nif:owngil_supported() of + false -> {skip, "worker loops need OWN_GIL (Python 3.14+)"}; + true -> + TestDir = filename:dirname(code:which(?MODULE)), + [{test_dir, TestDir} | Config] + end + end. + +end_per_suite(_Config) -> + ok = application:stop(erlang_python), + ok. + +init_per_testcase(_TestCase, Config) -> + ok = logger:add_handler(?MODULE, ?MODULE, #{config => self()}), + Config. + +end_per_testcase(_TestCase, _Config) -> + _ = logger:remove_handler(?MODULE), + flush(), + ok. + +%%% ============================================================================ +%%% Cases +%%% ============================================================================ + +%% @doc 10k short connections against four workers on one listen fd: no +%% failed connects, no erts_poll reports, fd count back to baseline. +test_churn_four_workers(Config) -> + Baseline = fd_count(), + Ctxs = [new_ctx(Config) || _ <- lists:seq(1, 4)], + {LSock, Port, LFd} = listen(), + lists:foreach(fun({I, C}) -> + ok = py_context:start_loop(C), + {ok, Dup} = py:dup_fd(LFd), + Tag = list_to_binary("w" ++ integer_to_list(I) ++ ":"), + {ok, <<"serving">>} = py_context:submit_await(C, py_test_workerloop, serve, [Dup, Tag]) + end, lists:zip(lists:seq(1, 4), Ctxs)), + N = 10000, + Clients = 50, + T0 = erlang:monotonic_time(millisecond), + Replies = parallel_roundtrips(Port, N, Clients), + Elapsed = erlang:monotonic_time(millisecond) - T0, + Failed = length([R || R <- Replies, not is_binary(R)]), + Tags = lists:usort([binary:part(R, 0, 3) || R <- Replies, is_binary(R)]), + ct:log("~p connections in ~p ms (~p conn/s), failed ~p, workers ~p", + [N, Elapsed, N * 1000 div max(1, Elapsed), Failed, Tags]), + 0 = Failed, + 4 = length(Tags), + [ok = py_context:stop_loop(C) || C <- Ctxs], + [stop_ctx(C) || C <- Ctxs], + gen_tcp:close(LSock), + timer:sleep(500), + [] = collect_reports(), + After = fd_count(), + ct:log("fds before ~p after ~p", [Baseline, After]), + true = After =< Baseline + 8, + ok. + +%% @doc 1000 keep-alive connections held open with periodic writes for 20 s; +%% the worker's memory stays bounded. +test_keepalive_connections_held(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {LSock, Port, LFd} = listen(), + {ok, Dup} = py:dup_fd(LFd), + {ok, <<"serving">>} = py_context:submit_await(C, py_test_workerloop, serve_keepalive, [Dup]), + Ref = py_context:get_nif_ref(C), + {ok, Mem0, _} = py_nif:context_memory_usage(Ref), + Socks = [begin + {ok, S} = gen_tcp:connect(?HOST, Port, [binary, {active, false}], 5000), + S + end || _ <- lists:seq(1, 1000)], + Rounds = 10, + lists:foreach(fun(_) -> + [ok = gen_tcp:send(S, <<"ping">>) || S <- Socks], + [{ok, <<"pong">>} = gen_tcp:recv(S, 4, 10000) || S <- Socks], + timer:sleep(2000) + end, lists:seq(1, Rounds)), + {ok, Mem1, _} = py_nif:context_memory_usage(Ref), + ct:log("memory ~p -> ~p bytes with 1000 held connections", [Mem0, Mem1]), + true = Mem1 < Mem0 + 256 * 1024 * 1024, + [gen_tcp:close(S) || S <- Socks], + timer:sleep(500), + ok = py_context:stop_loop(C), + gen_tcp:close(LSock), + stop_ctx(C), + [] = collect_reports(), + ok. + +%% @doc start_loop/stop_loop cycled 200 times: no leaked event workers, no +%% memory growth, context still fine. +test_start_stop_cycles(Config) -> + C = new_ctx(Config), + Ref = py_context:get_nif_ref(C), + Workers0 = length(supervisor:which_children(py_event_worker_sup)), + Procs0 = erlang:system_info(process_count), + {ok, Mem0, _} = py_nif:context_memory_usage(Ref), + lists:foreach(fun(I) -> + ok = py_context:start_loop(C), + {ok, I} = py_context:submit_await(C, py_test_workerloop, add, [I, 0]), + ok = py_context:stop_loop(C) + end, lists:seq(1, 200)), + {ok, 4} = py_context:eval(C, <<"2+2">>, #{}, 5000), + {ok, Mem1, _} = py_nif:context_memory_usage(Ref), + Workers1 = length(supervisor:which_children(py_event_worker_sup)), + Procs1 = erlang:system_info(process_count), + ct:log("workers ~p -> ~p, processes ~p -> ~p, memory ~p -> ~p", + [Workers0, Workers1, Procs0, Procs1, Mem0, Mem1]), + Workers0 = Workers1, + true = Procs1 =< Procs0 + 5, + true = Mem1 < Mem0 + 64 * 1024 * 1024, + stop_ctx(C), + ok. + +%% @doc 10k coroutine submits while 100 connections exchange data: every +%% result arrives, per caller ordering holds. +test_submit_storm_under_traffic(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {LSock, Port, LFd} = listen(), + {ok, Dup} = py:dup_fd(LFd), + {ok, <<"serving">>} = py_context:submit_await(C, py_test_workerloop, serve_keepalive, [Dup]), + Self = self(), + Traffic = spawn_link(fun() -> + Socks = [element(2, gen_tcp:connect(?HOST, Port, [binary, {active, false}], 5000)) + || _ <- lists:seq(1, 100)], + traffic_loop(Socks, Self) + end), + Callers = 10, + PerCaller = 1000, + [spawn_link(fun() -> + Refs = [element(2, py_context:submit(C, py_test_workerloop, add, [I, 0])) + || I <- lists:seq(1, PerCaller)], + Results = [py_event_loop:await(R, 30000) || R <- Refs], + Self ! {storm, Results} + end) || _ <- lists:seq(1, Callers)], + AllOk = lists:all(fun(Results) -> + Results =:= [{ok, I} || I <- lists:seq(1, PerCaller)] + end, [receive {storm, Rs} -> Rs after 120000 -> [] end || _ <- lists:seq(1, Callers)]), + Traffic ! stop, + receive {traffic_done, Exchanged} -> ct:log("traffic exchanged ~p messages", [Exchanged]) + after 30000 -> ct:fail(traffic_did_not_stop) + end, + true = AllOk, + ok = py_context:stop_loop(C), + gen_tcp:close(LSock), + stop_ctx(C), + ok. + +%% @doc A handler exceeding the context memory cap gets MemoryError there; +%% the loop keeps serving other connections. +test_memory_cap_in_handler(Config) -> + case probe_memory_limits(Config) of + false -> + {skip, "runtime started without enable_memory_limits"}; + true -> + {ok, C} = py_context:new(#{mode => owngil, memory_limit => 64 * 1024 * 1024}), + setup_ctx(C, Config), + ok = py_context:start_loop(C), + {LSock, Port, LFd} = listen(), + {ok, Dup} = py:dup_fd(LFd), + {ok, <<"serving">>} = py_context:submit_await(C, py_test_workerloop, serve_greedy, [Dup]), + %% greedy request: the handler tries to allocate past the cap + {ok, S1} = gen_tcp:connect(?HOST, Port, [binary, {active, false}], 5000), + ok = gen_tcp:send(S1, <<"hog">>), + R1 = gen_tcp:recv(S1, 0, 30000), + gen_tcp:close(S1), + ct:log("greedy handler replied ~p", [R1]), + {ok, <<"memoryerror">>} = R1, + %% normal request still served + {ok, S2} = gen_tcp:connect(?HOST, Port, [binary, {active, false}], 5000), + ok = gen_tcp:send(S2, <<"x">>), + {ok, <<"ok:x">>} = gen_tcp:recv(S2, 0, 5000), + gen_tcp:close(S2), + ok = py_context:stop_loop(C), + gen_tcp:close(LSock), + stop_ctx(C), + ok + end. + +%% @doc A loop wedged in a blocking C call is interrupted once the call +%% returns; the loop exits and the context recovers. +test_interrupt_wedged_loop(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {ok, _} = py_context:submit(C, py_test_workerloop, block_loop, [3]), + timer:sleep(200), + T0 = erlang:monotonic_time(millisecond), + ok = py_context:interrupt(C), + receive {py_loop_exit, C, {error, interrupted}} -> + Elapsed = erlang:monotonic_time(millisecond) - T0, + ct:log("wedged loop interrupted after ~p ms", [Elapsed]), + true = Elapsed < 6000 + after 10000 -> ct:fail(loop_not_interrupted) + end, + {ok, 4} = py_context:eval(C, <<"2+2">>, #{}, 5000), + stop_ctx(C), + ok. + +%%% ============================================================================ +%%% Helpers +%%% ============================================================================ + +new_ctx(Config) -> + {ok, C} = py_context:new(#{mode => owngil}), + setup_ctx(C, Config), + C. + +setup_ctx(C, Config) -> + TestDir = ?config(test_dir, Config), + Code = iolist_to_binary(io_lib:format( + "import sys\nif '~s' not in sys.path:\n sys.path.insert(0, '~s')\n" + "import py_test_workerloop\n", [TestDir, TestDir])), + ok = py_context:exec(C, Code). + +stop_ctx(C) -> + try py_context:stop(C) catch _:_ -> ok end, + ok. + +listen() -> + {ok, LSock} = gen_tcp:listen(0, [binary, {ip, ?HOST}, {active, false}, {backlog, 1024}]), + {ok, Port} = inet:port(LSock), + {ok, LFd} = inet:getfd(LSock), + {LSock, Port, LFd}. + +roundtrip(Port, Data) -> + case gen_tcp:connect(?HOST, Port, [binary, {active, false}], 5000) of + {ok, S} -> + ok = gen_tcp:send(S, Data), + R = case gen_tcp:recv(S, 0, 5000) of + {ok, Bin} -> Bin; + Other -> Other + end, + gen_tcp:close(S), + R; + Error -> + Error + end. + +parallel_roundtrips(Port, N, Clients) -> + Self = self(), + Per = N div Clients, + [spawn_link(fun() -> + Self ! {rt, [roundtrip(Port, <<"x">>) || _ <- lists:seq(1, Per)]} + end) || _ <- lists:seq(1, Clients)], + lists:append([receive {rt, L} -> L after 300000 -> [] end || _ <- lists:seq(1, Clients)]). + +traffic_loop(Socks, Parent) -> + traffic_loop(Socks, Parent, 0). + +traffic_loop(Socks, Parent, Count) -> + receive + stop -> + [gen_tcp:close(S) || S <- Socks], + Parent ! {traffic_done, Count} + after 0 -> + [ok = gen_tcp:send(S, <<"ping">>) || S <- Socks], + [{ok, <<"pong">>} = gen_tcp:recv(S, 4, 10000) || S <- Socks], + traffic_loop(Socks, Parent, Count + length(Socks)) + end. + +probe_memory_limits(Config) -> + case py_context:new(#{mode => owngil}) of + {ok, Ctx} -> + setup_ctx(Ctx, Config), + Ref = py_context:get_nif_ref(Ctx), + Result = py_nif:context_set_memory_limit(Ref, 0), + py_context:stop(Ctx), + Result =:= ok; + _ -> + false + end. + +fd_count() -> + case os:type() of + {unix, linux} -> length(element(2, file:list_dir("/proc/self/fd"))); + {unix, _} -> length(element(2, file:list_dir("/dev/fd"))); + _ -> 0 + end. + +log(#{msg := Msg}, #{config := Pid}) -> + Text = case Msg of + {string, S} -> unicode:characters_to_binary(S); + {report, R} -> unicode:characters_to_binary(io_lib:format("~p", [R])); + {Fmt, Args} -> unicode:characters_to_binary(io_lib:format(Fmt, Args)) + end, + case binary:match(Text, [<<"erts_poll">>, <<"stealing control">>]) of + nomatch -> ok; + _ -> Pid ! {poll_report, Text} + end, + ok. + +collect_reports() -> + receive {poll_report, T} -> [T | collect_reports()] + after 200 -> [] + end. + +flush() -> + receive _ -> flush() after 0 -> ok end. From 78eba7ae4cc400b15f58b05638ea603aa03027f3 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sat, 29 Aug 2026 10:15:03 +0200 Subject: [PATCH 03/15] Add isolated context mode: CPython in a child OS process A mode where Python code cannot take the node down or run forever: one child process per context over a Unix socket, signals with SIGKILL as backstop, rlimits and cgroups, restart on crash. Same public API; the context process is a gen_statem serving one caller at a time. Validated on macOS and FreeBSD. Version 4.2.0. --- CHANGELOG.md | 39 + README.md | 13 + c_src/py_nif.c | 29 + docs/interrupts.md | 19 +- docs/isolated.md | 267 +++++++ docs/security.md | 17 + docs/workers.md | 5 + priv/_erlang_impl/_etf.py | 477 ++++++++++++ priv/_erlang_impl/_isolated.py | 824 ++++++++++++++++++++ priv/py_isolated_child.py | 222 ++++++ rebar.config | 2 + src/erlang_python.app.src | 2 +- src/py.erl | 15 + src/py_context.erl | 118 ++- src/py_isolated.erl | 1193 +++++++++++++++++++++++++++++ src/py_nif.erl | 6 + test/py_isolated_SUITE.erl | 874 +++++++++++++++++++++ test/py_isolated_async_SUITE.erl | 516 +++++++++++++ test/py_isolated_soak_SUITE.erl | 277 +++++++ test/py_isolated_stress_SUITE.erl | 162 ++++ test/py_isolated_vm_SUITE.erl | 477 ++++++++++++ test/py_test_isolated.py | 374 +++++++++ 22 files changed, 5920 insertions(+), 8 deletions(-) create mode 100644 docs/isolated.md create mode 100644 priv/_erlang_impl/_etf.py create mode 100644 priv/_erlang_impl/_isolated.py create mode 100644 priv/py_isolated_child.py create mode 100644 src/py_isolated.erl create mode 100644 test/py_isolated_SUITE.erl create mode 100644 test/py_isolated_async_SUITE.erl create mode 100644 test/py_isolated_soak_SUITE.erl create mode 100644 test/py_isolated_stress_SUITE.erl create mode 100644 test/py_isolated_vm_SUITE.erl create mode 100644 test/py_test_isolated.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b5ddd25..ad58ce1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,44 @@ # Changelog +## 4.2.0 (2026-08-29) + +### Added + +- **`isolated` context mode** - `py_context:new(#{mode => isolated})` runs + CPython in a child OS process per context, with the same `call/eval/exec`, + callback, `erlang.send`/`whereis`, worker-loop and pool API as the embedded + modes. It is the first mode with a hard bound: `py_context:interrupt/1` + stops a blocking C call (a signal in the child) and `SIGKILL` is the + backstop after `kill_after` ms; `py_context:kill/1` kills at once. `rlimits` + (`as`, `cpu`, `nofile`) and a cgroup v2 directory bound the child; a + segfault in a C extension returns `{error, {child_exited, {signal, 11}}}` + and the node survives. The child restarts on crash within a budget + (`restart`, `max_restarts`, `restart_period`); `py_context:child_info/1` + reports its OS pid. Children are reaped by the VM and exit when the BEAM + dies (socket EOF watchdog, `PR_SET_PDEATHSIG` on Linux, `PROC_PDEATHSIG_CTL` + on FreeBSD). `cgroup` is refused outside Linux; rlimits apply everywhere: + `as` is kernel-enforced on Linux and FreeBSD and enforced by an RSS + watchdog in the child on macOS (`{child_exited, {memory_limit, Bytes}}`). + Validated on macOS (arm64) and FreeBSD 14.3 (OTP 28, Python 3.11). +- **`py_context:pass_fd/2`** - hands a file descriptor to an isolated child + over the control socket (`SCM_RIGHTS`), so `erlang.server.serve` works out + of process: Erlang binds once, N killable children accept. +- **Pure-Python ETF codec** (`priv/_erlang_impl/_etf.py`) with the type + mapping of `py_convert.c`; the child needs no C extension. Integers beyond + 64 bits round-trip exactly in isolated mode. +- `py:python_executable/0`, `py:kill/1`, `py_nif:os_kill/2`. +- `py_isolated` is a `gen_statem` (states `idle`, `{busy, Id}`, `looping`, + `stopping_loop`, `{restarting, Reason}`): `sys:get_state/1` and + `sys:trace/2` work on isolated contexts, requests arriving during a + restart are served by the new child, and `py_context:kill/1` returns once + the new child is up. +- Timeouts on an isolated context cancel their own request only (queued + requests are dropped, the executing one is interrupted); the kill backstop + is bound to that request, so a busy shared context is never killed because + another caller gave up. Soak-tested: callback storms, interrupt/kill + storms, loop churn, 60 s mixed workload with resource counters checked. +- Guide: `docs/isolated.md`, with what each of the three modes guarantees. + ## 4.1.0 (2026-08-15) ### Added diff --git a/README.md b/README.md index d98c347..66b2ae8 100644 --- a/README.md +++ b/README.md @@ -603,6 +603,7 @@ When creating Python contexts, you can choose the execution mode: |------|----------------|-------------| | `worker` | Any | Dedicated pthread per context, main interpreter namespace (default) | | `owngil` | 3.14+ | Dedicated pthread + subinterpreter with its own GIL, true parallelism | +| `isolated` | Any | CPython in a child OS process: killable, rlimit-bounded, crash-contained | ```erlang %% Default: worker mode (recommended) @@ -612,8 +613,18 @@ When creating Python contexts, you can choose the execution mode: %% OWN_GIL mode for true parallelism (Python 3.14+ required) %% Each context runs in its own pthread with independent GIL {ok, Ctx} = py_context:new(#{mode => owngil}). + +%% Isolated mode: a child process per context. A stuck call is killed, a +%% segfault only takes the child down, rlimits bound memory and CPU. +{ok, Ctx} = py_context:new(#{mode => isolated, kill_after => 1000, + rlimits => #{as => 512 * 1024 * 1024}}). ``` +**Isolated mode** is the only mode with a hard bound: `py_context:interrupt/1` +stops a blocking C call, and `SIGKILL` is the backstop. It costs a process per +context (about 16 MB and 40 ms to start) and roughly twice the call latency. +See [Isolated Contexts](docs/isolated.md). + **Worker mode is recommended** because it works with any Python version and automatically benefits from free-threaded Python (3.13t+) when available. Each context owns a dedicated pthread, providing stable thread affinity for libraries with thread-local state (numpy, torch, tensorflow). **Why OWN_GIL requires Python 3.14+**: Some C extensions (e.g., `_decimal`, `numpy`) have global state bugs in sub-interpreters on Python 3.12/3.13. These are fixed in Python 3.14. @@ -629,6 +640,7 @@ py:execution_mode(). %% => worker | owngil |------|----------------|-------------| | `worker` (default) | Any | One pthread per context; true parallelism on free-threaded 3.13t+ | | `owngil` | 3.14+ | Per-interpreter GIL, true parallelism across contexts | +| `isolated` | Any | One OS process per context, parallel and failure-isolated | ## Error Handling @@ -651,6 +663,7 @@ py:execution_mode(). %% => worker | owngil - [Logging and Tracing](docs/logging.md) - [Asyncio Event Loop](docs/asyncio.md) - Erlang-native asyncio with TCP/UDP support - [Worker Loops](docs/workers.md) - Long-lived loops in owngil contexts, serving on sockets Erlang owns +- [Isolated Contexts](docs/isolated.md) - Python in a child process: kill, rlimits, crash containment - [Reactor](docs/reactor.md) - FD-based protocol handling - [Security](docs/security.md) - Sandbox and blocked operations - [Changelog](https://github.com/benoitc/erlang-python/releases) diff --git a/c_src/py_nif.c b/c_src/py_nif.c index 527b738..cb8f459 100644 --- a/c_src/py_nif.c +++ b/c_src/py_nif.c @@ -36,6 +36,14 @@ * - py_callback.c: Callback system and asyncio support */ +/* pthread_timedjoin_np (used to bound the owngil worker join on Linux) + * is declared by only under _GNU_SOURCE. */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include #include "py_nif.h" #include "py_util.h" #include "py_event_loop.h" @@ -8079,6 +8087,26 @@ static void unload(ErlNifEnv *env, void *priv_data) { /* Other cleanup handled by finalize */ } +/** + * @brief Send a signal to an OS process (kill(2)). + * + * Used by isolated contexts to SIGKILL their child. The caller holds the + * child's port open until exit_status arrives, so the pid cannot have been + * recycled. + */ +static ERL_NIF_TERM nif_os_kill(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + (void)argc; + int pid, sig; + if (!enif_get_int(env, argv[0], &pid) || !enif_get_int(env, argv[1], &sig) || pid <= 0) { + return enif_make_badarg(env); + } + if (kill((pid_t)pid, sig) == 0) { + return ATOM_OK; + } + return enif_make_tuple2(env, ATOM_ERROR, + enif_make_atom(env, errno == ESRCH ? "esrch" : errno == EPERM ? "eperm" : "einval")); +} + static ErlNifFunc nif_funcs[] = { /* Initialization */ {"init", 0, nif_py_init, 0}, @@ -8220,6 +8248,7 @@ static ErlNifFunc nif_funcs[] = { {"create_test_pipe", 0, nif_create_test_pipe, 0}, {"close_test_fd", 1, nif_close_test_fd, 0}, {"dup_fd", 1, nif_dup_fd, 0}, + {"os_kill", 2, nif_os_kill, 0}, {"write_test_fd", 2, nif_write_test_fd, 0}, {"read_test_fd", 2, nif_read_test_fd, 0}, /* TCP test helpers */ diff --git a/docs/interrupts.md b/docs/interrupts.md index 84d4465..20461e5 100644 --- a/docs/interrupts.md +++ b/docs/interrupts.md @@ -65,12 +65,29 @@ context to deal with that: ok = py_context:destroy(Ctx). ``` +## Interrupting a blocking C call: isolated mode + +The limits below apply to the embedded modes. An `isolated` context runs +Python in a child process, where an interrupt is a signal that lands inside +`time.sleep`, a socket read or any other blocking call, and `SIGKILL` is the +backstop if the signal is ignored: + +```erlang +{ok, Ctx} = py_context:new(#{mode => isolated, kill_after => 1000}), +{error, timeout} = py_context:eval(Ctx, <<"__import__('time').sleep(60)">>, #{}, 200), +%% Usable at once, no 60 s wait +{ok, 4} = py_context:eval(Ctx, <<"2+2">>, #{}, 5000). +``` + +`py_context:kill/1` kills at once. See [Isolated Contexts](isolated.md). + ## Limits - CPython delivers an async exception at the next bytecode boundary. Code blocked inside a C call (`time.sleep`, a numpy kernel, a socket read) is not interrupted until that call returns. The call still times out on the - Erlang side; the context becomes usable once the C call finishes. + Erlang side; the context becomes usable once the C call finishes. Only + `isolated` mode interrupts such a call. - An interrupt targets the context, not an individual request. Interrupting a context that just finished one call and started another stops the new one. - `py:call/3,4` and `py:eval/1,2` use `infinity` by default. Pass an explicit diff --git a/docs/isolated.md b/docs/isolated.md new file mode 100644 index 0000000..b737bf2 --- /dev/null +++ b/docs/isolated.md @@ -0,0 +1,267 @@ +# Isolated Contexts + +This guide covers `isolated` mode, where a context's CPython interpreter runs +in a child OS process instead of inside the BEAM. You need it when Python +code must not be able to take the node down or run forever: user-supplied +scripts, C extensions you do not control, work that needs a hard time or +memory bound. The public API is the one you already use with `worker` and +`owngil`; you switch by configuration. + +## What the three modes guarantee + +| | `worker` | `owngil` | `isolated` | +|---|---|---|---| +| Where Python runs | BEAM process, one pthread per context | BEAM process, one pthread and one interpreter per context | Child OS process per context | +| Interrupt a Python loop | yes (`KeyboardInterrupt` at the next bytecode) | yes | yes | +| Interrupt a blocking C call (`time.sleep`, socket read, numpy kernel) | no, only when the call returns | no | yes (signal), then `SIGKILL` | +| Hard bound on a call | no: a stuck call keeps its thread until it returns | no | yes: `kill_after` then `SIGKILL` | +| Memory cap | no | obmalloc accounting, C extensions not counted | `RLIMIT_AS` (Linux, FreeBSD), RSS watchdog (macOS), cgroups v2 (Linux) | +| CPU bound | no | no | `RLIMIT_CPU` | +| Segfault in a C extension | kills the node | kills the node | kills the child, caller gets `{error, {child_exited, {signal, 11}}}` | +| Python state after a crash | n/a | n/a | lost; the child restarts and the context stays usable | +| Call latency (`eval("1+1")`, p50, same machine) | 16 us | ~20 us | 25 us | +| Memory per context | shared interpreter | one interpreter | one process, ~16 MB RSS bare | +| Startup | microseconds | milliseconds | ~40 ms | +| Zero-copy `py_buffer`, channels, `erlang.schedule`, object refs | yes | yes | no (see Limits) | + +## Start a context + +```erlang +{ok, Ctx} = py_context:new(#{mode => isolated}), +{ok, 4} = py_context:eval(Ctx, <<"2+2">>), +{ok, 4.0} = py_context:call(Ctx, math, sqrt, [16]), +ok = py_context:exec(Ctx, <<"x = 41">>), +{ok, 42} = py_context:eval(Ctx, <<"x + 1">>), +ok = py_context:stop(Ctx). +``` + +A pool works the same way, and `py:call/4` routes to it: + +```erlang +{ok, _} = py_context_router:start_pool(sandbox, 4, isolated), +{ok, 4.0} = py:call(sandbox, math, sqrt, [16]). +``` + +Options of `py_context:new/1` specific to this mode: + +| Option | Default | Meaning | +|---|---|---| +| `python` | interpreter matching the embedded runtime (`py:python_executable/0`), or `isolated_python` app env | Executable to run | +| `rlimits` | `#{}` | `#{as => Bytes, cpu => Seconds, nofile => N}`, applied with `setrlimit` before any user code | +| `cgroup` | none | Path of a cgroup v2 directory the child joins (limits written by you: `memory.max`, `cpu.max`, `pids.max`) | +| `env` | `#{}` | Extra environment variables for the child | +| `paths` | `[]` | Extra `sys.path` entries (registered `py_import` paths and imports are applied too) | +| `preload` | none | Code run once in the child before anything else | +| `kill_after` | `1000` | Milliseconds between a soft interrupt and `SIGKILL` | +| `restart` | `true` | Start a fresh child when the current one dies | +| `max_restarts`, `restart_period` | `5`, `10000` | Restart budget; past it the context process exits with the child's reason | +| `start_timeout` | `10000` | Milliseconds allowed for the child to connect | + +## Cancel work that ignores the embedded modes + +`py_context:interrupt/1` sends the child a signal that raises +`KeyboardInterrupt` in the running request, inside a blocking C call too. If +the request has not returned after `kill_after`, the child is killed: + +```erlang +{ok, Ctx} = py_context:new(#{mode => isolated, kill_after => 500}), +Self = self(), +spawn(fun() -> Self ! {done, py_context:eval(Ctx, <<"__import__('time').sleep(60)">>)} end), +timer:sleep(100), +ok = py_context:interrupt(Ctx), +receive {done, {error, interrupted}} -> ok end. +``` + +A timeout targets only the request that timed out: if the child is +executing it, the child is interrupted (and killed after `kill_after` if the +interrupt is not honoured); if it is still queued behind other callers' +requests, it is dropped from the queue and nobody else is interrupted. This +differs from the embedded modes, where an interrupt can only hit whatever +runs. `py_context:interrupt/1` remains context-wide: it interrupts the +request executing now. `py_context:kill/1` skips the soft step: + +```erlang +ok = py_context:kill(Ctx), +%% A fresh child is already serving; the Python state is gone +{ok, 4} = py_context:eval(Ctx, <<"2+2">>). +``` + +In-flight calls return `{error, interrupted}` (soft) or `{error, killed}` +(hard). + +## Bound memory and CPU + +```erlang +{ok, Ctx} = py_context:new(#{mode => isolated, + rlimits => #{as => 512 * 1024 * 1024, + cpu => 30, + nofile => 256}}). +``` + +Past `as`, allocations fail with `MemoryError` in the child (or the child +dies if it cannot cope); past `cpu`, the child dies with `SIGXCPU` and the +caller gets `{error, {child_exited, {signal, 24}}}`. rlimits are POSIX and +are the portable bound: Linux and FreeBSD enforce all three in the kernel. +macOS ignores `RLIMIT_AS`, so there the child enforces `as` itself: a +watchdog thread polls its resident set every 50 ms and exits when it passes +the limit, and the caller gets `{error, {child_exited, {memory_limit, Bytes}}}`. +`cpu` and `nofile` are kernel-enforced on all three. A free-threaded +CPython build reserves a large virtual range at startup, so its `as` limit +must be well above what a regular build needs (several GB). + +With cgroups v2 (Linux only), create the group and write the limits, then +hand the directory to the context. On any other platform the option is +refused before a child is spawned, with `{error, {cgroup_unsupported, Os}}`: + +```sh +mkdir /sys/fs/cgroup/py_sandbox +echo 268435456 > /sys/fs/cgroup/py_sandbox/memory.max +echo "50000 100000" > /sys/fs/cgroup/py_sandbox/cpu.max +echo 64 > /sys/fs/cgroup/py_sandbox/pids.max +``` + +```erlang +{ok, Ctx} = py_context:new(#{mode => isolated, cgroup => "/sys/fs/cgroup/py_sandbox"}). +``` + +The child joins the group before running any user code; if it cannot, the +start fails with `{error, {startup_error, [{cgroup, Reason}]}}`. + +## Survive a crash + +```erlang +{ok, Ctx} = py_context:new(#{mode => isolated}), +{error, {child_exited, {signal, 11}}} = + py_context:eval(Ctx, <<"__import__('ctypes').memset(0, 0, 1)">>), +{ok, 4} = py_context:eval(Ctx, <<"2+2">>). +``` + +The node, every other context and the context process itself are unaffected. +With `restart => false` the context process exits with +`{child_exited, Reason}` instead, so a supervisor of yours decides. + +## Call Erlang from the child + +`erlang.call`, `erlang.send`, `erlang.whereis`, `erlang.Pid`, `erlang.Atom` +work as in the embedded modes, from any Python thread, and callbacks nest: an +Erlang function called from Python may call back into the same context, from +the process running the callback. The context serves one caller at a time, +in order; a nested call from the callback process goes through at once, +while a call from any other process waits for the current request to finish +(so a callback that hands the nested call to another process and waits for +it would wait until its own timeout). + +```erlang +py:register_function(double, fun([X]) -> X * 2 end), +{ok, 84} = py_context:eval(Ctx, <<"__import__('erlang').call('double', 42)">>). +``` + +```python +import erlang + +def notify(pid): + erlang.send(pid, ('progress', 50)) # raises erlang.ProcessError if pid is dead +``` + +Round trips cross a Unix socket as external term format; the type mapping is +the one in [Type Conversion](type-conversion.md). One difference: integers +beyond 64 bits arrive intact (the NIF converter has no bignum path). + +## asyncio and worker loops + +The child runs a plain `asyncio` loop. A call that returns a coroutine is +awaited and its value returned: + +```python +async def fetch(n): + await asyncio.sleep(0.1) + return n * 2 +``` + +```erlang +{ok, 4} = py_context:call(Ctx, myapp, fetch, [2]). +``` + +The worker loop API of [Worker Loops](workers.md) works unchanged +(`start_loop`, `submit`, `submit_await`, `stop_loop`), and a wedged loop is +killed by `stop_loop/2` after its grace period. To serve on a socket Erlang +owns, hand the fd over with `py_context:pass_fd/2`; it crosses the control +socket with `SCM_RIGHTS`: + +```erlang +{ok, LSock} = gen_tcp:listen(8080, [binary, {active, false}]), +{ok, Fd} = inet:getfd(LSock), +[begin + {ok, Ctx} = py_context:new(#{mode => isolated}), + ok = py_context:start_loop(Ctx), + {ok, ChildFd} = py_context:pass_fd(Ctx, Fd), + {ok, _} = py_context:submit_await(Ctx, myapp, serve, [ChildFd]) + end || _ <- lists:seq(1, 4)]. +``` + +Inside a coroutine, `await erlang.async_call(name, *args)` keeps the loop +running while Erlang answers. + +## Process model + +- One child per context, started with `open_port` so the VM reaps it and + reports its exit status: no zombies, and a child that dies before + connecting is reported with its output (`{error, {child_exited_at_start, Reason, Output}}`). +- The context process is a `gen_statem` (`py_isolated`) with states + `idle`, `{busy, RequestId}`, `looping`, `stopping_loop` and + `{restarting, Reason}`; `sys:get_state(Ctx)` shows what it is doing and + `sys:trace(Ctx, true)` prints its events. It serves one caller at a time, + in order; a request that arrives while the child restarts waits for the + new child instead of failing. It outlives the process that created it, + and stops if that process crashes. +- The child and the context process talk over a Unix socket in a private + directory, framed exactly like the embedded callback pipe + (`<>`, body `<>`). +- A reader thread in the child owns the socket. It delivers requests to the + main thread, routes replies to whichever thread is waiting, handles + `interrupt` by signalling the main thread, and exits the process on EOF. + So when the BEAM dies, every child exits, even one stuck in a C call; on + Linux (`prctl(PR_SET_PDEATHSIG)`) and FreeBSD (`procctl(PROC_PDEATHSIG_CTL)`) + the kernel delivers `SIGKILL` for the same case. +- When the socket breaks or the child exits, pending calls fail with + `{error, {child_exited, Reason}}`, new calls fail the same way until the + restart has happened (which takes about 100 ms), and nothing hangs. +- Child stdout and stderr are forwarded to the Erlang logger, one line per + message, tagged with the context id and OS pid. `py_context:child_info/1` + returns `os_pid`, `python_version`, `executable` and `platform`. + +## Limits + +- Python object references cannot cross a process boundary: + `py_context:call_method/4` returns `{error, not_supported_in_isolated}`, + results are always converted to terms, and process-local environments + (`py:call(Ctx, ...)` per Erlang process) map to the child's single + namespace. +- `erlang.schedule*`, channels (`py_channel`, `py_byte_channel`), + `py_buffer`, shared dicts and the reactor need the embedded interpreter and + raise `RuntimeError("... not available in isolated mode")`. +- `py_context:loop_ref/1` returns `{error, not_supported_in_isolated}`; + `submit/4` without a running loop returns `{error, no_loop}` (there is no + event worker to step an idle loop). +- `erlang.call` from inside a coroutine blocks the loop, as in the embedded + modes; use `erlang.async_call`. +- The child decodes terms with the same rules as the NIF, so atoms sent from + Python are created in the VM's atom table. Do not let untrusted code mint + unbounded distinct atoms. +- No syscall filtering: process isolation plus rlimits is the boundary. A + seccomp (Linux) or Capsicum (FreeBSD) sandbox is a separate hardening step. +- Each call copies its arguments and result through the socket: a 1 MB + binary round-trips in about 1.3 ms, 16 MB in about 27 ms (worker mode: + 0.2 ms and 3 ms). For bulk data prefer a file, a socket the child reads + itself, or a shared mapping: `erlang-iommap` opens a file `MAP_SHARED` and + `region_binary/3` gives a binary over it, while the child maps the same + file with `mmap`; a 64 MB region costs 3 us on the Erlang side and 5 ms to + map in Python, against 12 ms to copy it through ETF. `py_buffer` is not + ported to that yet. + +## See also + +- [Interrupts](interrupts.md) for the embedded-mode interrupt semantics +- [Worker Loops](workers.md) for the loop API and serving on Erlang sockets +- [Memory](memory.md) for the owngil memory caps +- [Security](security.md) for the audit-hook sandbox of the embedded modes diff --git a/docs/security.md b/docs/security.md index 5ba726a..db402eb 100644 --- a/docs/security.md +++ b/docs/security.md @@ -144,6 +144,23 @@ if is_sandboxed(): print("Running inside Erlang VM - subprocess operations blocked") ``` +## Process Isolation + +The audit hook keeps Python from forking the VM; it does not protect the VM +from Python. A C extension can still segfault the node, and nothing can cap +the memory or CPU of embedded code. For that boundary run the context in a +child process: + +```erlang +{ok, Ctx} = py_context:new(#{mode => isolated, + rlimits => #{as => 256 * 1024 * 1024, cpu => 10}, + kill_after => 1000}). +``` + +A crash kills only the child, `py_context:kill/1` is total, and rlimits or +cgroups bound resources. See [Isolated Contexts](isolated.md). The child is +not sandboxed at the syscall level; that is a separate hardening step. + ## Signal Handling Note Signal handling is also not supported in the Erlang event loop. The `ErlangEventLoop` raises `NotImplementedError` for `add_signal_handler()` and `remove_signal_handler()`. Signal handling should be done at the Erlang VM level using Erlang's signal handling facilities. diff --git a/docs/workers.md b/docs/workers.md index 85fea63..74b83bf 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -28,6 +28,11 @@ Worker contexts get the same API on the shared main interpreter loop, which allows one running `ErlangEventLoop` per interpreter: use owngil (Python 3.14+) for several workers. +Isolated contexts (`mode => isolated`) get the same API on a plain asyncio +loop in their child process; hand sockets over with `py_context:pass_fd/2` +instead of `py:dup_fd/1`, and a wedged loop is killed by `stop_loop/2`. See +[Isolated Contexts](isolated.md). + ## Serve TCP on a socket Erlang owns Bind once in Erlang, duplicate the listen fd for each worker with diff --git a/priv/_erlang_impl/_etf.py b/priv/_erlang_impl/_etf.py new file mode 100644 index 0000000..47fceec --- /dev/null +++ b/priv/_erlang_impl/_etf.py @@ -0,0 +1,477 @@ +# Copyright 2026 Benoit Chesneau +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Erlang external term format (ETF) codec, pure Python. + +Used by the isolated child process, where no NIF is available. The type +mapping mirrors c_src/py_convert.c so code behaves the same in `worker`, +`owngil` and `isolated` mode: + + Erlang -> Python Python -> Erlang + true / false -> True / False None -> none + none/nil/undefined-> None bool -> true / false + other atom -> str int -> integer (any size) + integer -> int float -> float (nan/inf -> atoms) + float -> float str, bytes -> binary + binary (utf-8) -> str list -> list + binary (other) -> bytes tuple -> tuple + {bytes, Bin} -> bytes dict -> map + list / string -> list Atom -> atom + tuple -> tuple Pid / Ref / Port-> pid / ref / port + map -> dict numpy ndarray -> list (tolist) + pid / ref / port -> Pid / Ref / Port other object -> binary(str(obj)) + +Pids, refs and ports are opaque: the decoder keeps their raw ETF bytes and +the encoder emits them unchanged, so they round-trip exactly. +""" + +import math +import struct + +__all__ = [ + 'Atom', 'Pid', 'Ref', 'Port', + 'encode', 'decode', 'DecodeError', +] + +VERSION = 131 + +# Tags +NEW_FLOAT_EXT = 70 +BIT_BINARY_EXT = 77 +NEW_PID_EXT = 88 +NEW_PORT_EXT = 89 +NEWER_REFERENCE_EXT = 90 +SMALL_INTEGER_EXT = 97 +INTEGER_EXT = 98 +FLOAT_EXT = 99 +ATOM_EXT = 100 +REFERENCE_EXT = 101 +PORT_EXT = 102 +PID_EXT = 103 +SMALL_TUPLE_EXT = 104 +LARGE_TUPLE_EXT = 105 +NIL_EXT = 106 +STRING_EXT = 107 +LIST_EXT = 108 +BINARY_EXT = 109 +SMALL_BIG_EXT = 110 +LARGE_BIG_EXT = 111 +NEW_REFERENCE_EXT = 114 +SMALL_ATOM_EXT = 115 +MAP_EXT = 116 +ATOM_UTF8_EXT = 118 +SMALL_ATOM_UTF8_EXT = 119 +V4_PORT_EXT = 120 + + +class DecodeError(ValueError): + """Raised on malformed external term format data.""" + + +class Atom(str): + """An Erlang atom. A str subclass so `Atom('ok') == 'ok'`, and so code + written for the embedded erlang.Atom keeps working.""" + + __slots__ = () + + def __new__(cls, name): + if isinstance(name, bytes): + name = name.decode('utf-8') + if not isinstance(name, str): + raise TypeError('atom name must be str') + return str.__new__(cls, name) + + def __repr__(self): + return 'erlang.Atom(%r)' % str.__str__(self) + + +class _Opaque: + """Base for pid/ref/port: value semantics over the raw ETF bytes.""" + + __slots__ = ('_raw',) + + def __init__(self, raw): + if not isinstance(raw, (bytes, bytearray)): + raise TypeError('%s wants raw ETF bytes' % type(self).__name__) + self._raw = bytes(raw) + + def __eq__(self, other): + return type(other) is type(self) and other._raw == self._raw + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((type(self).__name__, self._raw)) + + def __repr__(self): + return '' % (type(self).__name__, self._raw.hex()) + + @property + def raw(self): + return self._raw + + +class Pid(_Opaque): + __slots__ = () + + def __repr__(self): + try: + node, ident, serial, _ = _decode_pid_fields(self._raw) + return '@%s>' % (ident, serial, node) + except Exception: + return _Opaque.__repr__(self) + + +class Ref(_Opaque): + __slots__ = () + + +class Port(_Opaque): + __slots__ = () + + +# --------------------------------------------------------------------------- +# Encoding +# --------------------------------------------------------------------------- + +_pack_u8 = struct.Struct('>B').pack +_pack_u16 = struct.Struct('>H').pack +_pack_u32 = struct.Struct('>I').pack +_pack_i32 = struct.Struct('>i').pack +_pack_f64 = struct.Struct('>d').pack + + +def encode(obj): + """Encode a Python object as a complete ETF binary (with version byte).""" + out = bytearray([VERSION]) + _encode(obj, out) + return bytes(out) + + +def _encode_atom(name, out): + data = name.encode('utf-8') + n = len(data) + if n > 255: + raise ValueError('atom too long: %d bytes' % n) + if n < 256: + out += _pack_u8(SMALL_ATOM_UTF8_EXT) + out += _pack_u8(n) + out += data + + +def _encode(obj, out): + if obj is None: + _encode_atom('none', out) + elif obj is True: + _encode_atom('true', out) + elif obj is False: + _encode_atom('false', out) + elif isinstance(obj, Atom): + _encode_atom(str.__str__(obj), out) + elif isinstance(obj, int): + _encode_int(obj, out) + elif isinstance(obj, float): + if math.isnan(obj): + _encode_atom('nan', out) + elif math.isinf(obj): + _encode_atom('infinity' if obj > 0 else 'neg_infinity', out) + else: + out += _pack_u8(NEW_FLOAT_EXT) + out += _pack_f64(obj) + elif isinstance(obj, str): + data = obj.encode('utf-8', 'surrogatepass') + out += _pack_u8(BINARY_EXT) + out += _pack_u32(len(data)) + out += data + elif isinstance(obj, (bytes, bytearray, memoryview)): + data = bytes(obj) + out += _pack_u8(BINARY_EXT) + out += _pack_u32(len(data)) + out += data + elif isinstance(obj, _Opaque): + out += obj._raw + elif isinstance(obj, tuple): + n = len(obj) + if n < 256: + out += _pack_u8(SMALL_TUPLE_EXT) + out += _pack_u8(n) + else: + out += _pack_u8(LARGE_TUPLE_EXT) + out += _pack_u32(n) + for item in obj: + _encode(item, out) + elif isinstance(obj, list): + n = len(obj) + if n == 0: + out += _pack_u8(NIL_EXT) + else: + out += _pack_u8(LIST_EXT) + out += _pack_u32(n) + for item in obj: + _encode(item, out) + out += _pack_u8(NIL_EXT) + elif isinstance(obj, dict): + out += _pack_u8(MAP_EXT) + out += _pack_u32(len(obj)) + for k, v in obj.items(): + _encode(k, out) + _encode(v, out) + elif _is_ndarray(obj): + _encode(obj.tolist(), out) + elif isinstance(obj, (set, frozenset)): + _encode(list(obj), out) + else: + # Same fallback as py_to_term: the string representation as a binary + _encode(str(obj), out) + + +def _encode_int(value, out): + if 0 <= value <= 255: + out += _pack_u8(SMALL_INTEGER_EXT) + out += _pack_u8(value) + elif -2147483648 <= value <= 2147483647: + out += _pack_u8(INTEGER_EXT) + out += _pack_i32(value) + else: + sign = 1 if value < 0 else 0 + mag = -value if sign else value + n = (mag.bit_length() + 7) // 8 + digits = mag.to_bytes(n, 'little') + if n < 256: + out += _pack_u8(SMALL_BIG_EXT) + out += _pack_u8(n) + else: + out += _pack_u8(LARGE_BIG_EXT) + out += _pack_u32(n) + out += _pack_u8(sign) + out += digits + + +def _is_ndarray(obj): + t = type(obj) + if t.__module__ == 'numpy' and t.__name__ == 'ndarray': + return True + return hasattr(obj, 'tolist') and hasattr(obj, 'ndim') + + +# --------------------------------------------------------------------------- +# Decoding +# --------------------------------------------------------------------------- + +_unpack_u16 = struct.Struct('>H').unpack_from +_unpack_u32 = struct.Struct('>I').unpack_from +_unpack_i32 = struct.Struct('>i').unpack_from +_unpack_f64 = struct.Struct('>d').unpack_from + +_ATOM_TRUE = 'true' +_ATOM_FALSE = 'false' +_NONE_ATOMS = frozenset(('none', 'nil', 'undefined')) + + +def decode(data): + """Decode a complete ETF binary (with version byte) to a Python object.""" + if not data or data[0] != VERSION: + raise DecodeError('bad ETF version byte') + value, pos = _decode(data, 1) + if pos != len(data): + raise DecodeError('trailing bytes after term') + return value + + +def _atom_value(name): + """Map an atom to its Python value the way term_to_py does.""" + if name == _ATOM_TRUE: + return True + if name == _ATOM_FALSE: + return False + if name in _NONE_ATOMS: + return None + return name + + +def _decode(data, pos): + try: + tag = data[pos] + except IndexError: + raise DecodeError('truncated term') from None + pos += 1 + + if tag == SMALL_INTEGER_EXT: + return data[pos], pos + 1 + if tag == INTEGER_EXT: + return _unpack_i32(data, pos)[0], pos + 4 + if tag == BINARY_EXT: + (n,) = _unpack_u32(data, pos) + pos += 4 + raw = bytes(data[pos:pos + n]) + if len(raw) != n: + raise DecodeError('truncated binary') + try: + return raw.decode('utf-8'), pos + n + except UnicodeDecodeError: + return raw, pos + n + if tag == SMALL_ATOM_UTF8_EXT: + n = data[pos] + pos += 1 + return _atom_value(bytes(data[pos:pos + n]).decode('utf-8')), pos + n + if tag == ATOM_UTF8_EXT: + (n,) = _unpack_u16(data, pos) + pos += 2 + return _atom_value(bytes(data[pos:pos + n]).decode('utf-8')), pos + n + if tag == ATOM_EXT: + (n,) = _unpack_u16(data, pos) + pos += 2 + return _atom_value(bytes(data[pos:pos + n]).decode('latin-1')), pos + n + if tag == SMALL_ATOM_EXT: + n = data[pos] + pos += 1 + return _atom_value(bytes(data[pos:pos + n]).decode('latin-1')), pos + n + if tag == SMALL_TUPLE_EXT or tag == LARGE_TUPLE_EXT: + if tag == SMALL_TUPLE_EXT: + n = data[pos] + pos += 1 + else: + (n,) = _unpack_u32(data, pos) + pos += 4 + items = [] + for _ in range(n): + item, pos = _decode(data, pos) + items.append(item) + # {bytes, Bin}: explicit bytes, as in term_to_py + if n == 2 and items[0] == 'bytes' and isinstance(items[1], (str, bytes)): + b = items[1] + if isinstance(b, str): + b = b.encode('utf-8', 'surrogatepass') + return b, pos + return tuple(items), pos + if tag == NIL_EXT: + return [], pos + if tag == STRING_EXT: + (n,) = _unpack_u16(data, pos) + pos += 2 + return list(data[pos:pos + n]), pos + n + if tag == LIST_EXT: + (n,) = _unpack_u32(data, pos) + pos += 4 + items = [] + for _ in range(n): + item, pos = _decode(data, pos) + items.append(item) + # Tail: NIL for a proper list. An improper tail is kept as a final + # element, the same as enif_get_list_length failing is not an option + # here; this never happens for term_to_binary of proper lists. + if data[pos] == NIL_EXT: + pos += 1 + else: + tail, pos = _decode(data, pos) + items.append(tail) + return items, pos + if tag == MAP_EXT: + (n,) = _unpack_u32(data, pos) + pos += 4 + result = {} + for _ in range(n): + k, pos = _decode(data, pos) + v, pos = _decode(data, pos) + result[_hashable(k)] = v + return result, pos + if tag == NEW_FLOAT_EXT: + return _unpack_f64(data, pos)[0], pos + 8 + if tag == FLOAT_EXT: + text = bytes(data[pos:pos + 31]).split(b'\x00', 1)[0] + return float(text), pos + 31 + if tag == SMALL_BIG_EXT or tag == LARGE_BIG_EXT: + if tag == SMALL_BIG_EXT: + n = data[pos] + pos += 1 + else: + (n,) = _unpack_u32(data, pos) + pos += 4 + sign = data[pos] + pos += 1 + value = int.from_bytes(bytes(data[pos:pos + n]), 'little') + return (-value if sign else value), pos + n + if tag in (NEW_PID_EXT, PID_EXT): + start = pos - 1 + _, pos = _decode_atom_raw(data, pos) # node + pos += 8 # id, serial + pos += 4 if tag == NEW_PID_EXT else 1 # creation + return Pid(data[start:pos]), pos + if tag in (NEWER_REFERENCE_EXT, NEW_REFERENCE_EXT): + start = pos - 1 + (n,) = _unpack_u16(data, pos) + pos += 2 + _, pos = _decode_atom_raw(data, pos) + pos += 4 if tag == NEWER_REFERENCE_EXT else 1 + pos += 4 * n + return Ref(data[start:pos]), pos + if tag == REFERENCE_EXT: + start = pos - 1 + _, pos = _decode_atom_raw(data, pos) + pos += 5 + return Ref(data[start:pos]), pos + if tag in (NEW_PORT_EXT, PORT_EXT, V4_PORT_EXT): + start = pos - 1 + _, pos = _decode_atom_raw(data, pos) + pos += 8 if tag == V4_PORT_EXT else 4 + pos += 1 if tag == PORT_EXT else 4 + return Port(data[start:pos]), pos + if tag == BIT_BINARY_EXT: + (n,) = _unpack_u32(data, pos) + pos += 4 + bits = data[pos] + pos += 1 + raw = bytes(data[pos:pos + n]) + return (raw, bits), pos + n + raise DecodeError('unsupported ETF tag %d' % tag) + + +def _decode_atom_raw(data, pos): + """Decode an atom (any encoding) returning its name; used inside + pid/ref/port where the value is not mapped.""" + tag = data[pos] + pos += 1 + if tag == SMALL_ATOM_UTF8_EXT or tag == SMALL_ATOM_EXT: + n = data[pos] + pos += 1 + elif tag == ATOM_UTF8_EXT or tag == ATOM_EXT: + (n,) = _unpack_u16(data, pos) + pos += 2 + else: + raise DecodeError('expected atom, got tag %d' % tag) + return bytes(data[pos:pos + n]).decode('utf-8', 'replace'), pos + n + + +def _decode_pid_fields(raw): + tag = raw[0] + node, pos = _decode_atom_raw(raw, 1) + (ident,) = _unpack_u32(raw, pos) + (serial,) = _unpack_u32(raw, pos + 4) + pos += 8 + if tag == NEW_PID_EXT: + (creation,) = _unpack_u32(raw, pos) + else: + creation = raw[pos] + return node, ident, serial, creation + + +def _hashable(key): + """Map keys must be hashable; lists (Erlang lists/strings) become tuples.""" + if isinstance(key, list): + return tuple(_hashable(k) for k in key) + if isinstance(key, dict): + return tuple(sorted((_hashable(k), _hashable(v)) for k, v in key.items())) + return key diff --git a/priv/_erlang_impl/_isolated.py b/priv/_erlang_impl/_isolated.py new file mode 100644 index 0000000..a276c36 --- /dev/null +++ b/priv/_erlang_impl/_isolated.py @@ -0,0 +1,824 @@ +# Copyright 2026 Benoit Chesneau +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Runtime of an `isolated` context: the far end of the socket that +src/py_isolated.erl talks to. + +Wire format, shared with the blocking callback pipe of the embedded modes: + + <> + Body = <> Payload is ETF unless noted + + Status 0 request, Erlang -> child {call,M,F,A,K} | {eval,Code,Locals} + | {exec,Code} | ping | shutdown + | start_loop | stop_loop + | {submit,Ref,M,F,A,K} | pass_fd + Status 1 error reply (either way) reason term + Status 2 ok reply (either way) value term + Status 3 request, child -> Erlang {call,Name,Args} | {send,Pid,Msg} + | {whereis,Name} + Status 4 event, child -> Erlang {ready,Info} | {startup_error,R} + | {async_result,Ref,R} + | {loop_exit,R} | {log,Level,Msg} + Status 5 control, Erlang -> child interrupt (handled on the reader + thread, never queued) + +Threads: + * the main thread executes requests, one at a time, and owns the asyncio + loop; while it waits for the reply to an erlang.call it keeps serving + requests, so a callback may call back into this context (nesting); + * the reader thread owns socket reads. It routes replies to whoever is + waiting (any Python thread may use erlang.call/send/whereis), queues + requests for the main thread, handles `interrupt` by signalling the main + thread, and exits the process on EOF so a BEAM death never leaves an + orphan, even if the main thread is stuck in a C call. +""" + +import asyncio +import inspect +import os +import queue +import signal +import socket +import struct +import sys +import threading +import traceback + +from . import _etf +from ._etf import Atom, Pid, Ref, Port, DecodeError + +__all__ = ['Runtime', 'install_erlang_module'] + +STATUS_REQUEST = 0 +STATUS_ERROR = 1 +STATUS_OK = 2 +STATUS_CALLBACK = 3 +STATUS_EVENT = 4 +STATUS_CONTROL = 5 + +_HEADER = struct.Struct('=QI') # native byte order, no padding +_HEADER_LEN = _HEADER.size + +_INTERRUPT_SIGNAL = signal.SIGUSR1 + + +class ProcessError(Exception): + """Raised by erlang.send when the target process does not exist.""" + + +class SuspensionRequired(BaseException): + """Kept for source compatibility with the embedded erlang module. It is + never raised in isolated mode: callbacks are real socket round-trips.""" + + +class PipeBroken(RuntimeError): + """The control socket to Erlang is gone.""" + + +class _Interrupted(KeyboardInterrupt): + """KeyboardInterrupt raised by the interrupt signal handler. A subclass so + user code catching KeyboardInterrupt keeps working, while the dispatcher + can tell an Erlang interrupt from a stray Ctrl-C.""" + + +class _Waiter: + """Reply slot for a request this process sent to Erlang.""" + + __slots__ = ('event', 'result', 'inbox', 'future', 'loop') + + def __init__(self, inbox=None, future=None, loop=None): + self.event = None if (inbox is not None or future is not None) else threading.Event() + self.result = None + self.inbox = inbox + self.future = future + self.loop = loop + + def deliver(self, result): + self.result = result + if self.inbox is not None: + self.inbox.put(('reply', self)) + elif self.future is not None: + try: + self.loop.call_soon_threadsafe(_resolve_future, self.future, result) + except RuntimeError: + pass # the user's loop is closed: nobody is waiting + else: + self.event.set() + + +def _resolve_future(future, result): + if future.cancelled(): + return + status, value = result + if status == STATUS_OK: + future.set_result(value) + else: + future.set_exception(_callback_error(value)) + + +def _callback_error(reason): + if isinstance(reason, str): + return RuntimeError(reason) + if isinstance(reason, tuple) and len(reason) == 2 and reason[0] == 'noproc': + return ProcessError('process %r does not exist' % (reason[1],)) + return RuntimeError('erlang call failed: %r' % (reason,)) + + +class Runtime: + """One per child process.""" + + def __init__(self, sock, context_pid=None): + self.sock = sock + self.context_pid = context_pid + self._wlock = threading.Lock() + self._idlock = threading.Lock() + self._next_id = 1 + self._pending = {} + self._plock = threading.Lock() + self.inbox = queue.SimpleQueue() + self.broken = False + self.broken_reason = None + self.globals = {'__name__': '__main__', '__builtins__': __builtins__} + self.running = False # a request is executing on main thread + self.loop = None + self.loop_running = False + self._received_fds = [] + self._fds_lock = threading.Lock() + self._cancelled = set() # request ids Erlang gave up on + self._cancel_lock = threading.Lock() + self._exec_stack = [] # main thread: ids being executed (nesting) + self.main_thread = threading.main_thread() + self._reader = None + + # -- lifecycle --------------------------------------------------------- + + def start(self): + signal.signal(_INTERRUPT_SIGNAL, self._on_interrupt_signal) + self._reader = threading.Thread(target=self._reader_main, + name='erlang-reader', daemon=True) + self._reader.start() + + def get_loop(self): + if self.loop is None: + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + return self.loop + + # -- signals ----------------------------------------------------------- + + def _on_interrupt_signal(self, signum, frame): + # Only a request in flight can be interrupted: an interrupt that + # lands between two requests is dropped, so it can never leak into + # the next one. + if self.running: + raise _Interrupted() + + def _signal_main(self): + try: + signal.pthread_kill(self.main_thread.ident, _INTERRUPT_SIGNAL) + except Exception: + pass + + # -- writing ----------------------------------------------------------- + + def _write_frame(self, frame_id, status, payload): + body = bytes([status]) + payload + data = _HEADER.pack(frame_id, len(body)) + body + # A signal landing inside sendall would tear the frame and + # desynchronise the stream: hold it until the write is complete. + on_main = threading.current_thread() is self.main_thread + if on_main: + signal.pthread_sigmask(signal.SIG_BLOCK, {_INTERRUPT_SIGNAL}) + try: + with self._wlock: + if self.broken: + raise PipeBroken(self.broken_reason or 'socket to Erlang is closed') + try: + self.sock.sendall(data) + except OSError as exc: + self._mark_broken('write failed: %s' % exc) + raise PipeBroken(self.broken_reason) from None + except BaseException: + # Anything else escaping mid-write leaves a torn frame + self._mark_broken('write interrupted') + raise + finally: + if on_main: + signal.pthread_sigmask(signal.SIG_UNBLOCK, {_INTERRUPT_SIGNAL}) + + def reply(self, frame_id, status, value): + self._write_frame(frame_id, status, _etf.encode(value)) + + def event(self, term): + self._write_frame(0, STATUS_EVENT, _etf.encode(term)) + + def _alloc_id(self): + with self._idlock: + n = self._next_id + self._next_id = n + 1 + return n + + # -- child -> Erlang requests ------------------------------------------- + + def request(self, term, timeout=None): + """Send a status-3 request and wait for its reply. + + On the main thread the wait also serves requests coming from Erlang, + so nested calls work. Returns (status, value).""" + if self.broken: + raise PipeBroken(self.broken_reason) + on_main = threading.current_thread() is self.main_thread + waiter = _Waiter(inbox=self.inbox if on_main else None) + frame_id = self._alloc_id() + with self._plock: + self._pending[frame_id] = waiter + try: + self._write_frame(frame_id, STATUS_CALLBACK, _etf.encode(term)) + except PipeBroken: + with self._plock: + self._pending.pop(frame_id, None) + raise + if on_main: + return self._wait_on_main(waiter) + if not waiter.event.wait(timeout): + with self._plock: + self._pending.pop(frame_id, None) + raise TimeoutError('no reply from Erlang') + return waiter.result + + def request_async(self, term): + """Send a status-3 request; returns an asyncio Future for the reply.""" + if self.broken: + raise PipeBroken(self.broken_reason) + loop = asyncio.get_running_loop() + future = loop.create_future() + waiter = _Waiter(future=future, loop=loop) + frame_id = self._alloc_id() + with self._plock: + self._pending[frame_id] = waiter + try: + self._write_frame(frame_id, STATUS_CALLBACK, _etf.encode(term)) + except PipeBroken: + with self._plock: + self._pending.pop(frame_id, None) + raise + return future + + def _wait_on_main(self, waiter): + while True: + kind, item = self.inbox.get() + if kind == 'reply': + if item is waiter: + return waiter.result + # stale reply for an interrupted wait: drop + continue + if kind == 'request': + self._serve(*item) + elif kind == 'broken': + raise PipeBroken(item) + elif kind == 'interrupt': + if self._exec_stack and self._exec_stack[-1] == item: + raise _Interrupted() + # stale: for a request that already finished + + # -- reader thread ----------------------------------------------------- + + def _reader_main(self): + try: + self._read_loop() + except Exception as exc: # never leave silently + self._mark_broken('reader failed: %r' % (exc,)) + # EOF or error: Erlang is gone (or closed us on purpose). Nothing + # useful can happen in this process any more. _exit so a main thread + # stuck in a C call cannot keep the process alive. + os._exit(0) + + def _read_loop(self): + sock = self.sock + buf = bytearray() + need_hdr = _HEADER_LEN + while True: + try: + data, fds, _flags, _addr = socket.recv_fds(sock, 1024 * 1024, 16) + except InterruptedError: + continue + except OSError as exc: + self._mark_broken('read failed: %s' % exc) + return + if fds: + with self._fds_lock: + self._received_fds.extend(fds) + if not data: + self._mark_broken('Erlang closed the socket') + return + buf += data + # Resumable parse: header, then body, buffering partial frames + while True: + if len(buf) < need_hdr: + break + frame_id, body_len = _HEADER.unpack_from(buf, 0) + total = _HEADER_LEN + body_len + if len(buf) < total: + break + body = bytes(buf[_HEADER_LEN:total]) + del buf[:total] + self._on_frame(frame_id, body) + + def _on_frame(self, frame_id, body): + if not body: + self._mark_broken('empty frame') + return + status = body[0] + try: + term = _etf.decode(body[1:]) if len(body) > 1 else None + except (DecodeError, struct.error, IndexError) as exc: + if status == STATUS_REQUEST: + self.reply(frame_id, STATUS_ERROR, + (Atom('bad_request'), 'malformed frame: %s' % exc)) + return + self._mark_broken('malformed frame from Erlang: %s' % exc) + return + if status in (STATUS_OK, STATUS_ERROR): + with self._plock: + waiter = self._pending.pop(frame_id, None) + if waiter is not None: + waiter.deliver((status, term)) + elif status == STATUS_CONTROL: + self._on_control(term) + elif status == STATUS_REQUEST: + self._on_request(frame_id, term) + else: + self._mark_broken('unexpected status %d from Erlang' % status) + + def _on_control(self, term): + if term == 'interrupt': + self._signal_main() + elif isinstance(term, tuple) and len(term) == 2 and term[0] == 'interrupt': + target = term[1] + stack = list(self._exec_stack) + if target == 'loop': + if self.loop_running: + self._signal_main() + elif stack and stack[-1] == target: + self._signal_main() + elif target in stack: + # An outer request blocked in a callback wait: its wait + # raises when the nested request finishes + self.inbox.put(('interrupt', target)) + # else: already finished (or still queued: cancel handles that) + elif isinstance(term, tuple) and len(term) == 2 and term[0] == 'cancel': + with self._cancel_lock: + self._cancelled.add(term[1]) + # A cancel that arrives after its request ran stays behind; + # keep the set bounded (ids only grow, oldest are stalest) + if len(self._cancelled) > 1024: + for old in sorted(self._cancelled)[:512]: + self._cancelled.discard(old) + elif term == 'stop_loop': + loop = self.loop + if loop is not None and self.loop_running: + loop.call_soon_threadsafe(loop.stop) + + def _on_request(self, frame_id, term): + """Requests that must not wait for the main thread are handled here; + everything else is queued for it.""" + tag = term[0] if isinstance(term, tuple) else term + if tag == 'ping': + self.reply(frame_id, STATUS_OK, Atom('pong')) + elif tag == 'submit': + self._on_submit(frame_id, term) + elif tag == 'stop_loop': + loop = self.loop + if loop is not None and self.loop_running: + loop.call_soon_threadsafe(loop.stop) + self.reply(frame_id, STATUS_OK, Atom('ok')) + else: + self.reply(frame_id, STATUS_ERROR, Atom('no_loop')) + elif self.loop_running and tag != 'shutdown': + # The main thread is inside run_forever: run the request as a + # loop callback so it does not wait for the loop to end. + self.loop.call_soon_threadsafe(self._serve, frame_id, term) + else: + self.inbox.put(('request', (frame_id, term))) + + def _on_submit(self, frame_id, term): + _, task_ref, module, func, args, kwargs = term + loop = self.loop + if loop is None or not self.loop_running: + self.reply(frame_id, STATUS_ERROR, Atom('no_loop')) + return + self.reply(frame_id, STATUS_OK, Atom('ok')) + + def schedule(): + try: + fn = _resolve(module, func) + result = fn(*_as_list(args), **_as_dict(kwargs)) + if inspect.isawaitable(result): + task = asyncio.ensure_future(result) + task.add_done_callback( + lambda t: self._report_task(task_ref, t)) + return + self.event((Atom('async_result'), task_ref, (Atom('ok'), result))) + except BaseException as exc: + self.event((Atom('async_result'), task_ref, + (Atom('error'), _exc_term(exc)))) + loop.call_soon_threadsafe(schedule) + + def _report_task(self, task_ref, task): + try: + if task.cancelled(): + result = (Atom('error'), Atom('cancelled')) + elif task.exception() is not None: + result = (Atom('error'), _exc_term(task.exception())) + else: + result = (Atom('ok'), task.result()) + self.event((Atom('async_result'), task_ref, result)) + except PipeBroken: + pass + + def _mark_broken(self, reason): + if self.broken: + return + self.broken = True + self.broken_reason = reason + with self._plock: + pending = list(self._pending.values()) + self._pending.clear() + for waiter in pending: + waiter.deliver((STATUS_ERROR, reason)) + self.inbox.put(('broken', reason)) + + # -- main thread ------------------------------------------------------- + + def serve_forever(self): + """Main-thread request loop. Returns when Erlang asks for shutdown.""" + while True: + try: + kind, item = self.inbox.get() + except _Interrupted: + continue # interrupt raced with the end of a request + if kind == 'request': + if self._serve(*item) == 'shutdown': + return + elif kind == 'broken': + return + # stale replies are dropped + + def _serve(self, frame_id, term): + """Execute one Erlang request and reply. Returns 'shutdown' when the + child should exit.""" + with self._cancel_lock: + if frame_id in self._cancelled: + self._cancelled.discard(frame_id) + return None # the caller timed out while this was queued + tag = term[0] if isinstance(term, tuple) else term + if tag == 'shutdown': + try: + self.reply(frame_id, STATUS_OK, Atom('ok')) + except PipeBroken: + pass + return 'shutdown' + if tag == 'start_loop': + return self._run_loop(frame_id) + if tag == 'pass_fd': + with self._fds_lock: + fd = self._received_fds.pop(0) if self._received_fds else None + if fd is None: + self.reply(frame_id, STATUS_ERROR, Atom('no_fd_received')) + else: + self.reply(frame_id, STATUS_OK, fd) + return None + + self._exec_stack.append(frame_id) + try: + status, value = self._execute(tag, term) + finally: + self._exec_stack.pop() + try: + self.reply(frame_id, status, value) + except PipeBroken: + pass + except KeyboardInterrupt: + # running is False here so the handler does not raise; a + # stray Ctrl-C style interrupt must still not lose the reply + try: + self.reply(frame_id, status, value) + except (PipeBroken, KeyboardInterrupt): + pass + return None + + def _execute(self, tag, term): + """Run call/eval/exec with interrupt handling. The handler raises only + while `running` is set, and a late signal is absorbed by the retry so + the reply is always sent.""" + prev = self.running + result = None + while result is None: + self.running = True + try: + result = STATUS_OK, self._dispatch(tag, term) + except KeyboardInterrupt: # includes _Interrupted + result = STATUS_ERROR, Atom('interrupted') + except PipeBroken as exc: + result = STATUS_ERROR, (Atom('pipe_broken'), str(exc)) + except StopIteration: + result = STATUS_ERROR, (Atom('StopIteration'), None) + except (SystemExit, GeneratorExit): + self.running = prev + raise + except BaseException as exc: + result = STATUS_ERROR, _exc_term(exc) + finally: + # Cleared first thing so a second signal landing in the + # bookkeeping above is dropped by the handler, not raised + self.running = False + self.running = prev + return result + + def _dispatch(self, tag, term): + if tag == 'init': + return self._init(term) + if tag == 'call': + _, module, func, args, kwargs = term + fn = _resolve(module, func, self.globals) + result = fn(*_as_list(args), **_as_dict(kwargs)) + elif tag == 'eval': + _, code, locals_ = term + loc = dict(self.globals) + loc.update(_as_dict(locals_)) + result = eval(compile(_as_text(code), '', 'eval'), self.globals, loc) + elif tag == 'exec': + _, code = term + exec(compile(_as_text(code), '', 'exec'), self.globals) + return Atom('ok') + else: + raise RuntimeError('unknown request %r' % (tag,)) + if inspect.isawaitable(result): + result = self.get_loop().run_until_complete(result) + return result + + def _init(self, term): + _, context_pid, paths, imports = term + self.context_pid = context_pid + for path in reversed(_as_list(paths)): + path = _as_text(path) + if path not in sys.path: + sys.path.insert(0, path) + import importlib + for name in _as_list(imports): + importlib.import_module(_as_text(name)) + return Atom('ok') + + def _run_loop(self, frame_id): + loop = self.get_loop() + self.loop_running = True + self.reply(frame_id, STATUS_OK, Atom('ok')) + self.running = True + result = Atom('ok') + self._exec_stack.append('loop') + try: + loop.run_forever() + except KeyboardInterrupt: + self.running = False + result = (Atom('error'), Atom('interrupted')) + except BaseException as exc: + self.running = False + result = (Atom('error'), _exc_term(exc)) + finally: + self._exec_stack.pop() + self.running = False + self.loop_running = False + # 10: submits scheduled between loop.stop() and here sit in the + # ready queue; run them into tasks, then cancel everything so + # each reports {error, cancelled} instead of vanishing + _drain_and_cancel(loop) + try: + self.event((Atom('loop_exit'), result)) + except PipeBroken: + pass + return None + + +# --------------------------------------------------------------------------- +# helpers +# --------------------------------------------------------------------------- + +def _as_list(v): + if v is None: + return [] + if isinstance(v, (list, tuple)): + return list(v) + return [v] + + +def _as_dict(v): + if v is None: + return {} + if isinstance(v, dict): + return {(k if isinstance(k, str) else str(k)): val for k, val in v.items()} + return {} + + +def _as_text(code): + if isinstance(code, bytes): + return code.decode('utf-8') + if isinstance(code, list): # Erlang charlist + return ''.join(chr(c) for c in code) + return code + + +def _resolve(module, func, globals_=None): + module = _as_text(module) + func = _as_text(func) + if module in ('__main__', '') and globals_ is not None and func in globals_: + return globals_[func] + import importlib + if module == '__main__' and globals_ is not None: + raise AttributeError("name '%s' is not defined in the context" % func) + mod = importlib.import_module(module) + try: + return getattr(mod, func) + except AttributeError: + raise AttributeError("module '%s' has no attribute '%s'" % (module, func)) from None + + +def _exc_term(exc): + if isinstance(exc, KeyboardInterrupt): + return Atom('interrupted') + try: + msg = str(exc) + except Exception: + msg = 'unknown' + return (Atom(type(exc).__name__), msg) + + +def _drain_and_cancel(loop): + for _ in range(3): + try: + loop.run_until_complete(asyncio.sleep(0)) + except BaseException: + break + _cancel_all_tasks(loop) + + +def _cancel_all_tasks(loop): + try: + tasks = [t for t in asyncio.all_tasks(loop) if not t.done()] + except RuntimeError: + return + for t in tasks: + t.cancel() + if tasks: + try: + loop.run_until_complete(asyncio.gather(*tasks, return_exceptions=True)) + except BaseException: + pass + + +# --------------------------------------------------------------------------- +# The `erlang` module seen by user code +# --------------------------------------------------------------------------- + +def install_erlang_module(runtime): + """Build the `erlang` module for this child and register it in + sys.modules. Mirrors the embedded module's public surface; anything that + cannot cross a process boundary raises a clear RuntimeError.""" + import types + mod = types.ModuleType('erlang', __doc__) + rt = runtime + + def _not_supported(name): + def fn(*args, **kwargs): + raise RuntimeError( + '%s is not available in isolated mode (it needs the embedded ' + 'interpreter); use erlang.call/erlang.send instead' % name) + fn.__name__ = name + return fn + + def call(name, *args, **kwargs): + if kwargs: + raise TypeError('erlang.call takes positional arguments only') + status, value = rt.request((Atom('call'), _as_text(name), list(args))) + if status == STATUS_OK: + return value + raise _callback_error(value) + + async def async_call(name, *args): + return await rt.request_async((Atom('call'), _as_text(name), list(args))) + + def send(pid, message): + if not isinstance(pid, Pid): + raise TypeError('erlang.send: pid must be an erlang.Pid, got %s' + % type(pid).__name__) + status, value = rt.request((Atom('send'), pid, message)) + if status != STATUS_OK: + raise _callback_error(value) + return None + + def whereis(name): + status, value = rt.request((Atom('whereis'), Atom(_as_text(name)))) + if status != STATUS_OK: + raise _callback_error(value) + return value + + def self_(): + return rt.context_pid + + def atom(name): + return Atom(name) + + def is_isolated(): + return True + + def run(main, *, debug=None): + loop = rt.get_loop() + if debug is not None: + loop.set_debug(debug) + return loop.run_until_complete(main) + + def new_event_loop(): + return asyncio.new_event_loop() + + def get_event_loop_policy(): + return asyncio.get_event_loop_policy() + + def install(*, silent=False): + return None + + def spawn_task(coro, *, name=None): + loop = rt.get_loop() + return loop.create_task(coro, name=name) + + def sleep(seconds): + try: + asyncio.get_running_loop() + except RuntimeError: + import time + time.sleep(seconds) + return None + return asyncio.sleep(seconds) + + def log(level, message): + rt.event((Atom('log'), Atom(_as_text(level)), str(message))) + + class Function: + __slots__ = ('name',) + + def __init__(self, name): + self.name = name + + def __call__(self, *args): + return call(self.name, *args) + + def __repr__(self): + return '' % self.name + + def __getattr__(name): + if name.startswith('_'): + raise AttributeError(name) + return Function(name) + + from . import _server as server + + ns = dict( + call=call, async_call=async_call, send=send, whereis=whereis, + self=self_, atom=atom, Atom=Atom, Pid=Pid, Ref=Ref, Port=Port, + ProcessError=ProcessError, SuspensionRequired=SuspensionRequired, + Function=Function, is_isolated=is_isolated, run=run, + new_event_loop=new_event_loop, get_event_loop_policy=get_event_loop_policy, + install=install, spawn_task=spawn_task, sleep=sleep, log=log, + server=server, __getattr__=__getattr__, + schedule=_not_supported('erlang.schedule'), + schedule_py=_not_supported('erlang.schedule_py'), + schedule_inline=_not_supported('erlang.schedule_inline'), + consume_time_slice=lambda *_a, **_k: False, + channel=_not_supported('erlang.channel'), + byte_channel=_not_supported('erlang.byte_channel'), + reactor=_not_supported('erlang.reactor'), + shared_dict=_not_supported('erlang.shared_dict'), + Channel=_not_supported('erlang.Channel'), + ByteChannel=_not_supported('erlang.ByteChannel'), + __all__=['call', 'async_call', 'send', 'whereis', 'self', 'atom', + 'Atom', 'Pid', 'Ref', 'ProcessError', 'SuspensionRequired', + 'run', 'sleep', 'spawn_task', 'server', 'is_isolated'], + ) + mod.__dict__.update(ns) + sys.modules['erlang'] = mod + return mod + + +def format_exception(exc): + return ''.join(traceback.format_exception(type(exc), exc, exc.__traceback__)) diff --git a/priv/py_isolated_child.py b/priv/py_isolated_child.py new file mode 100644 index 0000000..006e074 --- /dev/null +++ b/priv/py_isolated_child.py @@ -0,0 +1,222 @@ +# Copyright 2026 Benoit Chesneau +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Entry point of an `isolated` context child process. + +Started by src/py_isolated.erl as + + python3 py_isolated_child.py SOCKET_PATH [--rlimit-as BYTES] + [--rlimit-cpu SECONDS] [--rlimit-nofile N] [--cgroup DIR] + +Order of operations matters: limits are applied before anything else is +imported, the parent-death signal is armed before the socket connects, and +the ready/startup_error event is the first frame Erlang sees. +""" + +import os +import sys + + +def _die(reason): + sys.stderr.write('py_isolated_child: %s\n' % reason) + sys.stderr.flush() + os._exit(3) + + +def _parse_args(argv): + if len(argv) < 2: + _die('usage: py_isolated_child.py SOCKET_PATH [options]') + opts = {'socket': argv[1], 'rlimits': {}, 'cgroup': None} + i = 2 + while i < len(argv): + flag = argv[i] + if flag in ('--rlimit-as', '--rlimit-cpu', '--rlimit-nofile'): + opts['rlimits'][flag[len('--rlimit-'):]] = int(argv[i + 1]) + i += 2 + elif flag == '--cgroup': + opts['cgroup'] = argv[i + 1] + i += 2 + else: + _die('unknown option %s' % flag) + return opts + + +def _arm_parent_death(): + """Get SIGKILL when the parent (the BEAM) dies: prctl on Linux, procctl + on FreeBSD. Elsewhere the reader thread's EOF handling covers it.""" + SIGKILL = 9 + try: + import ctypes + libc = ctypes.CDLL(None, use_errno=True) + if sys.platform.startswith('linux'): + PR_SET_PDEATHSIG = 1 + libc.prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0) + elif sys.platform.startswith('freebsd'): + # procctl(P_PID, 0, PROC_PDEATHSIG_CTL, &sig) (FreeBSD 11.2+) + P_PID = 0 + PROC_PDEATHSIG_CTL = 11 + sig = ctypes.c_int(SIGKILL) + libc.procctl.argtypes = [ctypes.c_int, ctypes.c_int64, ctypes.c_int, ctypes.c_void_p] + libc.procctl(P_PID, 0, PROC_PDEATHSIG_CTL, ctypes.byref(sig)) + else: + return + # The parent may already be gone between fork and the call + if os.getppid() == 1: + os._exit(0) + except Exception: + pass + + +# macOS does not enforce RLIMIT_AS: `as` is enforced by a watchdog thread +# there (see _start_memory_watchdog) instead of setrlimit. +_AS_VIA_WATCHDOG = sys.platform == 'darwin' + + +def _apply_rlimits(limits): + if not limits: + return [] + import resource + if _AS_VIA_WATCHDOG: + limits = {k: v for k, v in limits.items() if k != 'as'} + names = { + 'as': getattr(resource, 'RLIMIT_AS', None), + 'cpu': getattr(resource, 'RLIMIT_CPU', None), + 'nofile': getattr(resource, 'RLIMIT_NOFILE', None), + } + errors = [] + for key, value in limits.items(): + res = names.get(key) + if res is None: + errors.append((key, 'not supported on this platform')) + continue + try: + _soft, hard = resource.getrlimit(res) + if hard != resource.RLIM_INFINITY and hard < value: + value = hard # cannot raise a hard limit, clamp to it + resource.setrlimit(res, (value, hard)) + except (ValueError, OSError) as exc: + errors.append((key, str(exc))) + return errors + + +def _start_memory_watchdog(limit, runtime): + """Portable memory bound: poll this process's resident set every 50 ms + and exit when it passes `limit`. Erlang reports the in-flight call as + {error, {child_exited, {memory_limit, Bytes}}}.""" + import resource + import threading + from _erlang_impl._etf import Atom + + def rss(): + r = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return r if sys.platform == 'darwin' else r * 1024 + + def watch(): + while True: + used = rss() + if used > limit: + try: + runtime.event((Atom('memory_limit'), used)) + except Exception: + pass + os._exit(3) + threading.Event().wait(0.05) + + t = threading.Thread(target=watch, name='erlang-memory-watchdog', daemon=True) + t.start() + + +def _join_cgroup(path): + """cgroup v2, best effort: the directory is created by the operator (or + by Erlang) with the limits already written; we only join it.""" + if not path: + return None + if not sys.platform.startswith('linux'): + return 'cgroups are Linux only (platform %s); use rlimits' % sys.platform + try: + with open(os.path.join(path, 'cgroup.procs'), 'w') as f: + f.write(str(os.getpid())) + return None + except OSError as exc: + return str(exc) + + +def _connect(path): + import socket + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(path) + # Default Unix socket buffers are small (8 KB on macOS); the kernel + # clamps to its maximum, so this is best effort. + for opt in (socket.SO_SNDBUF, socket.SO_RCVBUF): + try: + sock.setsockopt(socket.SOL_SOCKET, opt, 1024 * 1024) + except OSError: + pass + return sock + + +def main(argv): + opts = _parse_args(argv) + _arm_parent_death() + rlimit_errors = _apply_rlimits(opts['rlimits']) + cgroup_error = _join_cgroup(opts['cgroup']) + + priv = os.path.dirname(os.path.abspath(__file__)) + if priv not in sys.path: + sys.path.insert(0, priv) + + try: + sock = _connect(opts['socket']) + except OSError as exc: + _die('cannot connect to %s: %s' % (opts['socket'], exc)) + + from _erlang_impl import _isolated + from _erlang_impl._etf import Atom + + runtime = _isolated.Runtime(sock) + _isolated.install_erlang_module(runtime) + runtime.start() + if _AS_VIA_WATCHDOG and 'as' in opts['rlimits']: + _start_memory_watchdog(opts['rlimits']['as'], runtime) + + if rlimit_errors or cgroup_error: + problems = [(Atom('rlimit'), Atom(k), msg) for k, msg in rlimit_errors] + if cgroup_error: + problems.append((Atom('cgroup'), cgroup_error)) + try: + runtime.event((Atom('startup_error'), problems)) + finally: + os._exit(2) + + info = { + Atom('os_pid'): os.getpid(), + Atom('python_version'): '%d.%d.%d' % sys.version_info[:3], + Atom('executable'): sys.executable, + Atom('platform'): sys.platform, + } + runtime.event((Atom('ready'), info)) + + try: + runtime.serve_forever() + finally: + try: + sock.close() + except OSError: + pass + os._exit(0) + + +if __name__ == '__main__': + main(sys.argv) diff --git a/rebar.config b/rebar.config index c36b4fc..07d555b 100644 --- a/rebar.config +++ b/rebar.config @@ -63,6 +63,7 @@ <<"docs/threading.md">>, <<"docs/asyncio.md">>, <<"docs/workers.md">>, + <<"docs/isolated.md">>, <<"docs/reactor.md">>, <<"docs/process-bound-envs.md">>, <<"docs/security.md">>, @@ -95,6 +96,7 @@ <<"docs/threading.md">>, <<"docs/asyncio.md">>, <<"docs/workers.md">>, + <<"docs/isolated.md">>, <<"docs/reactor.md">>, <<"docs/process-bound-envs.md">>, <<"docs/security.md">>, diff --git a/src/erlang_python.app.src b/src/erlang_python.app.src index 251cf8f..70a95fa 100644 --- a/src/erlang_python.app.src +++ b/src/erlang_python.app.src @@ -1,6 +1,6 @@ {application, erlang_python, [ {description, "Execute Python applications from Erlang using dirty NIFs"}, - {vsn, "4.1.0"}, + {vsn, "4.2.0"}, {registered, []}, {mod, {erlang_python_app, []}}, {applications, [ diff --git a/src/py.erl b/src/py.erl index 0d16c9a..8b9218a 100644 --- a/src/py.erl +++ b/src/py.erl @@ -112,6 +112,8 @@ context/0, context/1, interrupt/1, + kill/1, + python_executable/0, start_contexts/0, start_contexts/1, stop_contexts/0, @@ -928,6 +930,19 @@ create_venv(Path, Opts) -> %% @private Get the Python executable path %% When embedded, sys.executable returns the embedding app (beam.smp) %% so we reconstruct the path from sys.prefix and version info +%% @doc Path of the Python interpreter matching the embedded runtime. +%% +%% Reconstructed from `sys.prefix' (when embedded, `sys.executable' is the +%% VM). Used as the default interpreter of isolated contexts and for venvs. +-spec python_executable() -> string(). +python_executable() -> + get_python_executable(). + +%% @doc Kill the child of an isolated context. See py_context:kill/1. +-spec kill(pid()) -> ok | {error, not_isolated}. +kill(Ctx) when is_pid(Ctx) -> + py_context:kill(Ctx). + -spec get_python_executable() -> string(). get_python_executable() -> %% Use a single expression to find the Python executable diff --git a/src/py_context.erl b/src/py_context.erl index fa1a72f..d857b87 100644 --- a/src/py_context.erl +++ b/src/py_context.erl @@ -69,6 +69,8 @@ ]). %% Internal exports +-export([kill/1, pass_fd/2, child_info/1]). + -export([init/3, init/4, init_ref_tab/0]). %% Exported for py_reactor_context @@ -83,7 +85,7 @@ %% reply is drained instead of being left in the caller's mailbox. -define(INTERRUPT_GRACE_MS, 1000). --type context_mode() :: worker | owngil. +-type context_mode() :: worker | owngil | isolated. -type context() :: pid(). -export_type([context_mode/0, context/0]). @@ -115,9 +117,12 @@ %% The process creates a Python context based on the mode: %% - `worker' - Create a thread-state worker (main interpreter namespace) %% - `owngil' - Create a sub-interpreter with its own GIL (Python 3.14+) +%% - `isolated' - Run CPython in a child OS process (see py_isolated) %% %% The `owngil' mode creates a dedicated pthread for each context, allowing -%% true parallel Python execution. Requires Python 3.14+. +%% true parallel Python execution. Requires Python 3.14+. The `isolated' +%% mode gives failure isolation: the child can be killed, capped with +%% rlimits, and a crash in a C extension only takes the child down. %% %% @param Id Unique identifier for this context %% @param Mode Context mode @@ -138,17 +143,25 @@ start_link(Id, Mode) -> {ok, pid()} | {error, term()}. start_link(Id, Mode, Opts) when is_map(Opts) -> Parent = self(), - Pid = spawn_link(fun() -> init(Parent, Id, Mode, Opts) end), + Pid = proc_lib:spawn_link(fun() -> init(Parent, Id, Mode, Opts) end), receive {Pid, started} -> {ok, Pid}; {Pid, {error, Reason}} -> {error, Reason} - after 5000 -> + after start_timeout(Mode, Opts) -> exit(Pid, kill), + _ = ets:member(?REF_TAB, Pid) andalso ets:delete(?REF_TAB, Pid), {error, timeout} end. +%% @private An isolated child has to spawn and connect; give it its +%% start_timeout plus a margin. Embedded contexts start in well under 5 s. +start_timeout(isolated, Opts) -> + maps:get(start_timeout, Opts, 10000) + 2000; +start_timeout(_Mode, _Opts) -> + 5000. + %% @doc Stop a py_context process. -spec stop(context()) -> ok. stop(Ctx) when is_pid(Ctx) -> @@ -169,7 +182,7 @@ stop(Ctx) when is_pid(Ctx) -> %% @doc Create a new context with options map. %% %% Options: -%% - `mode' - Context mode (worker | owngil), default: worker +%% - `mode' - Context mode (worker | owngil | isolated), default: worker %% - `memory_limit' - Cap in bytes on memory allocated by this context. %% Requires `mode => owngil' and the runtime started with %% `enable_memory_limits'; see py_nif:context_set_memory_limit/2 for what @@ -417,6 +430,15 @@ get_nif_ref(Ctx) when is_pid(Ctx) -> -spec interrupt(context()) -> ok | not_running. interrupt(Ctx) when is_pid(Ctx) -> case lookup_nif_ref(Ctx) of + {ok, isolated} -> + %% The context process is never blocked in a NIF: ask it. It + %% signals the child and arms the SIGKILL backstop. + MRef = erlang:monitor(process, Ctx), + Ctx ! {interrupt, self(), MRef}, + case await_ctrl_reply(Ctx, MRef, 5000) of + ok -> ok; + _ -> not_running + end; {ok, Ref} -> try py_nif:context_interrupt(Ref) of ok -> ok; @@ -428,6 +450,73 @@ interrupt(Ctx) when is_pid(Ctx) -> not_running end. +%% @doc Kill the child process of an isolated context with SIGKILL. +%% +%% Total and immediate, whatever the child is doing (a C call, a numpy +%% kernel, a blocked read). In-flight calls return `{error, killed}'. With +%% `restart => true' (the default) a fresh child is started and the context +%% stays usable, with its Python state gone. Embedded contexts (`worker', +%% `owngil') cannot be killed: they return `{error, not_isolated}'. +%% +%% @param Ctx Context process +%% @returns ok | {error, not_isolated} +-spec kill(context()) -> ok | {error, not_isolated}. +kill(Ctx) when is_pid(Ctx) -> + case lookup_nif_ref(Ctx) of + {ok, isolated} -> + MRef = erlang:monitor(process, Ctx), + Ctx ! {kill, self(), MRef}, + await_ctrl_reply(Ctx, MRef, 5000); + _ -> + {error, not_isolated} + end. + +%% @doc Hand a file descriptor to the child of an isolated context. +%% +%% The fd is sent over the control socket (`SCM_RIGHTS') and the number it +%% got in the child is returned; use it with `erlang.server.serve' from a +%% submitted coroutine. Get the fd with `py:dup_fd/1' on a listening socket. +%% +%% @param Ctx Context process +%% @param Fd File descriptor in this VM +%% @returns {ok, ChildFd} | {error, Reason} +-spec pass_fd(context(), non_neg_integer()) -> {ok, non_neg_integer()} | {error, term()}. +pass_fd(Ctx, Fd) when is_pid(Ctx), is_integer(Fd) -> + case lookup_nif_ref(Ctx) of + {ok, isolated} -> + MRef = erlang:monitor(process, Ctx), + Ctx ! {pass_fd, self(), MRef, Fd}, + await_ctrl_reply(Ctx, MRef, 5000); + _ -> + {error, not_isolated} + end. + +%% @doc Information about the child of an isolated context: `os_pid', +%% `python_version', `executable', `platform'. +-spec child_info(context()) -> {ok, map()} | {error, term()}. +child_info(Ctx) when is_pid(Ctx) -> + case lookup_nif_ref(Ctx) of + {ok, isolated} -> + MRef = erlang:monitor(process, Ctx), + Ctx ! {child_info, self(), MRef}, + await_ctrl_reply(Ctx, MRef, 5000); + _ -> + {error, not_isolated} + end. + +%% @private Interrupt on behalf of a timed-out request. An isolated context +%% cancels that request only (interrupting the child if it is the one +%% executing, dropping it from the queue otherwise); embedded contexts can +%% only interrupt whatever runs. +interrupt_request(Ctx, ReqMRef) -> + case lookup_nif_ref(Ctx) of + {ok, isolated} -> + Ctx ! {interrupt_request, ReqMRef}, + ok; + _ -> + interrupt(Ctx) + end. + %% @private Create the pid -> NIF reference table. Called by the supervisor %% before any context starts. -spec init_ref_tab() -> ok. @@ -513,6 +602,18 @@ submit(Ctx, Module, Func, Args) -> -spec submit(context(), atom() | binary(), atom() | binary(), list(), map()) -> {ok, reference()} | {error, term()}. submit(Ctx, Module, Func, Args, Kwargs) when is_pid(Ctx), is_list(Args), is_map(Kwargs) -> + case lookup_nif_ref(Ctx) of + {ok, isolated} -> + TaskRef = make_ref(), + MRef = erlang:monitor(process, Ctx), + Ctx ! {submit, self(), MRef, TaskRef, to_binary(Module), to_binary(Func), Args, Kwargs}, + await_ctrl_reply(Ctx, MRef, 5000); + _ -> + submit_embedded(Ctx, Module, Func, Args, Kwargs) + end. + +%% @private +submit_embedded(Ctx, Module, Func, Args, Kwargs) -> case loop_ref(Ctx) of {ok, LoopRef} -> TaskRef = make_ref(), @@ -563,7 +664,7 @@ await_reply(Ctx, MRef, Timeout) -> {'DOWN', MRef, process, Ctx, Reason} -> {error, {context_died, Reason}} after Timeout -> - _ = interrupt(Ctx), + _ = interrupt_request(Ctx, MRef), receive {MRef, _Late} -> erlang:demonitor(MRef, [flush]); @@ -586,6 +687,9 @@ await_ctrl_reply(Ctx, MRef, Timeout) -> {'DOWN', MRef, process, Ctx, Reason} -> {error, {context_died, Reason}} after Timeout -> + %% An isolated context drops the pending entry so a late reply is + %% not delivered to a caller that stopped waiting + Ctx ! {cancel_ctrl, MRef}, erlang:demonitor(MRef, [flush]), {error, timeout} end. @@ -622,6 +726,8 @@ init(Parent, Id, Mode) -> init(Parent, Id, Mode, #{}). %% @private +init(Parent, Id, isolated, Opts) -> + py_isolated:init(Parent, Id, isolated, Opts); init(Parent, Id, Mode, Opts) -> process_flag(trap_exit, true), case create_context(Mode) of diff --git a/src/py_isolated.erl b/src/py_isolated.erl new file mode 100644 index 0000000..4de2b7a --- /dev/null +++ b/src/py_isolated.erl @@ -0,0 +1,1193 @@ +%% Copyright 2026 Benoit Chesneau +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%% @doc Context process for `isolated' mode: CPython in a child OS process. +%%% +%%% A py_context started with `mode => isolated' runs this state machine +%%% instead of the embedded loop. The child is spawned through a port (so +%%% the VM reaps it and reports its exit status) and talks over a Unix +%%% socket using the frame format of the blocking callback pipe: +%%% +%%% ``` +%%% <> +%%% Body = <> +%%% ''' +%%% +%%% Status 0 request to the child, 1/2 error/ok reply (either direction), +%%% 3 request from the child (`erlang.call', `erlang.send', `erlang.whereis'), +%%% 4 event from the child, 5 control to the child (`interrupt', `cancel', +%%% `stop_loop'). +%%% +%%% == States == +%%% +%%%
    +%%%
  • `idle' - child up, nothing on its main thread.
    • +%%%
  • `{busy, Id}' - top-level request `Id' executing. Requests from +%%% other callers are postponed (served in order once the child is +%%% free); requests from a process running a callback for this +%%% context are nested and dispatched at once.
    • +%%%
  • `looping' - `start_loop' accepted, `run_forever' on the main +%%% thread; `call/eval/exec' answer `{error, loop_running}'.
    • +%%%
  • `stopping_loop' - `stop_loop' sent, waiting for the loop to exit; +%%% interrupt after the grace period, SIGKILL after `kill_after'.
    • +%%%
  • `{restarting, Reason}' - the child is gone or being killed; +%%% requests are postponed until the new child is up. In-flight +%%% requests fail with `{error, Reason}'.
    • +%%%
+%%% +%%% The message protocol with py_context is unchanged: requests are plain +%%% messages `{call, From, MRef, ...}' answered with `From ! {MRef, Reply}'. +%%% Use `sys:get_state/1' to see the state and `sys:trace/2' for events. +%%% +%%% @private +-module(py_isolated). + +-behaviour(gen_statem). + +-export([init/4, python_executable/1]). + +%% gen_statem callbacks +-export([callback_mode/0, init/1, handle_event/4, terminate/3, code_change/4, + format_status/1]). + +-define(REF_TAB, py_context_refs). +-define(STATUS_REQUEST, 0). +-define(STATUS_ERROR, 1). +-define(STATUS_OK, 2). +-define(STATUS_CALLBACK, 3). +-define(STATUS_EVENT, 4). +-define(STATUS_CONTROL, 5). + +-define(DEFAULT_KILL_AFTER_MS, 1000). +-define(DEFAULT_START_TIMEOUT_MS, 10000). +-define(DEFAULT_MAX_RESTARTS, 5). +-define(DEFAULT_RESTART_PERIOD_MS, 10000). +-define(SHUTDOWN_GRACE_MS, 1000). +-define(EXIT_STATUS_WAIT_MS, 5000). +-define(SOCKET_BUF, 1024 * 1024). + +-record(child, { + port :: port(), + os_pid :: pos_integer(), + listener :: socket:socket() | undefined, + sock :: socket:socket(), + sock_path :: string(), + buf = <<>> :: binary(), + info = #{} :: map() +}). + +%% A running worker loop (state `looping' / `stopping_loop') +-record(loop, { + owner :: pid() | undefined, + owner_mon :: reference() | undefined, + stop_waiters = [] :: [{pid(), reference()}] +}). + +-record(data, { + id :: term(), + parent :: pid() | undefined, + opts :: map(), + child :: #child{} | undefined, + next_id = 1 :: pos_integer(), + %% Id => {From, MRef, Kind} for requests the child has + pending = #{} :: map(), + %% TaskRef => SubmitterPid for submit/5 results + tasks = #{} :: map(), + %% MonitorRef => FrameId of callbacks running in their own process, + %% and Pid => MonitorRef of those processes (their requests are nested) + callbacks = #{} :: map(), + cb_pids = #{} :: map(), + restarts = [] :: [integer()], + loop :: #loop{} | undefined, + %% Request id (or `loop') the armed kill backstop is bound to + kill_target :: pos_integer() | loop | undefined, + %% Callers of kill/1 answered once the new child is up + kill_waiters = [] :: [{pid(), reference()}] +}). + +-define(IS_MAIN(K), (K =:= call orelse K =:= eval orelse K =:= exec orelse + (is_tuple(K) andalso element(1, K) =:= start_loop))). + +-type state() :: idle | {busy, pos_integer()} | looping | stopping_loop + | {restarting, term()}. +-export_type([state/0]). + +%% ============================================================================ +%% Entry point (called from py_context:init/4 in the context process) +%% ============================================================================ + +%% @private Runs in the process py_context:start_link/3 spawned with +%% proc_lib. The child start is bounded by `start_timeout'; the parent +%% is answered before the state machine takes over. +init(Parent, Id, _Mode, Opts) -> + process_flag(trap_exit, true), + %% gen_statem stops when its OTP parent (head of '$ancestors') exits, + %% for any reason. A context outlives the process that created it, as + %% the embedded contexts do; only a crash of that process (non-normal + %% EXIT, handled below) or a shutdown stops it. Point the parent slot at + %% ourselves so gen_statem leaves the creator's exit to us. + put('$ancestors', [self() | case get('$ancestors') of + L when is_list(L) -> L; + _ -> [] + end]), + ets:insert(?REF_TAB, {self(), isolated}), + Data0 = #data{id = Id, parent = Parent, opts = Opts}, + case start_child(Data0) of + {ok, Data} -> + Parent ! {self(), started}, + %% Arm the socket select before handling events + {State, Data1, Actions} = drain_socket(idle, Data, []), + gen_statem:enter_loop(?MODULE, [], State, Data1, Actions); + {error, Reason} -> + ets:delete(?REF_TAB, self()), + Parent ! {self(), {error, Reason}} + end. + +%% ============================================================================ +%% gen_statem callbacks +%% ============================================================================ + +callback_mode() -> + [handle_event_function, state_enter]. + +%% @private Not used: the process is started through init/4 and +%% gen_statem:enter_loop/5. +init(_Args) -> + {stop, use_init_4}. + +%% ---- state enter ----------------------------------------------------------- + +handle_event(enter, _Old, idle, #data{kill_waiters = Waiters} = Data) -> + [W ! {M, ok} || {W, M} <- Waiters], + {keep_state, Data#data{kill_waiters = []}}; +handle_event(enter, _Old, {restarting, _}, _Data) -> + %% SIGKILL was sent (or the child is exiting): the port reports it + %% within milliseconds; this is the safety net + {keep_state_and_data, [{state_timeout, ?EXIT_STATUS_WAIT_MS, exit_status}]}; +handle_event(enter, _Old, stopping_loop, _Data) -> + keep_state_and_data; +handle_event(enter, _Old, _State, _Data) -> + keep_state_and_data; + +%% ---- socket and port ------------------------------------------------------- + +handle_event(info, {'$socket', S, select, _}, State, #data{child = #child{sock = S}} = Data) -> + result(drain_socket(State, Data, [])); +handle_event(info, {'$socket', S, abort, {_, Reason}}, State, #data{child = #child{sock = S}} = Data) -> + result(socket_broken(Reason, State, Data, [])); +handle_event(info, {'$socket', _, _, _}, _State, _Data) -> + keep_state_and_data; +handle_event(info, {Port, {data, Out}}, _State, #data{child = #child{port = Port}} = Data) -> + log_output(Data, Out), + keep_state_and_data; +handle_event(info, {Port, {exit_status, Status}}, State, #data{child = #child{port = Port}} = Data) -> + child_exited(exit_reason(Status), State, Data); +handle_event(info, {Port, _}, _State, _Data) when is_port(Port) -> + keep_state_and_data; +handle_event(state_timeout, exit_status, {restarting, _} = State, #data{child = Child} = Data) -> + logger:error("py_context ~p (isolated): child ~p did not exit after SIGKILL", + [Data#data.id, Child#child.os_pid]), + child_exited({signal, 9}, State, Data); + +%% ---- requests -------------------------------------------------------------- + +handle_event(info, {call, From, MRef, Module, Func, Args, Kwargs}, State, Data) -> + request(From, MRef, call, {call, Module, Func, Args, Kwargs}, State, Data); +handle_event(info, {call, From, MRef, Module, Func, Args, Kwargs, _EnvRef}, State, Data) -> + request(From, MRef, call, {call, Module, Func, Args, Kwargs}, State, Data); +handle_event(info, {eval, From, MRef, Code, Locals}, State, Data) -> + request(From, MRef, eval, {eval, iolist_to_binary(Code), Locals}, State, Data); +handle_event(info, {eval, From, MRef, Code, Locals, _EnvRef}, State, Data) -> + request(From, MRef, eval, {eval, iolist_to_binary(Code), Locals}, State, Data); +handle_event(info, {exec, From, MRef, Code}, State, Data) -> + request(From, MRef, exec, {exec, iolist_to_binary(Code)}, State, Data); +handle_event(info, {exec, From, MRef, Code, _EnvRef}, State, Data) -> + request(From, MRef, exec, {exec, iolist_to_binary(Code)}, State, Data); +handle_event(info, {start_loop, From, MRef, Owner}, State, Data) -> + request(From, MRef, {start_loop, Owner}, start_loop, State, Data); +handle_event(info, {submit, From, MRef, TaskRef, Module, Func, Args, Kwargs}, State, Data) -> + request(From, MRef, {submit, TaskRef}, {submit, TaskRef, Module, Func, Args, Kwargs}, + State, Data); +handle_event(info, {pass_fd, From, MRef, Fd}, State, Data) -> + pass_fd(From, MRef, Fd, State, Data); +handle_event(info, {call_method, From, MRef, _ObjRef, _Method, _Args}, _State, _Data) -> + From ! {MRef, {error, not_supported_in_isolated}}, + keep_state_and_data; + +%% ---- introspection --------------------------------------------------------- + +handle_event(info, {get_interp_id, From, MRef}, _State, #data{child = Child}) -> + Id = case Child of + #child{os_pid = P} -> P; + _ -> 0 + end, + From ! {MRef, {ok, Id}}, + keep_state_and_data; +handle_event(info, {is_subinterp, From, MRef}, _State, _Data) -> + From ! {MRef, true}, + keep_state_and_data; +handle_event(info, {create_local_env, From, MRef}, _State, _Data) -> + %% Process-local environments are a NIF feature; the child has one + %% namespace per context. A fresh ref keeps py:call(Ctx, ...) working. + From ! {MRef, {ok, make_ref()}}, + keep_state_and_data; +handle_event(info, {get_nif_ref, From, MRef}, _State, _Data) -> + From ! {MRef, {error, not_supported_in_isolated}}, + keep_state_and_data; +handle_event(info, {loop_ref, From, MRef}, _State, _Data) -> + From ! {MRef, {error, not_supported_in_isolated}}, + keep_state_and_data; +handle_event(info, {child_info, From, MRef}, _State, #data{child = Child}) -> + Info = case Child of + #child{os_pid = P, info = I} -> I#{os_pid => P}; + undefined -> #{} + end, + From ! {MRef, {ok, Info}}, + keep_state_and_data; +handle_event(info, {cancel_ctrl, MRef}, _State, #data{pending = Pending} = Data) -> + %% The caller of a control request stopped waiting: drop the entry so + %% the late reply is not delivered + Drop = [Id || {Id, {_, M, _}} <- maps:to_list(Pending), M =:= MRef], + {keep_state, Data#data{pending = maps:without(Drop, Pending)}}; + +%% ---- interrupt, kill, stop ------------------------------------------------- + +handle_event(info, {interrupt, From, MRef}, State, Data) -> + case executing_request(State, Data) of + undefined -> + From ! {MRef, not_running}, + keep_state_and_data; + Target -> + From ! {MRef, ok}, + result(send_interrupt(Target, State, Data, [])) + end; +handle_event(info, {interrupt_request, ReqMRef}, State, #data{pending = Pending} = Data) -> + %% A timed-out request: if the child has it, interrupt that request + %% only. If it is still postponed here nothing happens: py_context has + %% already stopped waiting, and the reply goes nowhere. + case [Id || {Id, {_, M, _}} <- maps:to_list(Pending), M =:= ReqMRef] of + [Id] -> result(send_interrupt(Id, State, Data, [])); + [] -> keep_state_and_data + end; +handle_event({timeout, kill}, Target, State, #data{pending = Pending} = Data) -> + Still = case Target of + loop -> State =:= looping orelse State =:= stopping_loop; + Id -> maps:is_key(Id, Pending) + end, + case Still andalso Data#data.child =/= undefined of + true -> + logger:warning("py_context ~p (isolated): interrupt not honoured, killing child", + [Data#data.id]), + kill(killed, State, Data#data{kill_target = undefined}); + false -> + {keep_state, Data#data{kill_target = undefined}} + end; +handle_event(info, {kill, From, MRef}, State, Data) -> + case State of + {restarting, _} -> + %% Already on its way: answer with the others when it is back + {keep_state, Data#data{kill_waiters = [{From, MRef} | Data#data.kill_waiters]}}; + _ -> + kill(killed, State, Data#data{kill_waiters = [{From, MRef} | Data#data.kill_waiters]}) + end; +handle_event(info, {stop, From, MRef}, _State, Data) -> + Data1 = stop_child(Data, graceful), + From ! {MRef, ok}, + {stop, normal, Data1}; + +%% ---- worker loop ----------------------------------------------------------- + +handle_event(info, {stop_loop, From, MRef, GraceMs}, looping, #data{loop = Loop} = Data) -> + _ = send_frame(Data#data.child, 0, ?STATUS_CONTROL, stop_loop), + Loop1 = Loop#loop{stop_waiters = [{From, MRef} | Loop#loop.stop_waiters]}, + {next_state, stopping_loop, Data#data{loop = Loop1}, + [{state_timeout, GraceMs, interrupt}]}; +handle_event(info, {stop_loop, From, MRef, _GraceMs}, stopping_loop, #data{loop = Loop} = Data) -> + Loop1 = Loop#loop{stop_waiters = [{From, MRef} | Loop#loop.stop_waiters]}, + {keep_state, Data#data{loop = Loop1}}; +handle_event(info, {stop_loop, From, MRef, _GraceMs}, _State, _Data) -> + From ! {MRef, {error, no_loop}}, + keep_state_and_data; +handle_event(state_timeout, interrupt, stopping_loop, Data) -> + %% Cooperative stop did not land: interrupt, then the kill backstop + result(send_interrupt(loop, stopping_loop, Data, [])); +handle_event(info, {'DOWN', Mon, process, _Owner, _Reason}, looping, + #data{loop = #loop{owner_mon = Mon} = Loop} = Data) -> + %% Owner is gone: nobody will hear the exit, stop the loop + _ = send_frame(Data#data.child, 0, ?STATUS_CONTROL, stop_loop), + Loop1 = Loop#loop{owner = undefined, owner_mon = undefined}, + {next_state, stopping_loop, Data#data{loop = Loop1}, + [{state_timeout, 5000, interrupt}]}; + +%% ---- callbacks (child -> Erlang) ------------------------------------------- + +handle_event(info, {callback_reply, FrameId, Status, Term}, State, + #data{child = Child} = Data) when Child =/= undefined -> + case State of + {restarting, _} -> + keep_state_and_data; + _ -> + case send_frame(Child, FrameId, Status, Term) of + ok -> keep_state_and_data; + {error, Reason} -> result(socket_broken(Reason, State, Data, [])) + end + end; +handle_event(info, {callback_reply, _, _, _}, _State, _Data) -> + keep_state_and_data; +handle_event(info, {'DOWN', Mon, process, Pid, Reason}, State, #data{callbacks = Cbs} = Data) -> + case maps:take(Mon, Cbs) of + {Id, Rest} -> + Data1 = Data#data{callbacks = Rest, cb_pids = maps:remove(Pid, Data#data.cb_pids)}, + case Reason of + normal -> + {keep_state, Data1}; + _ -> + %% The callback process died without replying: the + %% child must not wait for it + Msg = iolist_to_binary(io_lib:format("callback crashed: ~p", [Reason])), + handle_event(info, {callback_reply, Id, ?STATUS_ERROR, Msg}, State, Data1) + end; + error -> + keep_state_and_data + end; + +%% ---- exits ----------------------------------------------------------------- + +handle_event(info, {'EXIT', Parent, Reason}, _State, #data{parent = Parent}) when Reason =/= normal -> + %% Whoever started us is gone: do not leave a child without an owner + {stop, Reason}; +handle_event(info, {'EXIT', _Pid, Reason}, _State, _Data) + when Reason =:= shutdown; Reason =:= kill -> + {stop, Reason}; +handle_event(info, {'EXIT', _Pid, {shutdown, _} = Reason}, _State, _Data) -> + {stop, Reason}; +handle_event(info, {'EXIT', _Pid, _Reason}, _State, _Data) -> + keep_state_and_data; +handle_event(info, _Other, _State, _Data) -> + keep_state_and_data. + +terminate(Reason, _State, #data{child = Child} = Data) -> + _ = case Child of + undefined -> Data; + _ when Reason =:= normal; Reason =:= shutdown -> stop_child(Data, graceful); + _ -> stop_child(Data, kill) + end, + ets:delete(?REF_TAB, self()), + ok. + +code_change(_OldVsn, State, Data, _Extra) -> + {ok, State, Data}. + +%% @private Keep sys:get_status readable: the socket buffer is noise +format_status(#{data := #data{child = #child{} = Child} = Data} = Status) -> + Status#{data => Data#data{child = Child#child{buf = <<>>}}}; +format_status(Status) -> + Status. + +%% @doc Python executable used for isolated children: the `python' option, +%% then the `isolated_python' application env, then the interpreter matching +%% the embedded runtime, then `python3' from PATH. +-spec python_executable(map()) -> string() | {error, term()}. +python_executable(Opts) -> + Candidate = case maps:get(python, Opts, undefined) of + undefined -> + case application:get_env(erlang_python, isolated_python) of + {ok, P} -> P; + undefined -> default_python() + end; + P -> P + end, + resolve_exe(to_list(Candidate)). + +default_python() -> + case persistent_term:get({?MODULE, python}, undefined) of + undefined -> + Exe = try py:python_executable() catch _:_ -> "python3" end, + persistent_term:put({?MODULE, python}, Exe), + Exe; + Exe -> + Exe + end. + +resolve_exe(Exe) -> + case filename:pathtype(Exe) of + absolute -> + case filelib:is_file(Exe) of + true -> Exe; + false -> {error, {python_not_found, Exe}} + end; + _ -> + case os:find_executable(Exe) of + false -> {error, {python_not_found, Exe}}; + Found -> Found + end + end. + +%% ============================================================================ +%% Child startup +%% ============================================================================ + +start_child(#data{opts = Opts} = St) -> + case check_platform_opts(Opts) of + ok -> start_child_1(St); + {error, _} = Err -> Err + end. + +%% cgroups exist only on Linux; rlimits are POSIX and apply everywhere. +%% RLIMIT_AS is enforced by the kernel on Linux and FreeBSD; on macOS the +%% child enforces `as' with a watchdog thread on its resident set. +check_platform_opts(Opts) -> + case {maps:get(cgroup, Opts, undefined), os:type()} of + {undefined, _} -> ok; + {_, {unix, linux}} -> ok; + {_, {unix, Os}} -> {error, {cgroup_unsupported, Os}} + end. + +start_child_1(#data{opts = Opts} = St) -> + case python_executable(Opts) of + {error, _} = Err -> + Err; + Python -> + case spawn_child(Python, Opts) of + {ok, Child} -> + handshake(St#data{child = Child}); + {error, _} = Err -> + Err + end + end. + +spawn_child(Python, Opts) -> + Dir = sock_dir(), + Path = filename:join(Dir, "ctx_" ++ integer_to_list(erlang:unique_integer([positive])) ++ ".sock"), + _ = file:delete(Path), + case socket:open(local, stream, default) of + {ok, L} -> + try + ok = socket:bind(L, #{family => local, path => Path}), + ok = socket:listen(L), + Script = filename:join(priv_dir(), "py_isolated_child.py"), + Args = [Script, Path | rlimit_args(Opts) ++ cgroup_args(Opts)], + PortOpts = [exit_status, stderr_to_stdout, binary, use_stdio, + {args, Args}, {env, env_opt(Opts)}], + Port = open_port({spawn_executable, Python}, PortOpts), + OsPid = case erlang:port_info(Port, os_pid) of + {os_pid, Pid} -> Pid; + _ -> 0 + end, + Timeout = maps:get(start_timeout, Opts, ?DEFAULT_START_TIMEOUT_MS), + case accept_child(L, Port, Timeout) of + {ok, S} -> + _ = file:delete(Path), + tune_socket(S), + {ok, #child{port = Port, os_pid = OsPid, listener = L, + sock = S, sock_path = Path}}; + {error, Reason} -> + _ = file:delete(Path), + socket:close(L), + kill_port(Port, OsPid), + {error, Reason} + end + catch + Class:Err:Stack -> + _ = file:delete(Path), + socket:close(L), + {error, {spawn_failed, {Class, Err, Stack}}} + end; + {error, Reason} -> + {error, {socket_open_failed, Reason}} + end. + +%% Accept while also watching the port: a child that dies before connecting +%% (bad interpreter, missing script) is reported with its output. +accept_child(L, Port, Timeout) -> + Deadline = erlang:monotonic_time(millisecond) + Timeout, + accept_child(L, Port, Deadline, []). + +accept_child(L, Port, Deadline, Out) -> + case socket:accept(L, nowait) of + {ok, S} -> + %% Output printed before connecting is still worth logging + [self() ! {Port, {data, D}} || D <- lists:reverse(Out)], + {ok, S}; + {select, {select_info, _, Handle}} -> + Left = max(0, Deadline - erlang:monotonic_time(millisecond)), + receive + {'$socket', L, select, Handle} -> + accept_child(L, Port, Deadline, Out); + {Port, {exit_status, Status}} -> + _ = socket:cancel(L, {select_info, accept, Handle}), + {error, {child_exited_at_start, exit_reason(Status), + drain_port_output(Port, Out)}}; + {Port, {data, D}} -> + %% Keep it here, not in the mailbox: re-sending it would + %% make this receive return at once and never time out + accept_child(L, Port, Deadline, [D | Out]) + after Left -> + _ = socket:cancel(L, {select_info, accept, Handle}), + {error, {start_timeout, drain_port_output(Port, Out)}} + end; + {error, Reason} -> + {error, {accept_failed, Reason}} + end. + +%% Default Unix socket buffers are small (8 KB on macOS); large payloads +%% would cross in hundreds of wakeups. Best effort: the kernel clamps. +tune_socket(S) -> + _ = socket:setopt(S, {otp, rcvbuf}, ?SOCKET_BUF), + _ = socket:setopt(S, {socket, rcvbuf}, ?SOCKET_BUF), + _ = socket:setopt(S, {socket, sndbuf}, ?SOCKET_BUF), + ok. + +drain_port_output(Port, Acc) -> + receive + {Port, {data, D}} -> drain_port_output(Port, [D | Acc]) + after 50 -> + iolist_to_binary(lists:reverse(Acc)) + end. + +%% Blocking handshake: ready event, init request, then the preload exec. +handshake(#data{child = Child, opts = Opts} = St0) -> + St = St0#data{}, + Timeout = maps:get(start_timeout, Opts, ?DEFAULT_START_TIMEOUT_MS), + case recv_frame_sync(Child, Timeout) of + {ok, {0, ?STATUS_EVENT, {ready, Info}}, Child1} -> + St1 = St#data{child = Child1#child{info = Info}}, + Paths = [to_bin(P) || P <- py_import:all_paths()] ++ extra_paths(Opts), + %% Registered imports are pre-cached in sys.modules, as + %% interp_apply_imports does for the embedded modes + Imports = lists:usort([to_bin(M) || {M, _} <- py_import:all_imports()]), + case sync_request(St1, {init, self(), Paths, Imports}, Timeout) of + {{ok, _}, St2} -> + run_preload(St2, Timeout); + {{error, Reason}, St2} -> + stop_child(St2, kill), + {error, {init_failed, Reason}} + end; + {ok, {0, ?STATUS_EVENT, {startup_error, Problems}}, Child1} -> + stop_child(St#data{child = Child1}, kill), + {error, {startup_error, Problems}}; + {ok, {0, ?STATUS_EVENT, {memory_limit, Rss}}, Child1} -> + %% The memory watchdog fired before the child was ready + stop_child(St#data{child = Child1}, kill), + {error, {startup_error, [{memory_limit, Rss}]}}; + {ok, Other, Child1} -> + stop_child(St#data{child = Child1}, kill), + {error, {unexpected_handshake, Other}}; + {error, Reason} -> + stop_child(St, kill), + {error, {handshake_failed, Reason}} + end. + +run_preload(#data{opts = Opts} = St, Timeout) -> + Code = case maps:get(preload, Opts, undefined) of + undefined -> py_preload_code(); + C -> [py_preload_code(), <<"\n">>, iolist_to_binary(C)] + end, + case iolist_to_binary(Code) of + <<>> -> + {ok, St}; + Bin -> + case sync_request(St, {exec, Bin}, Timeout) of + {{ok, _}, St1} -> + {ok, St1}; + {{error, Reason}, St1} -> + logger:warning("py_context ~p (isolated): preload failed: ~p", + [St#data.id, Reason]), + {ok, St1} + end + end. + +%% Global preload registered with py_preload (applied to every context) +py_preload_code() -> + try py_preload:get_code() of + Code when is_binary(Code) -> Code; + _ -> <<>> + catch + _:_ -> <<>> + end. + +extra_paths(Opts) -> + [to_bin(P) || P <- maps:get(paths, Opts, [])]. + +%% Send a request and wait for its reply, ignoring nothing: callbacks made +%% by the child during startup are served too. +sync_request(#data{child = Child, next_id = Id} = St, Term, Timeout) -> + case send_frame(Child, Id, ?STATUS_REQUEST, Term) of + ok -> sync_wait(St#data{next_id = Id + 1}, Id, Timeout); + {error, Reason} -> {{error, Reason}, St} + end. + +run_callback_bounded(Term, Timeout) -> + {Pid, Mon} = spawn_monitor(fun() -> exit({callback_done, run_callback(Term)}) end), + receive + {'DOWN', Mon, process, Pid, {callback_done, Result}} -> + Result; + {'DOWN', Mon, process, Pid, Reason} -> + {?STATUS_ERROR, iolist_to_binary(io_lib:format("callback crashed: ~p", [Reason]))} + after Timeout -> + erlang:demonitor(Mon, [flush]), + exit(Pid, kill), + {?STATUS_ERROR, <<"callback timed out during context start">>} + end. + +sync_wait(#data{child = Child} = St, Id, Timeout) -> + case recv_frame_sync(Child, Timeout) of + {ok, {FrameId, ?STATUS_CALLBACK, Term}, Child1} -> + %% Callbacks made while the child starts (preload, imports) run + %% in a separate process so a crash cannot take this one down. + %% They cannot re-enter this context yet: the loop is not + %% running, so such a call would wait for the handshake timeout. + St1 = St#data{child = Child1}, + {Status, Reply} = run_callback_bounded(Term, Timeout), + case send_frame(Child1, FrameId, Status, Reply) of + ok -> sync_wait(St1, Id, Timeout); + {error, Reason} -> {{error, Reason}, St1} + end; + {ok, {Id, Status, Term}, Child1} + when Status =:= ?STATUS_OK; Status =:= ?STATUS_ERROR -> + {reply_term(Status, Term), St#data{child = Child1}}; + {ok, {0, ?STATUS_EVENT, {log, Level, Msg}}, Child1} -> + log_event(St, Level, Msg), + sync_wait(St#data{child = Child1}, Id, Timeout); + {ok, {_, _, _}, Child1} -> + sync_wait(St#data{child = Child1}, Id, Timeout); + {error, Reason} -> + {{error, Reason}, St} + end. + +recv_frame_sync(#child{buf = Buf} = Child, Timeout) -> + case parse_frame(Buf) of + {ok, Frame, Rest} -> + {ok, Frame, Child#child{buf = Rest}}; + more -> + case socket:recv(Child#child.sock, 0, Timeout) of + {ok, <<>>} -> + {error, closed}; + {ok, Data} -> + recv_frame_sync(Child#child{buf = <>}, Timeout); + {error, {Reason, _Data}} -> + {error, Reason}; + {error, Reason} -> + {error, Reason} + end + end. + +%% ============================================================================ +%% Requests to the child +%% ============================================================================ + +%% Every path returns a gen_statem result. Main-thread requests (call, eval, +%% exec, start_loop) are served one caller at a time; the rest is handled by +%% the child's reader thread and can go in any state that has a child. +request(From, MRef, {start_loop, _}, _Term, State, _Data) + when State =:= looping; State =:= stopping_loop -> + From ! {MRef, {error, already_running}}, + keep_state_and_data; +request(From, MRef, Kind, _Term, looping, _Data) when ?IS_MAIN(Kind) -> + From ! {MRef, {error, loop_running}}, + keep_state_and_data; +request(_From, _MRef, _Kind, _Term, {restarting, _}, _Data) -> + {keep_state_and_data, [postpone]}; +request(_From, _MRef, Kind, _Term, stopping_loop, _Data) when ?IS_MAIN(Kind) -> + {keep_state_and_data, [postpone]}; +request(From, MRef, Kind, Term, {busy, _} = State, #data{cb_pids = CbPids} = Data) + when ?IS_MAIN(Kind) -> + case maps:is_key(From, CbPids) of + true -> + %% Nested: a callback of the executing request calls back in + result(dispatch(From, MRef, Kind, Term, State, Data)); + false -> + {keep_state_and_data, [postpone]} + end; +request(From, MRef, Kind, Term, idle, Data) when ?IS_MAIN(Kind) -> + case dispatch(From, MRef, Kind, Term, idle, Data) of + {idle, Data1, Actions} -> + {next_state, {busy, Data1#data.next_id - 1}, Data1, Actions}; + Other -> + result(Other) + end; +request(From, MRef, Kind, Term, State, Data) -> + result(dispatch(From, MRef, Kind, Term, State, Data)). + +dispatch(From, MRef, Kind, Term, State, #data{child = Child, next_id = Id, pending = Pending} = Data) -> + case send_frame(Child, Id, ?STATUS_REQUEST, Term) of + ok -> + {State, Data#data{next_id = Id + 1, pending = Pending#{Id => {From, MRef, Kind}}}, []}; + {error, Reason} -> + From ! {MRef, {error, {child_exited, Reason}}}, + socket_broken(Reason, State, Data, []) + end. + +pass_fd(_From, _MRef, _Fd, {restarting, _}, _Data) -> + {keep_state_and_data, [postpone]}; +pass_fd(From, MRef, Fd, _State, #data{child = Child, next_id = Id, pending = Pending} = Data) + when is_integer(Fd), Fd >= 0 -> + Frame = frame(Id, ?STATUS_REQUEST, term_to_binary(pass_fd)), + Msg = #{iov => [Frame], + ctrl => [#{level => socket, type => rights, data => <>}]}, + case socket:sendmsg(Child#child.sock, Msg) of + ok -> + {keep_state, Data#data{next_id = Id + 1, pending = Pending#{Id => {From, MRef, pass_fd}}}}; + {error, Reason} -> + From ! {MRef, {error, {pass_fd_failed, Reason}}}, + keep_state_and_data + end; +pass_fd(From, MRef, Fd, _State, _Data) -> + From ! {MRef, {error, {invalid_fd, Fd}}}, + keep_state_and_data. + +%% Turn a {State, Data, Actions} triple into a gen_statem result +result({stop, Reason, Data}) -> + {stop, Reason, Data}; +result({State, Data, Actions}) -> + {next_state, State, Data, Actions}. + +%% ============================================================================ +%% Frames from the child +%% ============================================================================ + +%% Read until the socket would block, processing complete frames. +drain_socket({restarting, _} = State, Data, Actions) -> + {State, Data, Actions}; +drain_socket(State, #data{child = #child{sock = S, buf = Buf} = Child} = Data, Actions) -> + case socket:recv(S, 0, nowait) of + {ok, <<>>} -> + socket_broken(closed, State, Data, Actions); + {ok, Bytes} -> + Data1 = Data#data{child = Child#child{buf = <>}}, + case process_frames(State, Data1, Actions) of + {{restarting, _}, _, _} = Broken -> Broken; + {State1, Data2, Actions1} -> drain_socket(State1, Data2, Actions1) + end; + {select, _SelectInfo} -> + process_frames(State, Data, Actions); + {error, {Reason, Bytes}} when is_binary(Bytes) -> + Data1 = Data#data{child = Child#child{buf = <>}}, + {State1, Data2, Actions1} = process_frames(State, Data1, Actions), + socket_broken(Reason, State1, Data2, Actions1); + {error, Reason} -> + socket_broken(Reason, State, Data, Actions) + end. + +process_frames({restarting, _} = State, Data, Actions) -> + {State, Data, Actions}; +process_frames(State, #data{child = #child{buf = Buf} = Child} = Data, Actions) -> + case parse_frame(Buf) of + {ok, Frame, Rest} -> + Data1 = Data#data{child = Child#child{buf = Rest}}, + {State1, Data2, Actions1} = handle_frame(Frame, State, Data1, Actions), + process_frames(State1, Data2, Actions1); + more -> + {State, Data, Actions}; + {error, Reason} -> + socket_broken({malformed_frame, Reason}, State, Data, Actions) + end. + +parse_frame(<>) -> + case Body of + <> -> + try + Term = case Payload of + <<>> -> undefined; + _ -> binary_to_term(Payload) + end, + {ok, {Id, Status, Term}, Rest} + catch + error:badarg -> {error, bad_etf} + end; + <<>> -> + {error, empty_body} + end; +parse_frame(_) -> + more. + + +handle_frame({Id, Status, Term}, State, #data{pending = Pending} = Data, Actions) + when Status =:= ?STATUS_OK; Status =:= ?STATUS_ERROR -> + case maps:take(Id, Pending) of + {{From, MRef, Kind}, Rest} -> + {Data1, Actions1} = cancel_kill_timer(Id, Data#data{pending = Rest}, Actions), + {Next, Data2} = deliver(Kind, From, MRef, reply_term(Status, Term), Data1), + %% Only the reply of the request holding the main thread frees + %% it; nested replies (callbacks calling back in) do not + State1 = case {Next, State} of + {looping, _} -> looping; + {done, {busy, Id}} -> idle; + _ -> State + end, + {State1, Data2, Actions1}; + error -> + {State, Data, Actions} + end; +handle_frame({Id, ?STATUS_CALLBACK, Term}, State, #data{callbacks = Cbs, cb_pids = CbPids} = Data, Actions) -> + Owner = self(), + {Pid, Mon} = spawn_monitor(fun() -> + {Status, Reply} = run_callback(Term), + Owner ! {callback_reply, Id, Status, Reply} + end), + {State, Data#data{callbacks = Cbs#{Mon => Id}, cb_pids = CbPids#{Pid => Mon}}, Actions}; +handle_frame({_, ?STATUS_EVENT, Event}, State, Data, Actions) -> + on_child_event(Event, State, Data, Actions); +handle_frame({_, _, _}, State, Data, Actions) -> + {State, Data, Actions}. + +%% Deliver a reply. Returns `done' for a main-thread request, `looping' +%% when a loop just started, `keep' otherwise. +deliver(exec, From, MRef, {ok, _}, Data) -> + From ! {MRef, ok}, + {done, Data}; +deliver({start_loop, Owner}, From, MRef, {ok, _}, Data) -> + Mon = case is_pid(Owner) of + true -> erlang:monitor(process, Owner); + false -> undefined + end, + From ! {MRef, ok}, + {looping, Data#data{loop = #loop{owner = Owner, owner_mon = Mon}}}; +deliver({submit, TaskRef}, From, MRef, {ok, _}, #data{tasks = Tasks} = Data) -> + From ! {MRef, {ok, TaskRef}}, + {keep, Data#data{tasks = Tasks#{TaskRef => From}}}; +deliver(Kind, From, MRef, Reply, Data) when ?IS_MAIN(Kind) -> + From ! {MRef, Reply}, + {done, Data}; +deliver(_Kind, From, MRef, Reply, Data) -> + From ! {MRef, Reply}, + {keep, Data}. + +on_child_event({async_result, TaskRef, Result}, State, #data{tasks = Tasks} = Data, Actions) -> + case maps:take(TaskRef, Tasks) of + {Pid, Rest} -> + Pid ! {async_result, TaskRef, Result}, + {State, Data#data{tasks = Rest}, Actions}; + error -> + {State, Data, Actions} + end; +on_child_event({loop_exit, Result}, State, Data, Actions) + when State =:= looping; State =:= stopping_loop -> + {Data1, Actions1} = cancel_kill_timer(loop, Data, Actions), + {idle, loop_exited(Result, Data1), Actions1}; +on_child_event({memory_limit, Rss}, State, Data, Actions) -> + %% The child's memory watchdog is exiting; the exit_status follows + socket_broken({memory_limit, Rss}, State, Data, Actions); +on_child_event({log, Level, Msg}, State, Data, Actions) -> + log_event(Data, Level, Msg), + {State, Data, Actions}; +on_child_event(_, State, Data, Actions) -> + {State, Data, Actions}. + +reply_term(?STATUS_OK, Term) -> {ok, Term}; +reply_term(?STATUS_ERROR, Term) -> {error, Term}. + +%% --------------------------------------------------------------------------- +%% Callbacks (child -> Erlang) +%% --------------------------------------------------------------------------- + +run_callback({call, Name, Args}) -> + ArgsList = case Args of + L when is_list(L) -> L; + T when is_tuple(T) -> tuple_to_list(T); + _ -> [Args] + end, + try py_callback:execute(to_bin(Name), ArgsList) of + {ok, Result} -> + {?STATUS_OK, Result}; + {error, {not_found, N}} -> + {?STATUS_ERROR, iolist_to_binary(io_lib:format("Function '~s' not registered", [N]))}; + {error, {Class, Reason, _Stack}} -> + {?STATUS_ERROR, iolist_to_binary(io_lib:format("~p: ~p", [Class, Reason]))} + catch + Class:Reason -> + {?STATUS_ERROR, iolist_to_binary(io_lib:format("~p:~p", [Class, Reason]))} + end; +run_callback({send, Pid, Msg}) when is_pid(Pid) -> + case node(Pid) =:= node() andalso not is_process_alive(Pid) of + true -> {?STATUS_ERROR, {noproc, Pid}}; + false -> Pid ! Msg, {?STATUS_OK, ok} + end; +run_callback({send, Other, _}) -> + {?STATUS_ERROR, {badarg, Other}}; +run_callback({whereis, Name}) -> + try + Atom = if is_atom(Name) -> Name; + is_binary(Name) -> binary_to_existing_atom(Name, utf8); + is_list(Name) -> list_to_existing_atom(Name) + end, + case erlang:whereis(Atom) of + undefined -> {?STATUS_OK, none}; + Pid -> {?STATUS_OK, Pid} + end + catch + _:_ -> {?STATUS_OK, none} + end; +run_callback(Other) -> + {?STATUS_ERROR, {unknown_request, Other}}. + + +%% ============================================================================ +%% Interrupt / kill +%% ============================================================================ + +%% What an interrupt targets: the innermost main-thread request the child +%% is executing (nested requests are dispatched while the outer waits), or +%% the loop. +executing_request(looping, _Data) -> loop; +executing_request(stopping_loop, _Data) -> loop; +executing_request({busy, _}, #data{pending = Pending}) -> + lists:max([Id || {Id, {_, _, Kind}} <- maps:to_list(Pending), ?IS_MAIN(Kind)]); +executing_request(_State, _Data) -> undefined. + +%% The child signals only if Target is what it is executing, so an +%% interrupt for a request that just completed cannot hit its successor. +%% The kill backstop is bound to Target. +send_interrupt(Target, State, #data{opts = Opts} = Data, Actions) -> + case send_frame(Data#data.child, 0, ?STATUS_CONTROL, {interrupt, Target}) of + ok -> + After = maps:get(kill_after, Opts, ?DEFAULT_KILL_AFTER_MS), + {State, Data#data{kill_target = Target}, + [{{timeout, kill}, After, Target} | Actions]}; + {error, Reason} -> + socket_broken(Reason, State, Data, Actions) + end. + +%% A reply for the interrupted request means the interrupt landed +cancel_kill_timer(Target, #data{kill_target = Target} = Data, Actions) -> + {Data#data{kill_target = undefined}, [{{timeout, kill}, cancel} | Actions]}; +cancel_kill_timer(_Target, Data, Actions) -> + {Data, Actions}. + +%% SIGKILL the child; the port's exit_status drives the restart. Callers +%% of kill/1 are answered when the new child is idle. +kill(Reason, State, #data{child = #child{port = Port, os_pid = OsPid}} = Data) -> + kill_port(Port, OsPid), + result(enter_restarting(Reason, State, Data, [])); +kill(_Reason, _State, _Data) -> + keep_state_and_data. + +kill_port(Port, OsPid) -> + case OsPid > 0 andalso erlang:port_info(Port) =/= undefined of + true -> _ = py_nif:os_kill(OsPid, 9), ok; + false -> ok + end. + + +%% ============================================================================ +%% Failure handling and restart +%% ============================================================================ + +%% Nothing can reach the child any more. Make sure it exits; the port's +%% exit_status (which follows within milliseconds) fails the pending +%% requests with the real cause and runs the restart policy. +socket_broken(_Reason, {restarting, _} = State, Data, Actions) -> + {State, Data, Actions}; +socket_broken(Reason, State, #data{child = #child{port = Port, os_pid = OsPid}} = Data, Actions) -> + logger:debug("py_context ~p (isolated): socket to child ~p broken: ~p", + [Data#data.id, OsPid, Reason]), + kill_port(Port, OsPid), + enter_restarting({child_exited, {socket, Reason}}, State, Data, Actions). + +enter_restarting(_Reason, {restarting, _} = State, Data, Actions) -> + {State, Data, Actions}; +enter_restarting(Reason, _State, Data, Actions) -> + %% Timers of the old child are meaningless now + {{restarting, Reason}, Data#data{kill_target = undefined}, + [{{timeout, kill}, cancel} | Actions]}. + +exit_reason(Status) when Status > 128 -> {signal, Status - 128}; +exit_reason(Status) -> {exit_status, Status}. + +%% The port reported the child's exit: fail what was in flight, then +%% restart within the budget or stop. +child_exited(Reason, State, #data{child = Child, opts = Opts} = Data0) -> + close_child(Child), + FailReason = case {State, Reason} of + {{restarting, killed}, _} -> killed; + %% The memory watchdog announced the exit; our SIGKILL may win the race + {{restarting, {child_exited, {socket, {memory_limit, _} = Mem}}}, _} -> {child_exited, Mem}; + %% We killed it because the socket broke: report the socket, unless + %% the child was already dying of something more telling + {{restarting, {child_exited, {socket, _}} = SockReason}, {signal, 9}} -> SockReason; + {{restarting, {child_exited, {socket, _}}}, _} -> {child_exited, Reason}; + {{restarting, Other}, _} -> Other; + {_, _} -> {child_exited, Reason} + end, + Data1 = fail_pending(FailReason, Data0#data{child = undefined, kill_target = undefined}), + case FailReason of + killed -> + logger:info("py_context ~p (isolated): child killed", [Data0#data.id]); + _ -> + logger:warning("py_context ~p (isolated): child exited: ~p", + [Data0#data.id, FailReason]) + end, + case maps:get(restart, Opts, true) andalso restart_allowed(Data1) of + true -> + Now = erlang:monotonic_time(millisecond), + Data2 = Data1#data{restarts = [Now | Data1#data.restarts]}, + case start_child(Data2) of + {ok, Data3} -> + logger:info("py_context ~p (isolated): child restarted (pid ~p)", + [Data0#data.id, (Data3#data.child)#child.os_pid]), + result(drain_socket(idle, Data3, [{{timeout, kill}, cancel}])); + {error, RestartError} -> + logger:error("py_context ~p (isolated): restart failed: ~p", + [Data0#data.id, RestartError]), + {stop, {child_restart_failed, RestartError}, Data1} + end; + false -> + {stop, {child_exited, Reason}, Data1} + end. + +%% In-flight requests, submitted tasks and a running loop fail with Reason. +%% Postponed requests are untouched: they are served by the next child, or +%% their callers get a DOWN if the process stops. +fail_pending(Reason, #data{pending = Pending, tasks = Tasks} = Data) -> + maps:foreach(fun(_, {From, MRef, _Kind}) -> + From ! {MRef, {error, Reason}} + end, Pending), + maps:foreach(fun(TaskRef, Pid) -> + Pid ! {async_result, TaskRef, {error, Reason}} + end, Tasks), + Data1 = case Data#data.loop of + undefined -> Data; + _ -> loop_exited({error, Reason}, Data) + end, + Data1#data{pending = #{}, tasks = #{}}. + +restart_allowed(#data{restarts = Restarts, opts = Opts}) -> + Max = maps:get(max_restarts, Opts, ?DEFAULT_MAX_RESTARTS), + Period = maps:get(restart_period, Opts, ?DEFAULT_RESTART_PERIOD_MS), + Now = erlang:monotonic_time(millisecond), + Recent = [T || T <- Restarts, Now - T =< Period], + length(Recent) < Max. + + +%% Graceful: ask the child to exit, wait briefly, then SIGKILL. +stop_child(#data{child = undefined} = Data, _How) -> + Data; +stop_child(#data{child = #child{port = Port, os_pid = OsPid} = Child} = Data, How) -> + case How of + graceful -> + _ = send_frame(Child, 0, ?STATUS_REQUEST, shutdown), + receive + {Port, {exit_status, _}} -> ok + after ?SHUTDOWN_GRACE_MS -> + kill_port(Port, OsPid), + wait_exit(Port) + end; + _ -> + kill_port(Port, OsPid), + wait_exit(Port) + end, + close_child(Child), + Data1 = fail_pending({child_exited, stopped}, Data), + Data1#data{child = undefined}. + +wait_exit(Port) -> + receive + {Port, {exit_status, _}} -> ok + after 2000 -> + ok + end. + +close_child(#child{port = Port, sock = S, listener = L}) -> + _ = socket:close(S), + _ = socket:close(L), + try port_close(Port) catch error:badarg -> ok end, + ok. + + +%% --------------------------------------------------------------------------- +%% Worker loop helpers +%% --------------------------------------------------------------------------- + +loop_exited(Result, #data{loop = #loop{owner = Owner, owner_mon = Mon, + stop_waiters = Waiters}} = Data) -> + case Mon of + undefined -> ok; + _ -> erlang:demonitor(Mon, [flush]) + end, + case is_pid(Owner) of + true -> Owner ! {py_loop_exit, self(), Result}; + false -> ok + end, + [W ! {M, ok} || {W, M} <- Waiters], + Data#data{loop = undefined}; +loop_exited(_Result, Data) -> + Data. + +%% --------------------------------------------------------------------------- +%% Wire helpers +%% --------------------------------------------------------------------------- + +frame(Id, Status, Payload) -> + Body = <>, + <>. + +send_frame(#child{sock = S}, Id, Status, Term) -> + case socket:send(S, frame(Id, Status, term_to_binary(Term))) of + ok -> ok; + {error, {Reason, _Rest}} -> {error, Reason}; + {error, Reason} -> {error, Reason} + end. + +log_output(#data{id = Id, child = #child{os_pid = OsPid}}, Data) -> + Lines = binary:split(Data, <<"\n">>, [global, trim_all]), + [logger:info("py_context ~p (isolated pid ~p): ~s", [Id, OsPid, L]) || L <- Lines], + ok. + +log_event(#data{id = Id}, Level, Msg) -> + Lvl = case Level of + error -> error; warning -> warning; debug -> debug; _ -> info + end, + logger:log(Lvl, "py_context ~p (isolated): ~s", [Id, Msg]). + +sock_dir() -> + Base = case os:getenv("TMPDIR") of + false -> "/tmp"; + T -> T + end, + Dir = filename:join(Base, "erlang_python_" ++ os:getpid()), + ok = filelib:ensure_dir(filename:join(Dir, "x")), + _ = file:change_mode(Dir, 8#700), + Dir. + +priv_dir() -> + case code:priv_dir(erlang_python) of + {error, bad_name} -> + filename:join(filename:dirname(filename:dirname(code:which(?MODULE))), "priv"); + Dir -> + Dir + end. + +rlimit_args(Opts) -> + Limits = maps:get(rlimits, Opts, #{}), + lists:append([case maps:get(K, Limits, undefined) of + undefined -> []; + V when is_integer(V), V >= 0 -> ["--rlimit-" ++ atom_to_list(K), integer_to_list(V)] + end || K <- [as, cpu, nofile]]). + +cgroup_args(Opts) -> + case maps:get(cgroup, Opts, undefined) of + undefined -> []; + Dir -> ["--cgroup", to_list(Dir)] + end. + +env_opt(Opts) -> + [{to_list(K), to_list(V)} || {K, V} <- maps:to_list(maps:get(env, Opts, #{}))]. + +to_bin(A) when is_atom(A) -> atom_to_binary(A, utf8); +to_bin(L) when is_list(L) -> unicode:characters_to_binary(L); +to_bin(B) when is_binary(B) -> B. + +to_list(A) when is_atom(A) -> atom_to_list(A); +to_list(B) when is_binary(B) -> unicode:characters_to_list(B); +to_list(L) when is_list(L) -> L. diff --git a/src/py_nif.erl b/src/py_nif.erl index 0357bc6..e868dc8 100644 --- a/src/py_nif.erl +++ b/src/py_nif.erl @@ -131,6 +131,7 @@ close_fd/1, %% File descriptor utilities dup_fd/1, + os_kill/2, %% Test helpers for fd monitoring (using pipes) create_test_pipe/0, close_test_fd/1, @@ -992,6 +993,11 @@ create_test_pipe() -> dup_fd(_Fd) -> ?NIF_STUB. +%% @doc Send a signal to an OS process (kill(2)). Used by isolated contexts. +-spec os_kill(pos_integer(), non_neg_integer()) -> ok | {error, esrch | eperm | einval}. +os_kill(_Pid, _Signal) -> + ?NIF_STUB. + %% @doc Close a test file descriptor. -spec close_test_fd(integer()) -> ok | {error, term()}. close_test_fd(_Fd) -> diff --git a/test/py_isolated_SUITE.erl b/test/py_isolated_SUITE.erl new file mode 100644 index 0000000..937bd9d --- /dev/null +++ b/test/py_isolated_SUITE.erl @@ -0,0 +1,874 @@ +%%% @doc Common Test suite for `isolated' context mode. +%%% +%%% The interpreter runs in a child OS process. Round-trip cases run in a +%%% worker group too, so the two modes are held to the same results. The +%%% isolation cases (kill, segfault, rlimits, reaping, socket break) are what +%%% the embedded modes cannot do; the ones marked "contrast" assert the +%%% embedded behaviour as well, to document the difference. +-module(py_isolated_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([ + all/0, + groups/0, + init_per_suite/1, + end_per_suite/1, + init_per_group/2, + end_per_group/2, + init_per_testcase/2, + end_per_testcase/2 +]). + +-export([ + test_call_eval_exec/1, + test_state_persists/1, + test_kwargs/1, + test_type_round_trip/1, + test_python_error/1, + test_missing_module_and_function/1, + test_large_payloads/1, + test_callback_round_trip/1, + test_nested_callback/1, + test_callback_error/1, + test_send_to_pid/1, + test_concurrent_callers/1, + test_timeout_interrupts_sleep/1, + test_queued_timeout_does_not_interrupt_others/1, + test_sys_state_reflects_activity/1, + test_requests_during_restart_are_served/1, + test_kill_reply_after_restart/1, + test_stop_while_busy_and_looping/1, + test_sys_get_status/1, + test_context_outlives_creator/1, + test_pool_of_isolated_contexts/1, + test_child_info/1, + test_sleep_is_interrupted/1, + test_sleep_not_interrupted_in_worker/1, + test_kill_backstop/1, + test_kill_restarts_child/1, + test_segfault_kills_only_child/1, + test_rlimit_as/1, + test_rlimit_cpu/1, + test_rlimit_nofile/1, + test_reaped_on_stop/1, + test_reaped_on_crash/1, + test_reaped_on_kill/1, + test_no_orphan_when_vm_dies/1, + test_socket_break_mid_call/1, + test_restart_false_exits_context/1, + test_restart_budget/1, + test_numpy_imports/1, + test_not_supported_fail_loud/1, + test_bad_python_fails_at_start/1, + test_startup_error_reported/1, + test_cgroup_option_platform/1, + test_env_option/1, + test_preload_option/1 +]). + +-define(TEST_MOD, py_test_isolated). + +all() -> + [{group, worker}, {group, isolated}, {group, isolation}]. + +groups() -> + RoundTrip = [ + test_call_eval_exec, + test_state_persists, + test_kwargs, + test_type_round_trip, + test_python_error, + test_missing_module_and_function, + test_large_payloads, + test_callback_round_trip, + test_nested_callback, + test_callback_error, + test_send_to_pid, + test_concurrent_callers, + test_timeout_interrupts_sleep + ], + Isolation = [ + test_pool_of_isolated_contexts, + test_child_info, + test_sleep_is_interrupted, + test_sleep_not_interrupted_in_worker, + test_kill_backstop, + test_kill_restarts_child, + test_segfault_kills_only_child, + test_rlimit_as, + test_rlimit_cpu, + test_rlimit_nofile, + test_reaped_on_stop, + test_reaped_on_crash, + test_reaped_on_kill, + test_no_orphan_when_vm_dies, + test_socket_break_mid_call, + test_restart_false_exits_context, + test_restart_budget, + test_numpy_imports, + test_not_supported_fail_loud, + test_queued_timeout_does_not_interrupt_others, + test_sys_state_reflects_activity, + test_requests_during_restart_are_served, + test_kill_reply_after_restart, + test_stop_while_busy_and_looping, + test_sys_get_status, + test_context_outlives_creator, + test_bad_python_fails_at_start, + test_startup_error_reported, + test_cgroup_option_platform, + test_env_option, + test_preload_option + ], + [{worker, [], RoundTrip}, + {isolated, [], RoundTrip}, + {isolation, [], Isolation}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(erlang_python), + [{test_dir, test_dir()} | Config]. + +end_per_suite(_Config) -> + ok = application:stop(erlang_python), + ok. + +init_per_group(isolation, Config) -> + [{mode, isolated} | Config]; +init_per_group(Mode, Config) -> + [{mode, Mode} | Config]. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + flush(), + ok. + +%%% ============================================================================ +%%% Round-trip cases (both modes) +%%% ============================================================================ + +test_call_eval_exec(Config) -> + C = new_ctx(Config), + {ok, 3} = py_context:call(C, ?TEST_MOD, add, [1, 2]), + {ok, 4.0} = py_context:call(C, math, sqrt, [16]), + {ok, 6} = py_context:eval(C, <<"2*3">>), + {ok, 10} = py_context:eval(C, <<"a + b">>, #{a => 4, b => 6}), + ok = py_context:exec(C, <<"def twice(x):\n return 2 * x\n">>), + {ok, 14} = py_context:call(C, '__main__', twice, [7]), + stop(C). + +test_state_persists(Config) -> + C = new_ctx(Config), + ok = py_context:exec(C, <<"counter = 0">>), + lists:foreach(fun(_) -> ok = py_context:exec(C, <<"counter += 1">>) end, + lists:seq(1, 10)), + {ok, 10} = py_context:eval(C, <<"counter">>), + stop(C). + +test_kwargs(Config) -> + C = new_ctx(Config), + {ok, {[1, 2], [{<<"a">>, 3}, {<<"b">>, <<"x">>}]}} = + py_context:call(C, ?TEST_MOD, kwargs_probe, [1, 2], #{a => 3, b => <<"x">>}), + stop(C). + +test_type_round_trip(Config) -> + C = new_ctx(Config), + Probe = fun(V) -> + {ok, Got} = py_context:call(C, ?TEST_MOD, identity, [V]), + Got + end, + TypeOf = fun(V) -> + {ok, T} = py_context:call(C, ?TEST_MOD, type_name, [V]), + T + end, + true = Probe(true), + false = Probe(false), + none = Probe(none), + none = Probe(undefined), + none = Probe(nil), + <<"str">> = TypeOf(<<"héllo"/utf8>>), + <<"héllo"/utf8>> = Probe(<<"héllo"/utf8>>), + <<"bytes">> = TypeOf(<<255, 0, 1>>), + <<255, 0, 1>> = Probe(<<255, 0, 1>>), + <<"bytes">> = TypeOf({bytes, <<"abc">>}), + <<"abc">> = Probe({bytes, <<"abc">>}), + 42 = Probe(42), + -1 = Probe(-1), + %% Integers beyond 64 bits: the NIF converter has no bignum path + %% (worker mode returns none); the ETF codec carries them exactly. + case ?config(mode, Config) of + isolated -> + Big = 1 bsl 100, + Big = Probe(Big), + NegBig = -(1 bsl 100), + NegBig = Probe(NegBig); + _ -> + ok + end, + 3.5 = Probe(3.5), + [] = Probe([]), + [1, [2, 3], {4}] = Probe([1, [2, 3], {4}]), + "abc" = Probe("abc"), + {1, 2, 3} = Probe({1, 2, 3}), + #{<<"k">> := [1, 2], 3 := {<<"a">>}} = Probe(#{<<"k">> => [1, 2], 3 => {a}}), + <<"some_atom">> = Probe(some_atom), + <<"str">> = TypeOf(some_atom), + Self = self(), + Self = Probe(Self), + <<"Pid">> = TypeOf(Self), + Ref = make_ref(), + Ref = Probe(Ref), + <<"Ref">> = TypeOf(Ref), + {ok, nan} = py_context:eval(C, <<"float('nan')">>), + {ok, infinity} = py_context:eval(C, <<"float('inf')">>), + {ok, neg_infinity} = py_context:eval(C, <<"float('-inf')">>), + {ok, {1, 2, 3}} = py_context:eval(C, <<"(1, 2, 3)">>), + stop(C). + +test_python_error(Config) -> + C = new_ctx(Config), + {error, {'ValueError', Msg}} = py_context:call(C, ?TEST_MOD, raise_value_error, [<<"boom">>]), + true = lists:prefix("boom", to_list(Msg)), + {error, {'ZeroDivisionError', _}} = py_context:eval(C, <<"1/0">>), + {error, {'SyntaxError', _}} = py_context:exec(C, <<"def (:">>), + %% Still usable + {ok, 2} = py_context:eval(C, <<"1+1">>), + stop(C). + +test_missing_module_and_function(Config) -> + C = new_ctx(Config), + {error, {'ModuleNotFoundError', _}} = py_context:call(C, no_such_module_xyz, f, []), + {error, {'AttributeError', _}} = py_context:call(C, math, no_such_function, []), + stop(C). + +test_large_payloads(Config) -> + C = new_ctx(Config), + lists:foreach(fun(Size) -> + Bin = crypto:strong_rand_bytes(Size), + {ok, Bin} = py_context:call(C, ?TEST_MOD, identity, [Bin]), + {ok, Out} = py_context:call(C, ?TEST_MOD, big_payload, [Size]), + Size = byte_size(Out) + end, [1024 * 1024, 16 * 1024 * 1024]), + stop(C). + +test_callback_round_trip(Config) -> + C = new_ctx(Config), + py_callback:register(<<"iso_double">>, fun([X]) -> X * 2 end), + {ok, 84} = py_context:call(C, ?TEST_MOD, callback, [<<"iso_double">>, 42]), + {ok, 84} = py_context:eval(C, <<"__import__('erlang').call('iso_double', 42)">>), + %% Attribute sugar: erlang.iso_double(...) + {ok, 84} = py_context:eval(C, <<"__import__('erlang').iso_double(42)">>), + py_callback:unregister(<<"iso_double">>), + stop(C). + +%% @doc A callback that calls back into the same context (nesting), two +%% levels deep. The socket protocol nests arbitrarily; worker mode's +%% suspension protocol does not, so that group only checks it is loud. +test_nested_callback(Config) -> + C = new_ctx(Config), + py_callback:register(<<"iso_nested">>, fun([X]) -> + {ok, R} = py_context:call(C, ?TEST_MOD, add, [X, 1]), + R + end), + py_callback:register(<<"iso_nested2">>, fun([X]) -> + {ok, R} = py_context:call(C, ?TEST_MOD, callback, [<<"iso_nested">>, X]), + R + 100 + end), + case ?config(mode, Config) of + isolated -> + {ok, 11} = py_context:call(C, ?TEST_MOD, callback, [<<"iso_nested">>, 10]), + {ok, 111} = py_context:call(C, ?TEST_MOD, callback, [<<"iso_nested2">>, 10]); + worker -> + case py_context:call(C, ?TEST_MOD, callback, [<<"iso_nested">>, 10]) of + {ok, 11} -> ok; + {error, _} -> ok + end + end, + py_callback:unregister(<<"iso_nested">>), + py_callback:unregister(<<"iso_nested2">>), + stop(C). + +test_callback_error(Config) -> + C = new_ctx(Config), + py_callback:register(<<"iso_crash">>, fun(_) -> error(deliberate) end), + {ok, <<"RuntimeError">>} = py_context:call(C, ?TEST_MOD, callback_error_type, [<<"iso_crash">>]), + {ok, <<"RuntimeError">>} = py_context:call(C, ?TEST_MOD, callback_error_type, [<<"iso_not_registered">>]), + py_callback:unregister(<<"iso_crash">>), + stop(C). + +test_send_to_pid(Config) -> + C = new_ctx(Config), + {ok, true} = py_context:call(C, ?TEST_MOD, send, [self(), {hello, 1}]), + receive {<<"hello">>, 1} -> ok after 2000 -> ct:fail(no_message) end, + stop(C). + +test_concurrent_callers(Config) -> + C = new_ctx(Config), + Self = self(), + N = 20, + Pids = [spawn_link(fun() -> + Results = [py_context:call(C, ?TEST_MOD, add, [I, J]) || J <- lists:seq(1, 25)], + Self ! {done, I, Results} + end) || I <- lists:seq(1, N)], + lists:foreach(fun(I) -> + receive + {done, I, Results} -> + Expected = [{ok, I + J} || J <- lists:seq(1, 25)], + Expected = Results + after 30000 -> + ct:fail({timeout, I}) + end + end, lists:seq(1, N)), + _ = Pids, + stop(C). + +test_timeout_interrupts_sleep(Config) -> + C = new_ctx(Config), + T0 = erlang:monotonic_time(millisecond), + {error, timeout} = py_context:eval(C, <<"__import__('time').sleep(0.3)">>, #{}, 100), + Elapsed = erlang:monotonic_time(millisecond) - T0, + %% Both modes return promptly; the sleep itself ends by the time we + %% call again in worker mode (0.3 s), immediately in isolated mode. + true = Elapsed < 1500, + {ok, 4} = py_context:eval(C, <<"2+2">>, #{}, 5000), + stop(C). + +%%% ============================================================================ +%%% Isolation cases +%%% ============================================================================ + +test_pool_of_isolated_contexts(_Config) -> + {ok, Ctxs} = py_context_router:start_pool(iso_pool, 4, isolated), + 4 = length(Ctxs), + Pids = lists:usort([begin + {ok, #{os_pid := P}} = py_context:child_info(Cx), P + end || Cx <- Ctxs]), + 4 = length(Pids), + [{ok, 4} = py_context:eval(Cx, <<"2+2">>) || Cx <- Ctxs], + {ok, 4.0} = py:call(iso_pool, math, sqrt, [16]), + ok = py_context_router:stop_pool(iso_pool), + timer:sleep(200), + [false = os_pid_alive(P) || P <- Pids], + ok. + +test_child_info(Config) -> + C = new_ctx(Config), + {ok, #{os_pid := Pid, python_version := V, executable := Exe}} = py_context:child_info(C), + true = is_integer(Pid) andalso Pid > 0, + true = is_binary(V), + true = is_binary(Exe), + true = os_pid_alive(Pid), + {error, not_isolated} = py_context:child_info(self()), + stop(C). + +%% @doc The case the embedded modes cannot pass: a blocking C call is +%% interrupted at once. +test_sleep_is_interrupted(Config) -> + C = new_ctx(Config), + Self = self(), + spawn_link(fun() -> + Self ! {result, py_context:call(C, ?TEST_MOD, sleep_then, [60, ok])} + end), + timer:sleep(200), + T0 = erlang:monotonic_time(millisecond), + ok = py_context:interrupt(C), + receive + {result, R} -> + {error, interrupted} = R, + Elapsed = erlang:monotonic_time(millisecond) - T0, + ct:log("interrupted after ~p ms", [Elapsed]), + true = Elapsed < 1000 + after 5000 -> + ct:fail(sleep_not_interrupted) + end, + %% Same child, still usable, state intact + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +%% @doc Contrast: in worker mode the interrupt only lands when the C call +%% returns, so a 1.5 s sleep takes its full time. +test_sleep_not_interrupted_in_worker(_Config) -> + {ok, C} = py_context:new(#{mode => worker}), + Self = self(), + spawn_link(fun() -> + Self ! {result, py_context:eval(C, <<"__import__('time').sleep(1.5)">>)} + end), + timer:sleep(200), + T0 = erlang:monotonic_time(millisecond), + _ = py_context:interrupt(C), + receive + {result, _} -> + Elapsed = erlang:monotonic_time(millisecond) - T0, + ct:log("worker returned after ~p ms", [Elapsed]), + true = Elapsed >= 1000 + after 10000 -> + ct:fail(worker_never_returned) + end, + py_context:stop(C), + ok. + +%% @doc Signals blocked in the child: the soft interrupt cannot land and the +%% kill backstop fires after kill_after. +test_kill_backstop(Config) -> + C = new_ctx(Config, #{kill_after => 300}), + {ok, #{os_pid := Pid1}} = py_context:child_info(C), + Self = self(), + spawn_link(fun() -> + Self ! {result, py_context:call(C, ?TEST_MOD, blocked_sleep, [60])} + end), + timer:sleep(200), + T0 = erlang:monotonic_time(millisecond), + ok = py_context:interrupt(C), + receive + {result, {error, killed}} -> + Elapsed = erlang:monotonic_time(millisecond) - T0, + ct:log("killed after ~p ms", [Elapsed]), + true = Elapsed < 3000 + after 10000 -> + ct:fail(backstop_did_not_fire) + end, + false = os_pid_alive(Pid1), + {ok, #{os_pid := Pid2}} = py_context:child_info(C), + true = Pid1 =/= Pid2, + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +test_kill_restarts_child(Config) -> + C = new_ctx(Config), + ok = py_context:exec(C, <<"state = 'before'">>), + {ok, #{os_pid := Pid1}} = py_context:child_info(C), + ok = py_context:kill(C), + false = os_pid_alive(Pid1), + {ok, #{os_pid := Pid2}} = py_context:child_info(C), + true = Pid1 =/= Pid2, + %% State is gone, the context is not + {error, {'NameError', _}} = py_context:eval(C, <<"state">>), + {ok, 4} = py_context:eval(C, <<"2+2">>), + {error, not_isolated} = py_context:kill(self()), + stop(C). + +%% @doc The headline case: a segfault kills one child, the node and every +%% other context survive. +test_segfault_kills_only_child(Config) -> + C = new_ctx(Config), + Other = new_ctx(Config), + {ok, W} = py_context:new(#{mode => worker}), + {ok, #{os_pid := Pid1}} = py_context:child_info(C), + {error, {child_exited, {signal, Sig}}} = py_context:call(C, ?TEST_MOD, segfault, []), + true = is_segfault_signal(Sig), + false = os_pid_alive(Pid1), + true = is_process_alive(C), + {ok, #{os_pid := Pid2}} = py_context:child_info(C), + true = Pid2 =/= Pid1, + {ok, 4} = py_context:eval(C, <<"2+2">>), + {ok, 4} = py_context:eval(Other, <<"2+2">>), + {ok, 4} = py_context:eval(W, <<"2+2">>), + py_context:stop(W), + stop(Other), + stop(C). + +%% @doc `as' is enforced by the kernel (Linux, FreeBSD) or by the child's +%% RSS watchdog (macOS); either way the allocation fails or the child dies, +%% the node is unaffected and the context recovers. +test_rlimit_as(Config) -> + case sanitized_child() of + true -> {skip, "sanitizer runtime in the child needs unbounded address space"}; + false -> test_rlimit_as_1(Config) + end. + +test_rlimit_as_1(Config) -> + %% Free-threaded CPython reserves a large address range at startup, so + %% a limit that a regular build fits in keeps it from even importing + %% the socket module. + Probe = new_ctx(Config), + FreeThreaded = py_context:eval(Probe, + <<"hasattr(__import__('sys'), '_is_gil_enabled') and not __import__('sys')._is_gil_enabled()">>), + stop(Probe), + case FreeThreaded of + {ok, true} -> {skip, "free-threaded CPython needs an as limit far above test sizes"}; + _ -> test_rlimit_as_2(Config) + end. + +test_rlimit_as_2(Config) -> + C = new_ctx(Config, #{rlimits => #{as => 1024 * 1024 * 1024}}), + Result = py_context:call(C, ?TEST_MOD, allocate_and_touch, [2 * 1024 * 1024 * 1024], #{}, 120000), + ct:log("allocate past as limit: ~p", [Result]), + case {Result, rlimit_as_enforced()} of + {{error, {'MemoryError', _}}, _} -> ok; + {{error, {child_exited, {memory_limit, _}}}, false} -> ok; + {{error, {child_exited, _}}, true} -> ok; + Other -> ct:fail({unexpected, Other}) + end, + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +test_rlimit_cpu(Config) -> + C = new_ctx(Config, #{rlimits => #{cpu => 1}}), + Result = py_context:call(C, ?TEST_MOD, spin, [30], #{}, 20000), + ct:log("spin past RLIMIT_CPU: ~p", [Result]), + %% SIGXCPU (24 on Linux and BSD/macOS) + {error, {child_exited, {signal, Sig}}} = Result, + true = Sig =:= 24 orelse Sig =:= 30, + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +test_rlimit_nofile(Config) -> + C = new_ctx(Config, #{rlimits => #{nofile => 32}}), + {ok, 32} = py_context:eval(C, <<"__import__('resource').getrlimit(__import__('resource').RLIMIT_NOFILE)[0]">>), + stop(C). + +test_reaped_on_stop(Config) -> + C = new_ctx(Config), + {ok, #{os_pid := Pid}} = py_context:child_info(C), + ok = py_context:stop(C), + wait_gone(Pid), + ok. + +test_reaped_on_crash(Config) -> + C = new_ctx(Config, #{restart => false}), + {ok, #{os_pid := Pid}} = py_context:child_info(C), + unlink(C), + Mon = erlang:monitor(process, C), + _ = py_context:call(C, ?TEST_MOD, segfault, []), + receive {'DOWN', Mon, process, C, _} -> ok after 5000 -> ct:fail(context_survived) end, + wait_gone(Pid), + ok. + +test_reaped_on_kill(Config) -> + C = new_ctx(Config), + {ok, #{os_pid := Pid}} = py_context:child_info(C), + ok = py_context:kill(C), + wait_gone(Pid), + stop(C). + +%% @doc A child of another VM must not outlive that VM. +test_no_orphan_when_vm_dies(_Config) -> + case peer_available() of + false -> + {skip, "peer module not available"}; + true -> + %% standard_io works without distribution + {ok, Peer, _Node} = peer:start_link(#{ + connection => standard_io, + args => lists:append([["-pa", P] || P <- code:get_path()]) + }), + {ok, _} = peer:call(Peer, application, ensure_all_started, [erlang_python]), + {ok, C} = peer:call(Peer, py_context, new, [#{mode => isolated}]), + {ok, #{os_pid := Pid}} = peer:call(Peer, py_context, child_info, [C]), + true = os_pid_alive(Pid), + %% Park the child in a blocking C call so only the EOF watchdog + %% (or PDEATHSIG) can end it + ok = peer:cast(Peer, py_context, eval, [C, <<"__import__('time').sleep(60)">>]), + timer:sleep(300), + %% Hard stop: the peer VM is killed, nothing in it runs cleanup + peer:stop(Peer), + wait_gone(Pid), + ok + end. + +%% @doc The socket breaks under a pending call: it fails with a clear error, +%% the next call does not hang, and the restart recovers. +test_socket_break_mid_call(Config) -> + C = new_ctx(Config), + Result = py_context:call(C, ?TEST_MOD, close_control_socket, [], #{}, 10000), + ct:log("call under socket break: ~p", [Result]), + {error, {child_exited, _}} = Result, + T0 = erlang:monotonic_time(millisecond), + {ok, 4} = py_context:eval(C, <<"2+2">>, #{}, 5000), + true = erlang:monotonic_time(millisecond) - T0 < 3000, + stop(C). + +test_restart_false_exits_context(Config) -> + C = new_ctx(Config, #{restart => false}), + unlink(C), + Mon = erlang:monitor(process, C), + {error, {child_exited, {signal, Sig}}} = py_context:call(C, ?TEST_MOD, segfault, []), + true = is_segfault_signal(Sig), + receive + {'DOWN', Mon, process, C, {child_exited, {signal, Sig}}} -> ok + after 5000 -> + ct:fail(context_did_not_exit) + end, + {error, {context_died, _}} = py_context:eval(C, <<"1">>), + ok. + +test_restart_budget(Config) -> + C = new_ctx(Config, #{max_restarts => 2, restart_period => 60000}), + unlink(C), + Mon = erlang:monitor(process, C), + _ = py_context:call(C, ?TEST_MOD, segfault, []), + {ok, 4} = py_context:eval(C, <<"2+2">>), + _ = py_context:call(C, ?TEST_MOD, segfault, []), + {ok, 4} = py_context:eval(C, <<"2+2">>), + %% Third crash exceeds the budget + _ = py_context:call(C, ?TEST_MOD, segfault, []), + receive {'DOWN', Mon, process, C, _} -> ok after 5000 -> ct:fail(budget_not_enforced) end, + ok. + +test_numpy_imports(Config) -> + C = new_ctx(Config), + case py_context:eval(C, <<"__import__('importlib.util').util.find_spec('numpy') is not None">>) of + {ok, true} -> + {ok, 45} = py_context:call(C, ?TEST_MOD, numpy_sum, [10]), + {ok, [[1, 2], [3, 4]]} = py_context:eval(C, <<"__import__('numpy').array([[1,2],[3,4]])">>), + stop(C); + _ -> + stop(C), + {skip, "numpy not installed for the child interpreter"} + end. + +%% @doc A caller whose request is still queued times out: its request is +%% dropped, the request that is executing is not interrupted, and the kill +%% backstop does not fire because the context is busy. +test_queued_timeout_does_not_interrupt_others(Config) -> + C = new_ctx(Config, #{kill_after => 200}), + {ok, #{os_pid := Pid}} = py_context:child_info(C), + Self = self(), + %% Occupies the child for 1.5 s + spawn_link(fun() -> + Self ! {long, py_context:call(C, ?TEST_MOD, sleep_then, [1.5, done], #{}, 10000)} + end), + timer:sleep(100), + %% Queued behind it, gives up after 200 ms + {error, timeout} = py_context:eval(C, <<"'never'">>, #{}, 200), + receive + {long, R} -> {ok, <<"done">>} = R + after 5000 -> + ct:fail(long_call_lost) + end, + %% Same child, no kill happened, and the cancelled eval never ran + {ok, #{os_pid := Pid}} = py_context:child_info(C), + {ok, 4} = py_context:eval(C, <<"2+2">>, #{}, 5000), + stop(C). + +%% @doc The gen_statem state names what the context is doing. +test_sys_state_reflects_activity(Config) -> + C = new_ctx(Config), + {idle, _} = sys:get_state(C), + Self = self(), + spawn_link(fun() -> Self ! {done, py_context:call(C, ?TEST_MOD, sleep_then, [0.5, x])} end), + timer:sleep(100), + {{busy, Id}, _} = sys:get_state(C), + true = is_integer(Id), + receive {done, {ok, <<"x">>}} -> ok after 5000 -> ct:fail(no_reply) end, + {idle, _} = sys:get_state(C), + ok = py_context:start_loop(C), + {looping, _} = sys:get_state(C), + ok = py_context:stop_loop(C), + {idle, _} = sys:get_state(C), + stop(C). + +%% @doc A request arriving while the child restarts waits for the new child +%% instead of failing. +test_requests_during_restart_are_served(Config) -> + C = new_ctx(Config), + Self = self(), + Crasher = spawn_link(fun() -> Self ! {crash, py_context:call(C, ?TEST_MOD, segfault, [])} end), + %% Sent right behind the segfault: postponed through {restarting, _} + spawn_link(fun() -> Self ! {next, py_context:eval(C, <<"2+2">>, #{}, 10000)} end), + receive + {crash, {error, {child_exited, {signal, Sig}}}} -> true = is_segfault_signal(Sig) + after 10000 -> ct:fail(no_crash_report) + end, + receive {next, {ok, 4}} -> ok after 10000 -> ct:fail(request_not_served_after_restart) end, + _ = Crasher, + stop(C). + +%% @doc kill/1 answers once the new child is up, so the next call cannot +%% race the restart. +test_kill_reply_after_restart(Config) -> + C = new_ctx(Config), + {ok, #{os_pid := Pid1}} = py_context:child_info(C), + ok = py_context:kill(C), + {ok, #{os_pid := Pid2}} = py_context:child_info(C), + true = Pid1 =/= Pid2, + {idle, _} = sys:get_state(C), + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +%% @doc stop/1 from a third process while a call runs and while a loop +%% runs: nothing hangs, waiting callers get a reply or a DOWN. +test_stop_while_busy_and_looping(Config) -> + C1 = new_ctx(Config), + Self = self(), + spawn_link(fun() -> Self ! {busy, py_context:call(C1, ?TEST_MOD, sleep_then, [5, x], #{}, 10000)} end), + timer:sleep(100), + ok = py_context:stop(C1), + receive + {busy, {error, _}} -> ok + after 5000 -> ct:fail(busy_caller_hung) + end, + C2 = new_ctx(Config), + ok = py_context:start_loop(C2), + ok = py_context:stop(C2), + receive {py_loop_exit, C2, _} -> ok after 5000 -> ct:fail(no_loop_exit_on_stop) end, + false = is_process_alive(C2), + ok. + +test_sys_get_status(Config) -> + C = new_ctx(Config), + {status, C, {module, gen_statem}, _} = sys:get_status(C), + ok = sys:trace(C, true), + {ok, 4} = py_context:eval(C, <<"2+2">>), + ok = sys:trace(C, false), + stop(C). + +%% @doc The process that created the context may exit normally; the +%% context keeps serving (as embedded contexts do). +test_context_outlives_creator(Config) -> + Self = self(), + spawn(fun() -> Self ! {ctx, new_ctx(Config)} end), + C = receive {ctx, X} -> X after 15000 -> ct:fail(no_ctx) end, + timer:sleep(100), + true = is_process_alive(C), + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +test_not_supported_fail_loud(Config) -> + C = new_ctx(Config), + {error, not_supported_in_isolated} = py_context:call_method(C, make_ref(), <<"x">>, []), + {error, {'RuntimeError', Msg}} = py_context:eval(C, <<"__import__('erlang').schedule('x')">>), + true = string:find(to_list(Msg), "isolated") =/= nomatch, + {error, {'RuntimeError', _}} = py_context:eval(C, <<"__import__('erlang').Channel()">>), + {error, not_supported_in_isolated} = py_context:loop_ref(C), + stop(C). + +test_bad_python_fails_at_start(_Config) -> + {error, {python_not_found, _}} = py_context:new(#{mode => isolated, python => "/no/such/python"}), + {error, {child_exited_at_start, _, _}} = py_context:new(#{mode => isolated, python => "/bin/sh"}), + ok. + +test_startup_error_reported(_Config) -> + case sanitized_child() of + true -> {skip, "sanitizer runtime in the child needs unbounded address space"}; + false -> test_startup_error_reported_1() + end. + +test_startup_error_reported_1() -> + %% An impossible rlimit is reported, not silently ignored. The kernel + %% may kill the child before it connects (child_exited_at_start), the + %% child may report the failed setrlimit (startup_error), or the macOS + %% watchdog may end it at once; all are loud. + case py_context:new(#{mode => isolated, rlimits => #{as => 1}}) of + {error, {startup_error, _}} -> ok; + {error, {child_exited_at_start, _, _}} -> ok; + {error, {handshake_failed, _}} -> ok; + {ok, C} -> py_context:stop(C), ct:fail(limit_ignored); + Other -> ct:fail({unexpected, Other}) + end. + +%% @doc cgroups exist only on Linux: elsewhere the option is refused before +%% a child is spawned, and rlimits remain the way to bound the child. +test_cgroup_option_platform(_Config) -> + case os:type() of + {unix, linux} -> + %% A non-writable path is reported by the child + {error, {startup_error, [{cgroup, _}]}} = + py_context:new(#{mode => isolated, cgroup => "/nonexistent/cgroup"}), + ok; + {unix, Os} -> + {error, {cgroup_unsupported, Os}} = + py_context:new(#{mode => isolated, cgroup => "/sys/fs/cgroup/x"}), + %% Limits still apply without cgroups + {ok, C} = py_context:new(#{mode => isolated, rlimits => #{nofile => 48, cpu => 5}}), + {ok, 48} = py_context:eval(C, <<"__import__('resource').getrlimit(__import__('resource').RLIMIT_NOFILE)[0]">>), + {ok, 5} = py_context:eval(C, <<"__import__('resource').getrlimit(__import__('resource').RLIMIT_CPU)[0]">>), + stop(C) + end. + +test_env_option(Config) -> + C = new_ctx(Config, #{env => #{"PY_ISOLATED_PROBE" => "yes"}}), + {ok, <<"yes">>} = py_context:eval(C, <<"__import__('os').environ.get('PY_ISOLATED_PROBE')">>), + stop(C). + +test_preload_option(Config) -> + C = new_ctx(Config, #{preload => <<"preloaded = 'yes'">>}), + {ok, <<"yes">>} = py_context:eval(C, <<"preloaded">>), + stop(C). + +%%% ============================================================================ +%%% Helpers +%%% ============================================================================ + +new_ctx(Config) -> + new_ctx(Config, #{}). + +new_ctx(Config, Extra) -> + Mode = ?config(mode, Config), + TestDir = ?config(test_dir, Config), + Opts = maps:merge(#{mode => Mode, paths => [TestDir]}, Extra), + {ok, C} = py_context:new(Opts), + case Mode of + worker -> + ok = py_context:exec(C, iolist_to_binary(io_lib:format( + "import sys\nif '~s' not in sys.path: sys.path.insert(0, '~s')", + [TestDir, TestDir]))); + _ -> + ok + end, + C. + +stop(C) -> + ok = py_context:stop(C), + ok. + +test_dir() -> + filename:join(code:lib_dir(erlang_python), "test"). + +os_pid_alive(Pid) -> + case py_nif:os_kill(Pid, 0) of + ok -> + %% Alive, or a zombie: a zombie shows as Z in ps + case string:trim(os:cmd("ps -o stat= -p " ++ integer_to_list(Pid))) of + "" -> false; + "Z" ++ _ -> zombie; + _ -> true + end; + {error, esrch} -> + false; + {error, eperm} -> + true + end. + +wait_gone(Pid) -> + wait_gone(Pid, 50). + +wait_gone(Pid, 0) -> + ct:fail({child_still_present, Pid, os_pid_alive(Pid)}); +wait_gone(Pid, N) -> + case os_pid_alive(Pid) of + false -> ok; + _ -> timer:sleep(100), wait_gone(Pid, N - 1) + end. + +%% A sanitizer runtime (LD_PRELOAD=libasan in the ASan job) is inherited +%% by the child: it aborts on a segfault and reserves terabytes of address +%% space, so rlimit cases cannot mean anything there. +sanitized_child() -> + Pre = case os:getenv("LD_PRELOAD") of false -> ""; P -> P end, + string:find(Pre, "asan") =/= nomatch orelse os:getenv("ASAN_OPTIONS") =/= false. + +is_segfault_signal(11) -> true; +is_segfault_signal(6) -> sanitized_child(); +is_segfault_signal(_) -> false. + +rlimit_as_enforced() -> + case os:type() of + {unix, linux} -> true; + {unix, freebsd} -> true; + _ -> false + end. + +peer_available() -> + code:ensure_loaded(peer) =:= {module, peer}. + +to_list(B) when is_binary(B) -> binary_to_list(B); +to_list(L) when is_list(L) -> L. + +flush() -> + receive _ -> flush() after 0 -> ok end. diff --git a/test/py_isolated_async_SUITE.erl b/test/py_isolated_async_SUITE.erl new file mode 100644 index 0000000..02ad05c --- /dev/null +++ b/test/py_isolated_async_SUITE.erl @@ -0,0 +1,516 @@ +%%% @doc Common Test suite: asyncio in an isolated context. +%%% +%%% The child runs a plain asyncio loop. This suite mirrors +%%% py_worker_loop_SUITE (start_loop/submit/stop_loop, serving on fds Erlang +%%% owns) and the coroutine cases of py_async_task_SUITE. The headline case, +%%% a loop wedged in a blocking C call that stop_loop/2 kills, has a worker +%%% group counterpart documenting that the embedded loop cannot be stopped +%%% that way. +-module(py_isolated_async_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([ + all/0, + groups/0, + init_per_suite/1, + end_per_suite/1, + init_per_group/2, + end_per_group/2, + end_per_testcase/2 +]). + +-export([ + test_start_stop_loop/1, + test_start_twice/1, + test_stop_idle/1, + test_submit_idle_and_running/1, + test_submit_ordering/1, + test_submit_errors_reported/1, + test_calls_rejected_while_running/1, + test_interrupt_ends_loop/1, + test_owner_death_stops_loop/1, + test_stop_context_while_looping/1, + test_long_submitted_call/1, + test_preload_before_loop/1, + test_tcp_serve_on_passed_fd/1, + test_udp_serve_on_passed_fd/1, + test_adopt_accepted_fd/1, + test_three_workers_one_listen_fd/1, + test_pass_fd_invalid/1, + test_call_awaits_coroutine/1, + test_gather_is_concurrent/1, + test_async_error/1, + test_concurrent_submitted_tasks/1, + test_large_async_result/1, + test_async_call_in_coroutine/1, + test_async_calls_concurrent/1, + test_async_call_error/1, + test_send_from_coroutine/1, + test_run_helper_compat/1, + test_stream_via_send/1, + test_blocked_loop_is_killed/1, + test_blocked_loop_survives_in_worker/1 +]). + +-define(TEST_MOD, py_test_isolated). +-define(HOST, {127, 0, 0, 1}). + +all() -> + [{group, isolated}, {group, worker_contrast}]. + +groups() -> + Cases = [ + test_start_stop_loop, + test_start_twice, + test_stop_idle, + test_submit_idle_and_running, + test_submit_ordering, + test_submit_errors_reported, + test_calls_rejected_while_running, + test_interrupt_ends_loop, + test_owner_death_stops_loop, + test_stop_context_while_looping, + test_long_submitted_call, + test_preload_before_loop, + test_tcp_serve_on_passed_fd, + test_udp_serve_on_passed_fd, + test_adopt_accepted_fd, + test_three_workers_one_listen_fd, + test_pass_fd_invalid, + test_call_awaits_coroutine, + test_gather_is_concurrent, + test_async_error, + test_concurrent_submitted_tasks, + test_large_async_result, + test_async_call_in_coroutine, + test_async_calls_concurrent, + test_async_call_error, + test_send_from_coroutine, + test_run_helper_compat, + test_stream_via_send, + test_blocked_loop_is_killed + ], + [{isolated, [], Cases}, + {worker_contrast, [], [test_blocked_loop_survives_in_worker]}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(erlang_python), + [{test_dir, filename:join(code:lib_dir(erlang_python), "test")} | Config]. + +end_per_suite(_Config) -> + ok = application:stop(erlang_python), + ok. + +init_per_group(worker_contrast, Config) -> + [{mode, worker} | Config]; +init_per_group(Mode, Config) -> + [{mode, Mode} | Config]. + +end_per_group(_Group, _Config) -> + ok. + +end_per_testcase(_TestCase, _Config) -> + flush(), + ok. + +%%% ============================================================================ +%%% Worker loop lifecycle +%%% ============================================================================ + +test_start_stop_loop(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {ok, 3} = py_context:submit_await(C, ?TEST_MOD, async_add, [1, 2]), + ok = py_context:stop_loop(C), + receive {py_loop_exit, C, ok} -> ok after 2000 -> ct:fail(no_loop_exit) end, + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +test_start_twice(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {error, already_running} = py_context:start_loop(C), + ok = py_context:stop_loop(C), + stop(C). + +test_stop_idle(Config) -> + C = new_ctx(Config), + {error, no_loop} = py_context:stop_loop(C), + stop(C). + +test_submit_idle_and_running(Config) -> + C = new_ctx(Config), + %% Without a running loop, submit reports no loop (the embedded modes + %% step the loop through the event worker; the child has no such thing) + {error, no_loop} = py_context:submit_await(C, ?TEST_MOD, async_add, [1, 2]), + ok = py_context:start_loop(C), + {ok, 3} = py_context:submit_await(C, ?TEST_MOD, async_add, [1, 2]), + {ok, 7} = py_context:submit_await(C, ?TEST_MOD, add, [3, 4]), + ok = py_context:stop_loop(C), + stop(C). + +test_submit_ordering(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + Refs = [begin + {ok, R} = py_context:submit(C, ?TEST_MOD, async_add, [I, 0]), + {I, R} + end || I <- lists:seq(1, 100)], + lists:foreach(fun({I, R}) -> + {ok, I} = py_event_loop:await(R, 5000) + end, Refs), + ok = py_context:stop_loop(C), + stop(C). + +test_submit_errors_reported(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {error, {'ModuleNotFoundError', _}} = py_context:submit_await(C, no_such_mod_xyz, f, []), + {error, {'AttributeError', _}} = py_context:submit_await(C, ?TEST_MOD, no_such_fn, []), + {error, {'KeyError', _}} = py_context:submit_await(C, ?TEST_MOD, async_raise, [<<"k">>]), + {error, {'ValueError', _}} = py_context:submit_await(C, ?TEST_MOD, raise_value_error, [<<"v">>]), + ok = py_context:stop_loop(C), + stop(C). + +test_calls_rejected_while_running(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {error, loop_running} = py_context:eval(C, <<"1">>), + {error, loop_running} = py_context:exec(C, <<"x = 1">>), + {error, loop_running} = py_context:call(C, math, sqrt, [4]), + ok = py_context:stop_loop(C), + {ok, 1} = py_context:eval(C, <<"1">>), + stop(C). + +test_interrupt_ends_loop(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + ok = py_context:interrupt(C), + receive {py_loop_exit, C, {error, interrupted}} -> ok + after 3000 -> ct:fail(loop_not_interrupted) + end, + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +test_owner_death_stops_loop(Config) -> + C = new_ctx(Config), + Owner = spawn(fun() -> receive die -> ok end end), + ok = py_context:start_loop(C, #{owner => Owner}), + Owner ! die, + wait_until(fun() -> py_context:eval(C, <<"1">>) =:= {ok, 1} end, 5000), + stop(C). + +test_stop_context_while_looping(Config) -> + C = new_ctx(Config), + {ok, #{os_pid := Pid}} = py_context:child_info(C), + ok = py_context:start_loop(C), + ok = py_context:stop(C), + wait_until(fun() -> py_nif:os_kill(Pid, 0) =:= {error, esrch} end, 5000), + ok. + +%% @doc No 30 s cap on a submitted call (the pipe write deadline of the +%% embedded modes does not apply): a 2 s task completes, and the loop keeps +%% answering meanwhile. +test_long_submitted_call(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {ok, R} = py_context:submit(C, ?TEST_MOD, slow_task, [2]), + {ok, 3} = py_context:submit_await(C, ?TEST_MOD, async_add, [1, 2]), + {ok, <<"slow_done">>} = py_event_loop:await(R, 10000), + ok = py_context:stop_loop(C), + stop(C). + +test_preload_before_loop(Config) -> + C = new_ctx(Config, #{preload => <<"import py_test_isolated\npy_test_isolated.counter_increment(5)">>}), + ok = py_context:start_loop(C), + {ok, 5} = py_context:submit_await(C, ?TEST_MOD, counter_value, []), + ok = py_context:stop_loop(C), + stop(C). + +%%% ============================================================================ +%%% Serving on fds Erlang owns +%%% ============================================================================ + +test_tcp_serve_on_passed_fd(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {LSock, Port, ChildFd} = listen_pass(C), + {ok, <<"serving">>} = py_context:submit_await(C, py_test_workerloop, serve, [ChildFd]), + [<<"ok:x">> = roundtrip(Port, <<"x">>) || _ <- lists:seq(1, 100)], + {ok, 100} = py_context:submit_await(C, py_test_workerloop, served_count, []), + {ok, <<"stopped">>} = py_context:submit_await(C, py_test_workerloop, stop, [ChildFd]), + ok = py_context:stop_loop(C), + gen_tcp:close(LSock), + stop(C). + +test_udp_serve_on_passed_fd(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {ok, USock} = gen_udp:open(0, [binary, {ip, ?HOST}, {active, false}]), + {ok, Port} = inet:port(USock), + {ok, Fd} = inet:getfd(USock), + {ok, ChildFd} = py_context:pass_fd(C, Fd), + {ok, <<"serving">>} = py_context:submit_await(C, py_test_workerloop, serve_udp, [ChildFd]), + {ok, Client} = gen_udp:open(0, [binary, {ip, ?HOST}, {active, false}]), + ok = gen_udp:send(Client, ?HOST, Port, <<"ping">>), + {ok, {_, _, <<"udp:ping">>}} = gen_udp:recv(Client, 0, 2000), + {ok, <<"stopped">>} = py_context:submit_await(C, py_test_workerloop, stop, [ChildFd]), + gen_udp:close(Client), + gen_udp:close(USock), + ok = py_context:stop_loop(C), + stop(C). + +test_adopt_accepted_fd(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {ok, LSock} = gen_tcp:listen(0, [binary, {ip, ?HOST}, {active, false}]), + {ok, Port} = inet:port(LSock), + Self = self(), + spawn_link(fun() -> + {ok, S} = gen_tcp:connect(?HOST, Port, [binary, {active, false}], 2000), + ok = gen_tcp:send(S, <<"adopted?">>), + Self ! {client, gen_tcp:recv(S, 0, 3000)}, + gen_tcp:close(S) + end), + {ok, Conn} = gen_tcp:accept(LSock, 2000), + {ok, ConnFd} = inet:getfd(Conn), + {ok, ChildFd} = py_context:pass_fd(C, ConnFd), + {ok, <<"adopted">>} = py_context:submit_await(C, py_test_workerloop, adopt, [ChildFd]), + gen_tcp:close(Conn), + receive {client, {ok, <<"ok:adopted?">>}} -> ok + after 3000 -> ct:fail(no_reply_through_adopted_fd) + end, + gen_tcp:close(LSock), + ok = py_context:stop_loop(C), + stop(C). + +%% @doc gunicorn shape, out of process: one listen socket, three child +%% processes accepting on their copy of it. +test_three_workers_one_listen_fd(Config) -> + Ctxs = [new_ctx(Config) || _ <- lists:seq(1, 3)], + {ok, LSock} = gen_tcp:listen(0, [binary, {ip, ?HOST}, {active, false}, {backlog, 512}]), + {ok, Port} = inet:port(LSock), + {ok, LFd} = inet:getfd(LSock), + lists:foreach(fun({I, C}) -> + ok = py_context:start_loop(C), + {ok, ChildFd} = py_context:pass_fd(C, LFd), + Tag = list_to_binary("w" ++ integer_to_list(I) ++ ":"), + {ok, <<"serving">>} = py_context:submit_await(C, py_test_workerloop, serve, [ChildFd, Tag]) + end, lists:zip(lists:seq(1, 3), Ctxs)), + Replies = [roundtrip(Port, <<"x">>) || _ <- lists:seq(1, 300)], + 300 = length([R || R <- Replies, binary:part(R, byte_size(R) - 4, 4) =:= <<"ok:x">>]), + Tags = lists:usort([binary:part(R, 0, 3) || R <- Replies]), + ct:log("workers that served: ~p", [Tags]), + %% Which child wins accept() is up to the kernel; a fast worker can + %% starve another over 300 connections. Two distinct workers prove + %% the socket is shared. + true = length(Tags) >= 2, + [ok = py_context:stop_loop(C) || C <- Ctxs], + [stop(C) || C <- Ctxs], + gen_tcp:close(LSock), + ok. + +test_pass_fd_invalid(Config) -> + C = new_ctx(Config), + {error, _} = py_context:pass_fd(C, 123456), + {error, {invalid_fd, -1}} = py_context:pass_fd(C, -1), + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +%%% ============================================================================ +%%% Coroutines +%%% ============================================================================ + +test_call_awaits_coroutine(Config) -> + C = new_ctx(Config), + {ok, 3} = py_context:call(C, ?TEST_MOD, async_add, [1, 2]), + ok = py_context:exec(C, <<"import py_test_isolated">>), + {ok, 3} = py_context:eval(C, <<"py_test_isolated.async_add(a, b)">>, #{a => 1, b => 2}), + stop(C). + +test_gather_is_concurrent(Config) -> + C = new_ctx(Config), + T0 = erlang:monotonic_time(millisecond), + {ok, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]} = py_context:call(C, ?TEST_MOD, async_sleep_gather, [10, 0.1]), + Elapsed = erlang:monotonic_time(millisecond) - T0, + ct:log("10 x sleep(0.1) gathered in ~p ms", [Elapsed]), + true = Elapsed < 600, + stop(C). + +test_async_error(Config) -> + C = new_ctx(Config), + {error, {'KeyError', _}} = py_context:call(C, ?TEST_MOD, async_raise, [<<"nope">>]), + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +test_concurrent_submitted_tasks(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + Refs = [begin {ok, R} = py_context:submit(C, ?TEST_MOD, task_value, [I]), {I, R} end + || I <- lists:seq(1, 100)], + lists:foreach(fun({I, R}) -> + Expected = I * I, + {ok, Expected} = py_event_loop:await(R, 10000) + end, Refs), + ok = py_context:stop_loop(C), + stop(C). + +test_large_async_result(Config) -> + C = new_ctx(Config), + Size = 16 * 1024 * 1024, + {ok, Bin} = py_context:call(C, ?TEST_MOD, async_big, [Size]), + Size = byte_size(Bin), + stop(C). + +test_async_call_in_coroutine(Config) -> + C = new_ctx(Config), + py_callback:register(<<"as_double">>, fun([X]) -> X * 2 end), + {ok, 84} = py_context:call(C, ?TEST_MOD, async_erlang_call, [<<"as_double">>, 42]), + py_callback:unregister(<<"as_double">>), + stop(C). + +test_async_calls_concurrent(Config) -> + C = new_ctx(Config), + py_callback:register(<<"as_double">>, fun([X]) -> X * 2 end), + Expected = [I * 2 || I <- lists:seq(0, 99)], + {ok, Expected} = py_context:call(C, ?TEST_MOD, async_erlang_calls, [<<"as_double">>, 100]), + py_callback:unregister(<<"as_double">>), + stop(C). + +test_async_call_error(Config) -> + C = new_ctx(Config), + py_callback:register(<<"as_fail">>, fun(_) -> error(deliberate) end), + {ok, <<"RuntimeError">>} = py_context:call(C, ?TEST_MOD, async_erlang_call_error, [<<"as_fail">>]), + py_callback:unregister(<<"as_fail">>), + stop(C). + +test_send_from_coroutine(Config) -> + C = new_ctx(Config), + {ok, <<"sent">>} = py_context:call(C, ?TEST_MOD, async_send, [self(), coro_msg]), + receive <<"coro_msg">> -> ok after 2000 -> ct:fail(no_message) end, + stop(C). + +test_run_helper_compat(Config) -> + C = new_ctx(Config), + {ok, 3} = py_context:call(C, ?TEST_MOD, run_helper_compat, []), + stop(C). + +%% @doc Streaming out of an isolated context: a submitted coroutine pushes +%% items with erlang.send; order and the done marker are asserted. +test_stream_via_send(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {ok, R} = py_context:submit(C, ?TEST_MOD, stream_to, [self(), 1000]), + Items = collect_items([]), + Expected = lists:seq(0, 999), + Expected = Items, + {ok, 1000} = py_event_loop:await(R, 5000), + ok = py_context:stop_loop(C), + stop(C). + +%%% ============================================================================ +%%% The headline async case +%%% ============================================================================ + +%% @doc A coroutine wedged in time.sleep inside the loop: stop_loop/2 asks, +%% interrupts, then kills. The context is usable right after. +test_blocked_loop_is_killed(Config) -> + C = new_ctx(Config, #{kill_after => 500}), + {ok, #{os_pid := Pid1}} = py_context:child_info(C), + ok = py_context:start_loop(C), + %% The interrupt signal reaches time.sleep; block it so only the + %% backstop can end the loop + {ok, _} = py_context:submit(C, ?TEST_MOD, blocked_sleep, [60]), + timer:sleep(300), + T0 = erlang:monotonic_time(millisecond), + Result = py_context:stop_loop(C, 300), + Elapsed = erlang:monotonic_time(millisecond) - T0, + ct:log("stop_loop on a wedged loop: ~p after ~p ms", [Result, Elapsed]), + ok = Result, + true = Elapsed < 5000, + receive {py_loop_exit, C, _} -> ok after 2000 -> ct:fail(no_loop_exit) end, + {ok, #{os_pid := Pid2}} = py_context:child_info(C), + true = Pid1 =/= Pid2, + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +%% @doc Contrast: the embedded loop cannot be killed; a wedged loop makes +%% stop_loop/2 time out and the sleep runs to completion. +test_blocked_loop_survives_in_worker(Config) -> + C = new_ctx(Config), + ok = py_context:start_loop(C), + {ok, _} = py_context:submit(C, py_test_workerloop, block_loop, [4]), + timer:sleep(300), + Result = py_context:stop_loop(C, 300), + ct:log("worker stop_loop on a wedged loop: ~p", [Result]), + {error, timeout} = Result, + %% Wait out the sleep so the context can be stopped cleanly + timer:sleep(4500), + _ = (try py_context:stop(C) catch _:_ -> ok end), + ok. + +%%% ============================================================================ +%%% Helpers +%%% ============================================================================ + +listen_pass(C) -> + {ok, LSock} = gen_tcp:listen(0, [binary, {ip, ?HOST}, {active, false}, {backlog, 128}]), + {ok, Port} = inet:port(LSock), + {ok, Fd} = inet:getfd(LSock), + {ok, ChildFd} = py_context:pass_fd(C, Fd), + {LSock, Port, ChildFd}. + +roundtrip(Port, Data) -> + {ok, S} = gen_tcp:connect(?HOST, Port, [binary, {active, false}], 2000), + ok = gen_tcp:send(S, Data), + {ok, Reply} = gen_tcp:recv(S, 0, 3000), + gen_tcp:close(S), + Reply. + +collect_items(Acc) -> + receive + {<<"item">>, I} -> collect_items([I | Acc]); + <<"done">> -> lists:reverse(Acc) + after 5000 -> + ct:fail({incomplete, length(Acc)}) + end. + +wait_until(Fun, TimeoutMs) -> + Deadline = erlang:monotonic_time(millisecond) + TimeoutMs, + wait_until_loop(Fun, Deadline). + +wait_until_loop(Fun, Deadline) -> + case Fun() of + true -> ok; + _ -> + case erlang:monotonic_time(millisecond) > Deadline of + true -> ct:fail(condition_not_met); + false -> timer:sleep(50), wait_until_loop(Fun, Deadline) + end + end. + +new_ctx(Config) -> + new_ctx(Config, #{}). + +new_ctx(Config, Extra) -> + Mode = ?config(mode, Config), + TestDir = ?config(test_dir, Config), + Opts = maps:merge(#{mode => Mode, paths => [TestDir]}, Extra), + {ok, C} = py_context:new(Opts), + case Mode of + worker -> + ok = py_context:exec(C, iolist_to_binary(io_lib:format( + "import sys\nif '~s' not in sys.path: sys.path.insert(0, '~s')", + [TestDir, TestDir]))); + _ -> + ok + end, + C. + +stop(C) -> + ok = py_context:stop(C), + ok. + +flush() -> + receive _ -> flush() after 0 -> ok end. diff --git a/test/py_isolated_soak_SUITE.erl b/test/py_isolated_soak_SUITE.erl new file mode 100644 index 0000000..76fab9d --- /dev/null +++ b/test/py_isolated_soak_SUITE.erl @@ -0,0 +1,277 @@ +%%% @doc Soak test for `isolated' mode: a random mix of everything the mode +%%% offers, run for a while, with resource counters checked before and +%%% after. The point is to show no deadlock (every operation returns), no +%%% runaway loop (the mix keeps making progress) and no leak (Erlang +%%% processes, ports, ETS entries, memory, VM file descriptors and OS +%%% children return to their baseline). +%%% +%%% Duration is 60 s by default; set `PY_ISOLATED_SOAK_SECONDS' to change it. +-module(py_isolated_soak_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([all/0, init_per_suite/1, end_per_suite/1]). +-export([ + test_mixed_workload_no_leak/1, + test_callback_storm_no_deadlock/1, + test_interrupt_kill_storm/1, + test_loop_start_stop_churn/1 +]). + +-define(TEST_MOD, py_test_isolated). + +all() -> [ + test_callback_storm_no_deadlock, + test_interrupt_kill_storm, + test_loop_start_stop_churn, + test_mixed_workload_no_leak +]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(erlang_python), + py_callback:register(<<"soak_echo">>, fun([X]) -> X end), + py_callback:register(<<"soak_incr">>, fun([X]) -> X + 1 end), + py_callback:register(<<"soak_fail">>, fun(_) -> error(deliberate) end), + py_callback:register(<<"soak_tid">>, fun([T, I]) -> T * 1000 + I end), + [{test_dir, filename:join(code:lib_dir(erlang_python), "test")} | Config]. + +end_per_suite(_Config) -> + py_callback:unregister(<<"soak_echo">>), + py_callback:unregister(<<"soak_incr">>), + py_callback:unregister(<<"soak_fail">>), + py_callback:unregister(<<"soak_tid">>), + ok = application:stop(erlang_python), + ok. + +%% @doc Many Erlang callers, each mixing plain calls, callbacks, nested +%% callbacks, thread-pool callbacks and callback errors on a shared context. +%% Every call must return within its timeout. +test_callback_storm_no_deadlock(Config) -> + C = new_ctx(Config), + py_callback:register(<<"soak_nested">>, fun([X]) -> + {ok, R} = py_context:call(C, ?TEST_MOD, add, [X, 1], #{}, 20000), R + end), + Self = self(), + Workers = 16, + Rounds = 40, + [spawn_link(fun() -> Self ! {done, I, storm(C, I, Rounds, [])} end) || I <- lists:seq(1, Workers)], + Failures = lists:append([receive {done, _, F} -> F after 120000 -> ct:fail(worker_hung) end + || _ <- lists:seq(1, Workers)]), + ct:log("failures: ~p", [Failures]), + [] = Failures, + py_callback:unregister(<<"soak_nested">>), + ok = py_context:stop(C), + ok. + +storm(_C, _I, 0, Acc) -> + Acc; +storm(C, I, N, Acc) -> + Op = (I + N) rem 6, + R = case Op of + 0 -> py_context:call(C, ?TEST_MOD, add, [I, N], #{}, 20000); + 1 -> py_context:call(C, ?TEST_MOD, callback, [<<"soak_echo">>, {I, N}], #{}, 20000); + 2 -> py_context:call(C, ?TEST_MOD, callback, [<<"soak_nested">>, N], #{}, 20000); + 3 -> py_context:call(C, ?TEST_MOD, pool_calls, [<<"soak_incr">>, 4, 20], #{}, 20000); + 4 -> py_context:call(C, ?TEST_MOD, callback_error_type, [<<"soak_fail">>], #{}, 20000); + 5 -> py_context:call(C, ?TEST_MOD, thread_calls, [<<"soak_tid">>, 4, 5], #{}, 20000) + end, + Expected = case Op of + 0 -> {ok, I + N}; + 1 -> {ok, {I, N}}; + 2 -> {ok, N + 1}; + 3 -> {ok, false}; %% incr, not double: the helper compares to i*2 + 4 -> {ok, <<"RuntimeError">>}; + 5 -> {ok, {<<"ok">>, true, 20}} + end, + Acc1 = case R of + Expected -> Acc; + {ok, _} when Op =:= 3 -> Acc; %% value checked by shape only + Other -> [{I, N, Op, Other} | Acc] + end, + storm(C, I, N - 1, Acc1). + +%% @doc Interrupts and kills racing with calls: nothing hangs, the context +%% always answers again, and no child is left behind. +test_interrupt_kill_storm(Config) -> + C = new_ctx(Config, #{kill_after => 200, max_restarts => 1000}), + Pids = lists:map(fun(N) -> + {ok, #{os_pid := P}} = py_context:child_info(C), + Self = self(), + spawn_link(fun() -> + Self ! {res, py_context:call(C, ?TEST_MOD, sleep_then, [5, N], #{}, 30000)} + end), + timer:sleep(20 + N rem 30), + case N rem 3 of + 0 -> py_context:interrupt(C); + 1 -> py_context:kill(C); + 2 -> py_context:interrupt(C), py_context:kill(C) + end, + receive {res, R} -> + case R of + {error, interrupted} -> ok; + {error, killed} -> ok; + {error, {child_exited, _}} -> ok; + {ok, N} -> ok; + Other -> ct:fail({unexpected, N, Other}) + end + after 15000 -> ct:fail({hung_after_interrupt, N}) + end, + {ok, 4} = py_context:eval(C, <<"2+2">>, #{}, 30000), + P + end, lists:seq(1, 30)), + ok = py_context:stop(C), + timer:sleep(300), + Alive = [P || P <- lists:usort(Pids), py_nif:os_kill(P, 0) =:= ok], + [] = Alive, + ok. + +%% @doc start_loop / submit / stop_loop repeated, with a wedged loop every +%% few rounds so the kill backstop runs. +test_loop_start_stop_churn(Config) -> + C = new_ctx(Config, #{kill_after => 200, max_restarts => 1000}), + lists:foreach(fun(N) -> + ok = py_context:start_loop(C), + {ok, 3} = py_context:submit_await(C, ?TEST_MOD, async_add, [1, 2], #{}, 30000), + case N rem 4 of + 0 -> + {ok, _} = py_context:submit(C, ?TEST_MOD, blocked_sleep, [30]), + timer:sleep(50), + ok = py_context:stop_loop(C, 100); + _ -> + ok = py_context:stop_loop(C, 2000) + end, + receive {py_loop_exit, C, _} -> ok after 5000 -> ct:fail({no_loop_exit, N}) end, + {ok, 4} = py_context:eval(C, <<"2+2">>, #{}, 30000) + end, lists:seq(1, 24)), + ok = py_context:stop(C), + ok. + +%% @doc The long one: contexts started and stopped, calls with payloads, +%% callbacks, coroutines, interrupts, crashes, for a fixed duration. +%% Counters must return to baseline. +test_mixed_workload_no_leak(Config) -> + Seconds = list_to_integer(os:getenv("PY_ISOLATED_SOAK_SECONDS", "60")), + %% Warm up so lazily created resources are in the baseline + Warm = new_ctx(Config), + {ok, _} = py_context:call(Warm, ?TEST_MOD, callback, [<<"soak_echo">>, 1]), + ok = py_context:stop(Warm), + timer:sleep(500), + erlang:garbage_collect(), + Base = counters(), + ct:log("baseline: ~p", [Base]), + Deadline = erlang:monotonic_time(millisecond) + Seconds * 1000, + Self = self(), + Workers = [spawn_link(fun() -> Self ! {worker, I, mixed(Config, I, Deadline, 0, [])} end) + || I <- lists:seq(1, 6)], + Stats = [receive {worker, _, S} -> S after (Seconds + 120) * 1000 -> ct:fail(worker_hung) end + || _ <- Workers], + Ops = lists:sum([O || {O, _} <- Stats]), + Errs = lists:append([E || {_, E} <- Stats]), + ct:log("ops: ~p, unexpected errors: ~p", [Ops, lists:sublist(Errs, 20)]), + ct:print("soak: ~p ops in ~p s, ~p unexpected errors", [Ops, Seconds, length(Errs)]), + true = Ops > 0, + [] = Errs, + timer:sleep(1000), + erlang:garbage_collect(), + After = counters(), + ct:log("after: ~p", [After]), + check_no_growth(Base, After), + ok. + +mixed(Config, I, Deadline, Ops, Errs) -> + case erlang:monotonic_time(millisecond) > Deadline of + true -> {Ops, Errs}; + false -> + C = new_ctx(Config, #{kill_after => 300, max_restarts => 1000}), + E1 = mixed_ops(C, I, 25, Errs), + ok = py_context:stop(C), + mixed(Config, I, Deadline, Ops + 25, E1) + end. + +mixed_ops(_C, _I, 0, Errs) -> + Errs; +mixed_ops(C, I, N, Errs) -> + Op = (I * 7 + N) rem 9, + R = case Op of + 0 -> py_context:eval(C, <<"sum(range(1000))">>, #{}, 30000); + 1 -> py_context:call(C, ?TEST_MOD, identity, [crypto:strong_rand_bytes(256 * 1024)], #{}, 30000); + 2 -> py_context:call(C, ?TEST_MOD, callback, [<<"soak_echo">>, [I, N]], #{}, 30000); + 3 -> py_context:call(C, ?TEST_MOD, async_sleep_gather, [5, 0.001], #{}, 30000); + 4 -> py_context:eval(C, <<"__import__('time').sleep(5)">>, #{}, 50); + 5 -> py_context:call(C, ?TEST_MOD, pool_calls, [<<"soak_echo">>, 4, 10], #{}, 30000); + 6 -> py_context:call(C, ?TEST_MOD, segfault, [], #{}, 30000); + 7 -> py_context:call(C, ?TEST_MOD, send, [self(), {soak, N}], #{}, 30000); + 8 -> py_context:kill(C) + end, + Ok = case {Op, R} of + {0, {ok, 499500}} -> true; + {1, {ok, B}} when is_binary(B) -> true; + {2, {ok, [I, N]}} -> true; + {3, {ok, [0, 1, 2, 3, 4]}} -> true; + {4, {error, timeout}} -> true; + {5, {ok, _}} -> true; + {6, {error, {child_exited, {signal, _}}}} -> true; + {7, {ok, true}} -> receive {<<"soak">>, N} -> true after 5000 -> false end; + {8, ok} -> true; + _ -> false + end, + %% After any op the context must answer + Alive = py_context:eval(C, <<"1">>, #{}, 30000) =:= {ok, 1}, + Errs1 = case Ok andalso Alive of + true -> Errs; + false -> [{op, Op, R, alive, Alive} | Errs] + end, + mixed_ops(C, I, N - 1, Errs1). + +%%% ============================================================================ +%%% Counters +%%% ============================================================================ + +counters() -> + #{ + processes => erlang:system_info(process_count), + ports => erlang:system_info(port_count), + refs => ets:info(py_context_refs, size), + memory_mb => erlang:memory(total) div (1024 * 1024), + binary_mb => erlang:memory(binary) div (1024 * 1024), + fds => beam_fd_count(), + children => child_count() + }. + +check_no_growth(Base, After) -> + Same = [processes, ports, refs, children], + lists:foreach(fun(K) -> + B = maps:get(K, Base), A = maps:get(K, After), + A =< B + 2 orelse ct:fail({leak, K, B, A}) + end, Same), + %% fds: allow a few for CT's own logging + maps:get(fds, After) =< maps:get(fds, Base) + 8 orelse + ct:fail({fd_leak, maps:get(fds, Base), maps:get(fds, After)}), + %% memory: within 64 MB of baseline after GC + maps:get(memory_mb, After) =< maps:get(memory_mb, Base) + 64 orelse + ct:fail({memory_growth, maps:get(memory_mb, Base), maps:get(memory_mb, After)}), + ok. + +beam_fd_count() -> + case os:type() of + {unix, linux} -> + length(filelib:wildcard("/proc/" ++ os:getpid() ++ "/fd/*")); + _ -> + Out = os:cmd("lsof -p " ++ os:getpid() ++ " 2>/dev/null | wc -l"), + list_to_integer(string:trim(Out)) - 1 + end. + +child_count() -> + Out = os:cmd("ps -ax -o ppid=,command= 2>/dev/null | grep py_isolated_child | grep -v grep | grep -c ' " ++ os:getpid() ++ " ' "), + case string:trim(Out) of + "" -> 0; + N -> list_to_integer(N) + end. + +new_ctx(Config) -> + new_ctx(Config, #{}). + +new_ctx(Config, Extra) -> + TestDir = ?config(test_dir, Config), + {ok, C} = py_context:new(maps:merge(#{mode => isolated, paths => [TestDir]}, Extra)), + C. diff --git a/test/py_isolated_stress_SUITE.erl b/test/py_isolated_stress_SUITE.erl new file mode 100644 index 0000000..92fbc2d --- /dev/null +++ b/test/py_isolated_stress_SUITE.erl @@ -0,0 +1,162 @@ +%%% @doc Stress and profiling for `isolated' mode. +%%% +%%% Numbers are logged, not asserted tightly: the point is to show the cost +%%% of the process boundary next to worker mode on the same machine, and +%%% that churn does not leak OS processes or memory. +-module(py_isolated_stress_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([all/0, init_per_suite/1, end_per_suite/1]). + +-export([ + test_call_latency_vs_worker/1, + test_callback_round_trips/1, + test_context_churn_no_leak/1, + test_startup_time/1, + test_payload_throughput/1, + test_parallel_contexts_cpu_bound/1 +]). + +all() -> [ + test_call_latency_vs_worker, + test_callback_round_trips, + test_context_churn_no_leak, + test_startup_time, + test_payload_throughput, + test_parallel_contexts_cpu_bound +]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(erlang_python), + Config. + +end_per_suite(_Config) -> + ok = application:stop(erlang_python), + ok. + +%% @doc 10k sequential evals per mode; p50/p99 per call. +test_call_latency_vs_worker(_Config) -> + N = 10000, + {ok, I} = py_context:new(#{mode => isolated}), + {ok, W} = py_context:new(#{mode => worker}), + {ok, 2} = py_context:eval(I, <<"1+1">>), + {ok, 2} = py_context:eval(W, <<"1+1">>), + IsoLat = latencies(fun() -> {ok, 2} = py_context:eval(I, <<"1+1">>) end, N), + WrkLat = latencies(fun() -> {ok, 2} = py_context:eval(W, <<"1+1">>) end, N), + IsoCall = latencies(fun() -> {ok, 4.0} = py_context:call(I, math, sqrt, [16]) end, N), + WrkCall = latencies(fun() -> {ok, 4.0} = py_context:call(W, math, sqrt, [16]) end, N), + ct:log("eval isolated: ~s~n worker: ~s", [stats(IsoLat), stats(WrkLat)]), + ct:log("call isolated: ~s~n worker: ~s", [stats(IsoCall), stats(WrkCall)]), + ct:print("eval p50 isolated ~p us vs worker ~p us", [pct(IsoLat, 50), pct(WrkLat, 50)]), + py_context:stop(I), + py_context:stop(W), + ok. + +test_callback_round_trips(_Config) -> + {ok, C} = py_context:new(#{mode => isolated}), + py_callback:register(<<"stress_echo">>, fun([X]) -> X end), + Code = <<"__import__('erlang').call('stress_echo', 1)">>, + Lat = latencies(fun() -> {ok, 1} = py_context:eval(C, Code) end, 1000), + ct:log("eval+callback isolated: ~s", [stats(Lat)]), + %% 1000 callbacks inside one request + ok = py_context:exec(C, <<"import erlang\ndef burst(n):\n return sum(erlang.call('stress_echo', i) for i in range(n))\n">>), + T0 = erlang:monotonic_time(microsecond), + {ok, 499500} = py_context:call(C, '__main__', burst, [1000]), + Per = (erlang:monotonic_time(microsecond) - T0) / 1000, + ct:log("1000 callbacks in one request: ~.1f us each", [Per]), + py_callback:unregister(<<"stress_echo">>), + py_context:stop(C), + ok. + +%% @doc 100 contexts started and stopped: no child left, RSS reported. +test_context_churn_no_leak(_Config) -> + Pids = lists:map(fun(_) -> + {ok, C} = py_context:new(#{mode => isolated}), + {ok, #{os_pid := P}} = py_context:child_info(C), + {ok, 2} = py_context:eval(C, <<"1+1">>), + ok = py_context:stop(C), + P + end, lists:seq(1, 100)), + timer:sleep(500), + Alive = [P || P <- Pids, py_nif:os_kill(P, 0) =:= ok], + ct:log("children still alive after churn: ~p", [Alive]), + [] = Alive, + %% Memory per child + {ok, C} = py_context:new(#{mode => isolated}), + {ok, #{os_pid := P}} = py_context:child_info(C), + Rss = string:trim(os:cmd("ps -o rss= -p " ++ integer_to_list(P))), + ct:log("child RSS after start: ~s KB", [Rss]), + ct:print("child RSS: ~s KB", [Rss]), + {ok, _} = py_context:eval(C, <<"__import__('json').dumps([1]*1000)">>), + Rss2 = string:trim(os:cmd("ps -o rss= -p " ++ integer_to_list(P))), + ct:log("child RSS after json import: ~s KB", [Rss2]), + py_context:stop(C), + ok. + +test_startup_time(_Config) -> + Times = lists:map(fun(_) -> + T0 = erlang:monotonic_time(microsecond), + {ok, C} = py_context:new(#{mode => isolated}), + T = erlang:monotonic_time(microsecond) - T0, + ok = py_context:stop(C), + T + end, lists:seq(1, 20)), + ct:log("isolated context start (spawn -> ready -> init): ~s", [stats(Times)]), + ct:print("startup p50 ~p ms", [pct(Times, 50) div 1000]), + ok. + +test_payload_throughput(_Config) -> + {ok, I} = py_context:new(#{mode => isolated}), + {ok, W} = py_context:new(#{mode => worker}), + ok = py_context:exec(I, <<"def ident(x): return x">>), + ok = py_context:exec(W, <<"def ident(x): return x">>), + lists:foreach(fun(Size) -> + Bin = crypto:strong_rand_bytes(Size), + TI = timed(fun() -> {ok, Bin} = py_context:call(I, '__main__', ident, [Bin]) end), + TW = timed(fun() -> {ok, Bin} = py_context:call(W, '__main__', ident, [Bin]) end), + ct:log("~p MB round trip: isolated ~.1f ms (~.1f MB/s), worker ~.1f ms", + [Size div (1024 * 1024), TI / 1000, 2 * Size / 1048576 / (TI / 1.0e6), TW / 1000]) + end, [1024 * 1024, 16 * 1024 * 1024, 64 * 1024 * 1024]), + py_context:stop(I), + py_context:stop(W), + ok. + +%% @doc Four isolated children run CPU-bound work in parallel: the GIL is +%% per process, so wall time is close to one child's time. +test_parallel_contexts_cpu_bound(_Config) -> + Ctxs = [begin {ok, C} = py_context:new(#{mode => isolated}), C end || _ <- lists:seq(1, 4)], + Code = <<"sum(i*i for i in range(2000000))">>, + T1 = timed(fun() -> {ok, _} = py_context:eval(hd(Ctxs), Code) end), + Self = self(), + T4 = timed(fun() -> + [spawn_link(fun() -> Self ! {done, py_context:eval(C, Code)} end) || C <- Ctxs], + [receive {done, {ok, _}} -> ok after 60000 -> ct:fail(timeout) end || _ <- Ctxs] + end), + ct:log("cpu-bound: 1 child ~.1f ms, 4 children in parallel ~.1f ms", [T1 / 1000, T4 / 1000]), + %% On a dedicated machine T4 is close to T1 (see the log). CI VMs are + %% overcommitted, so only assert the children did not serialise. + true = T4 < 4 * T1, + [py_context:stop(C) || C <- Ctxs], + ok. + +%%% ============================================================================ +%%% Helpers +%%% ============================================================================ + +latencies(Fun, N) -> + lists:sort([timed(Fun) || _ <- lists:seq(1, N)]). + +timed(Fun) -> + T0 = erlang:monotonic_time(microsecond), + Fun(), + erlang:monotonic_time(microsecond) - T0. + +pct(Sorted, P) -> + Idx = max(1, min(length(Sorted), round(length(Sorted) * P / 100))), + lists:nth(Idx, Sorted). + +stats(Sorted) -> + Mean = lists:sum(Sorted) / length(Sorted), + io_lib:format("p50 ~p us, p99 ~p us, max ~p us, mean ~.1f us", + [pct(Sorted, 50), pct(Sorted, 99), lists:last(Sorted), Mean]). diff --git a/test/py_isolated_vm_SUITE.erl b/test/py_isolated_vm_SUITE.erl new file mode 100644 index 0000000..7572403 --- /dev/null +++ b/test/py_isolated_vm_SUITE.erl @@ -0,0 +1,477 @@ +%%% @doc Common Test suite: an isolated context as a participant in the VM. +%%% +%%% Mirrors, case for case, what py_pid_send_SUITE, py_callback_encoding_SUITE, +%%% py_thread_callback_SUITE and py_actor_SUITE prove for the embedded modes: +%%% pids, erlang.send, whereis, callback result encoding, Python threads +%%% calling Erlang, and actor-style state. Every case runs in a worker group +%%% too, so a divergence between modes fails as a pair. +-module(py_isolated_vm_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([ + all/0, + groups/0, + init_per_suite/1, + end_per_suite/1, + init_per_group/2, + end_per_group/2, + end_per_testcase/2 +]). + +-export([ + test_pid_is_pid/1, + test_pid_equality_and_hash/1, + test_pid_in_structure/1, + test_send_simple/1, + test_send_multiple_ordered/1, + test_send_complex_term/1, + test_send_is_nonblocking/1, + test_send_to_dead_process/1, + test_send_bad_pid/1, + test_send_from_coroutine/1, + test_whereis/1, + test_suspension_is_base_exception/1, + test_callback_inside_except_exception/1, + test_encoding_binary_with_escapes/1, + test_encoding_binary_non_utf8/1, + test_encoding_large_binary/1, + test_encoding_atom_becomes_str/1, + test_encoding_empty_list/1, + test_encoding_erlang_string/1, + test_encoding_nested_containers/1, + test_encoding_pid_and_ref/1, + test_encoding_floats/1, + test_encoding_booleans_and_none/1, + test_encoding_python_types/1, + test_threads_call_erlang/1, + test_threadpool_calls/1, + test_threadpool_error/1, + test_threadpool_nested/1, + test_threads_high_concurrency/1, + test_counter_actor/1, + test_state_reset_on_restart/1, + test_state_isolated_between_contexts/1, + test_ping_pong/1, + test_feed_through_callback/1 +]). + +-define(TEST_MOD, py_test_isolated). + +all() -> + [{group, worker}, {group, isolated}]. + +groups() -> + Cases = [ + test_pid_is_pid, + test_pid_equality_and_hash, + test_pid_in_structure, + test_send_simple, + test_send_multiple_ordered, + test_send_complex_term, + test_send_is_nonblocking, + test_send_to_dead_process, + test_send_bad_pid, + test_send_from_coroutine, + test_whereis, + test_suspension_is_base_exception, + test_callback_inside_except_exception, + test_encoding_binary_with_escapes, + test_encoding_binary_non_utf8, + test_encoding_large_binary, + test_encoding_atom_becomes_str, + test_encoding_empty_list, + test_encoding_erlang_string, + test_encoding_nested_containers, + test_encoding_pid_and_ref, + test_encoding_floats, + test_encoding_booleans_and_none, + test_encoding_python_types, + test_threads_call_erlang, + test_threadpool_calls, + test_threadpool_error, + test_threadpool_nested, + test_threads_high_concurrency, + test_counter_actor, + test_state_reset_on_restart, + test_state_isolated_between_contexts, + test_ping_pong, + test_feed_through_callback + ], + [{worker, [], Cases}, {isolated, [], Cases}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(erlang_python), + [{test_dir, filename:join(code:lib_dir(erlang_python), "test")} | Config]. + +end_per_suite(_Config) -> + ok = application:stop(erlang_python), + ok. + +init_per_group(Mode, Config) -> + [{mode, Mode} | Config]. + +end_per_group(_Group, _Config) -> + ok. + +end_per_testcase(_TestCase, _Config) -> + flush(), + ok. + +%%% ============================================================================ +%%% Pids, send, whereis +%%% ============================================================================ + +test_pid_is_pid(Config) -> + C = new_ctx(Config), + {ok, true} = py_context:call(C, ?TEST_MOD, is_pid, [self()]), + {ok, <<"Pid">>} = py_context:call(C, ?TEST_MOD, type_name, [self()]), + Self = self(), + {ok, Self} = py_context:call(C, ?TEST_MOD, identity, [Self]), + stop(C). + +test_pid_equality_and_hash(Config) -> + C = new_ctx(Config), + Self = self(), + Other = spawn(fun() -> receive stop -> ok end end), + {ok, true} = py_context:call(C, ?TEST_MOD, pid_equal, [Self, Self]), + {ok, false} = py_context:call(C, ?TEST_MOD, pid_equal, [Self, Other]), + {ok, true} = py_context:call(C, ?TEST_MOD, pid_hash_equal, [Self, Self]), + Other ! stop, + stop(C). + +test_pid_in_structure(Config) -> + C = new_ctx(Config), + Self = self(), + {ok, #{<<"owner">> := Self, <<"list">> := [Self, {Self, 1}]}} = + py_context:call(C, ?TEST_MOD, pid_in_structure, [Self]), + stop(C). + +test_send_simple(Config) -> + C = new_ctx(Config), + {ok, true} = py_context:call(C, ?TEST_MOD, send, [self(), <<"hello">>]), + receive <<"hello">> -> ok after 2000 -> ct:fail(no_message) end, + stop(C). + +test_send_multiple_ordered(Config) -> + C = new_ctx(Config), + N = 500, + {ok, N} = py_context:call(C, ?TEST_MOD, send_many, [self(), N]), + Items = collect_items([]), + Expected = lists:seq(0, N - 1), + Expected = Items, + stop(C). + +test_send_complex_term(Config) -> + C = new_ctx(Config), + Term = {hello, 42, [1, 2, 3], #{<<"key">> => <<"value">>}, true, none, 1.5}, + {ok, true} = py_context:call(C, ?TEST_MOD, send, [self(), Term]), + receive + {<<"hello">>, 42, [1, 2, 3], #{<<"key">> := <<"value">>}, true, none, 1.5} -> ok + after 2000 -> + ct:fail(no_message) + end, + stop(C). + +test_send_is_nonblocking(Config) -> + C = new_ctx(Config), + Sink = spawn(fun() -> receive stop -> ok end end), + {ok, Ms} = py_context:call(C, ?TEST_MOD, send_timing, [Sink, 1000]), + ct:log("1000 erlang.send took ~.1f ms (~.1f us each)", [Ms, Ms]), + true = Ms < 5000, + Sink ! stop, + stop(C). + +test_send_to_dead_process(Config) -> + C = new_ctx(Config), + Dead = spawn(fun() -> ok end), + timer:sleep(50), + false = is_process_alive(Dead), + {ok, <<"process_error">>} = py_context:call(C, ?TEST_MOD, send_to_dead, [Dead]), + stop(C). + +test_send_bad_pid(Config) -> + C = new_ctx(Config), + {ok, <<"type_error">>} = py_context:call(C, ?TEST_MOD, send_bad_pid, []), + stop(C). + +test_send_from_coroutine(Config) -> + C = new_ctx(Config), + {ok, <<"sent">>} = py_context:call(C, ?TEST_MOD, send_from_coroutine, [self(), from_coro]), + receive <<"from_coro">> -> ok after 2000 -> ct:fail(no_message) end, + stop(C). + +test_whereis(Config) -> + C = new_ctx(Config), + Name = py_isolated_vm_probe, + true = register(Name, self()), + Self = self(), + {ok, Self} = py_context:call(C, ?TEST_MOD, whereis, [<<"py_isolated_vm_probe">>]), + {ok, none} = py_context:call(C, ?TEST_MOD, whereis, [<<"no_such_registered_name_xyz">>]), + unregister(Name), + stop(C). + +test_suspension_is_base_exception(Config) -> + C = new_ctx(Config), + {ok, true} = py_context:call(C, ?TEST_MOD, suspension_is_base_exception, []), + stop(C). + +test_callback_inside_except_exception(Config) -> + C = new_ctx(Config), + py_callback:register(<<"vm_echo">>, fun([X]) -> X end), + {ok, {<<"ok">>, 42}} = py_context:call(C, ?TEST_MOD, call_inside_except_exception, [<<"vm_echo">>, 42]), + py_callback:unregister(<<"vm_echo">>), + stop(C). + +%%% ============================================================================ +%%% Callback result encoding (py_callback_encoding_SUITE) +%%% ============================================================================ + +test_encoding_binary_with_escapes(Config) -> + C = new_ctx(Config), + Value = <<"back\\slash \"dq\" 'sq'\nnewline\ttab\r">>, + Value = probe(C, Value), + <<"str">> = probe_type(C, Value), + stop(C). + +test_encoding_binary_non_utf8(Config) -> + C = new_ctx(Config), + Value = <<0, 1, 255, 254, 128>>, + Value = probe(C, Value), + <<"bytes">> = probe_type(C, Value), + stop(C). + +test_encoding_large_binary(Config) -> + C = new_ctx(Config), + Value = binary:copy(<<"abcdefghij">>, 20000), + Value = probe(C, Value), + stop(C). + +test_encoding_atom_becomes_str(Config) -> + C = new_ctx(Config), + <<"some_atom">> = probe(C, some_atom), + <<"str">> = probe_type(C, some_atom), + stop(C). + +test_encoding_empty_list(Config) -> + C = new_ctx(Config), + [] = probe(C, []), + <<"list">> = probe_type(C, []), + stop(C). + +test_encoding_erlang_string(Config) -> + C = new_ctx(Config), + "abc" = probe(C, "abc"), + <<"list">> = probe_type(C, "abc"), + <<"abc">> = probe(C, <<"abc">>), + <<"str">> = probe_type(C, <<"abc">>), + stop(C). + +test_encoding_nested_containers(Config) -> + C = new_ctx(Config), + Value = #{<<"k">> => [1, 2.5, {a, b}, #{<<"inner">> => [[], {}]}]}, + Expected = #{<<"k">> => [1, 2.5, {<<"a">>, <<"b">>}, #{<<"inner">> => [[], {}]}]}, + Expected = probe(C, Value), + <<"dict">> = probe_type(C, Value), + stop(C). + +test_encoding_pid_and_ref(Config) -> + C = new_ctx(Config), + Pid = self(), + Pid = probe(C, Pid), + <<"Pid">> = probe_type(C, Pid), + Ref = make_ref(), + Ref = probe(C, Ref), + <<"Ref">> = probe_type(C, Ref), + stop(C). + +test_encoding_floats(Config) -> + C = new_ctx(Config), + lists:foreach(fun(F) -> F = probe(C, F) end, + [3.14159265358979, 1.0e-300, 1.7976931348623157e308, -0.0]), + stop(C). + +test_encoding_booleans_and_none(Config) -> + C = new_ctx(Config), + true = probe(C, true), + false = probe(C, false), + <<"bool">> = probe_type(C, true), + lists:foreach(fun(A) -> + none = probe(C, A), + <<"NoneType">> = probe_type(C, A) + end, [undefined, nil, none]), + stop(C). + +test_encoding_python_types(Config) -> + C = new_ctx(Config), + <<"tuple">> = probe_type(C, {1, 2}), + <<"int">> = probe_type(C, 42), + <<"float">> = probe_type(C, 1.5), + stop(C). + +%%% ============================================================================ +%%% Python threads calling Erlang (py_thread_callback_SUITE) +%%% ============================================================================ + +test_threads_call_erlang(Config) -> + C = new_ctx(Config), + py_callback:register(<<"vm_tid">>, fun([T, I]) -> T * 1000 + I end), + {ok, {<<"ok">>, true, 40}} = py_context:call(C, ?TEST_MOD, thread_calls, [<<"vm_tid">>, 4, 10], #{}, 30000), + py_callback:unregister(<<"vm_tid">>), + stop(C). + +test_threadpool_calls(Config) -> + C = new_ctx(Config), + py_callback:register(<<"vm_double">>, fun([X]) -> X * 2 end), + {ok, true} = py_context:call(C, ?TEST_MOD, pool_calls, [<<"vm_double">>, 8, 200], #{}, 30000), + py_callback:unregister(<<"vm_double">>), + stop(C). + +test_threadpool_error(Config) -> + C = new_ctx(Config), + py_callback:register(<<"vm_fail">>, fun(_) -> throw(deliberate) end), + {ok, <<"RuntimeError">>} = py_context:call(C, ?TEST_MOD, pool_error, [<<"vm_fail">>]), + py_callback:unregister(<<"vm_fail">>), + stop(C). + +%% @doc From a pool thread: two erlang.call round trips nested in one +%% expression. (A callback that re-enters the context itself would deadlock +%% in every mode: the main thread is busy waiting on the pool.) +test_threadpool_nested(Config) -> + C = new_ctx(Config), + py_callback:register(<<"vm_nested">>, fun([X]) -> X + 11 end), + {ok, 42} = py_context:call(C, ?TEST_MOD, pool_nested, [<<"vm_nested">>], #{}, 30000), + py_callback:unregister(<<"vm_nested">>), + stop(C). + +test_threads_high_concurrency(Config) -> + C = new_ctx(Config), + py_callback:register(<<"vm_tid">>, fun([T, I]) -> T * 1000 + I end), + {ok, {<<"ok">>, true, 1600}} = py_context:call(C, ?TEST_MOD, thread_calls, [<<"vm_tid">>, 32, 50], #{}, 60000), + py_callback:unregister(<<"vm_tid">>), + stop(C). + +%%% ============================================================================ +%%% Actor-style state (py_actor_SUITE) +%%% ============================================================================ + +test_counter_actor(Config) -> + C = new_ctx(Config), + lists:foreach(fun(I) -> + {ok, I} = py_context:call(C, ?TEST_MOD, counter_increment, []) + end, lists:seq(1, 100)), + {ok, 110} = py_context:call(C, ?TEST_MOD, counter_increment, [10]), + {ok, 110} = py_context:call(C, ?TEST_MOD, counter_value, []), + stop(C). + +test_state_reset_on_restart(Config) -> + C1 = new_ctx(Config), + {ok, V0} = py_context:call(C1, ?TEST_MOD, counter_value, []), + V1 = V0 + 1, + V2 = V0 + 2, + {ok, V1} = py_context:call(C1, ?TEST_MOD, counter_increment, []), + {ok, V2} = py_context:call(C1, ?TEST_MOD, counter_increment, []), + stop(C1), + C2 = new_ctx(Config), + %% A new context: worker mode shares the interpreter, so the module + %% state persists; isolated mode starts a new process and it does not. + {ok, V} = py_context:call(C2, ?TEST_MOD, counter_value, []), + case ?config(mode, Config) of + isolated -> 0 = V; + worker -> V2 = V + end, + stop(C2). + +test_state_isolated_between_contexts(Config) -> + C1 = new_ctx(Config), + C2 = new_ctx(Config), + ok = py_context:exec(C1, <<"who = 'one'">>), + ok = py_context:exec(C2, <<"who = 'two'">>), + {ok, <<"one">>} = py_context:eval(C1, <<"who">>), + {ok, <<"two">>} = py_context:eval(C2, <<"who">>), + stop(C1), + stop(C2). + +%%% ============================================================================ +%%% Message flow both ways +%%% ============================================================================ + +%% @doc 1000 rounds of Erlang -> Python -> Erlang fun -> Python -> Erlang, +%% ordering asserted and no message left behind. +test_ping_pong(Config) -> + C = new_ctx(Config), + py_callback:register(<<"vm_incr">>, fun([X]) -> X + 1 end), + {ok, 1000} = py_context:call(C, ?TEST_MOD, ping_pong, [<<"vm_incr">>, 1000], #{}, 60000), + py_callback:unregister(<<"vm_incr">>), + receive Any -> ct:fail({unexpected_message, Any}) after 0 -> ok end, + stop(C). + +%% @doc Erlang feeds terms to the child through a callback it polls. +test_feed_through_callback(Config) -> + C = new_ctx(Config), + Feeder = spawn_link(fun() -> feeder(lists:seq(1, 200)) end), + py_callback:register(<<"vm_next">>, fun([]) -> + Feeder ! {next, self()}, + receive {item, I} -> I after 5000 -> none end + end), + {ok, Items} = py_context:call(C, ?TEST_MOD, poll_feed, [<<"vm_next">>, 200], #{}, 60000), + Expected = lists:seq(1, 200), + Expected = Items, + py_callback:unregister(<<"vm_next">>), + Feeder ! stop, + stop(C). + +feeder([]) -> + receive stop -> ok; {next, From} -> From ! {item, none}, feeder([]) end; +feeder([H | T] = L) -> + receive + stop -> ok; + {next, From} -> From ! {item, H}, feeder(T) + after 10000 -> + feeder(L) + end. + +%%% ============================================================================ +%%% Helpers +%%% ============================================================================ + +probe(C, Value) -> + py_callback:register(<<"vm_probe">>, fun(_) -> Value end), + {ok, Got} = py_context:call(C, ?TEST_MOD, callback, [<<"vm_probe">>]), + py_callback:unregister(<<"vm_probe">>), + Got. + +probe_type(C, Value) -> + py_callback:register(<<"vm_probe">>, fun(_) -> Value end), + {ok, Type} = py_context:call(C, ?TEST_MOD, callback_type, [<<"vm_probe">>]), + py_callback:unregister(<<"vm_probe">>), + Type. + +collect_items(Acc) -> + receive + {<<"item">>, I} -> collect_items([I | Acc]); + <<"done">> -> lists:reverse(Acc) + after 5000 -> + ct:fail({incomplete, length(Acc)}) + end. + +new_ctx(Config) -> + Mode = ?config(mode, Config), + TestDir = ?config(test_dir, Config), + {ok, C} = py_context:new(#{mode => Mode, paths => [TestDir]}), + case Mode of + worker -> + ok = py_context:exec(C, iolist_to_binary(io_lib:format( + "import sys\nif '~s' not in sys.path: sys.path.insert(0, '~s')", + [TestDir, TestDir]))); + _ -> + ok + end, + C. + +stop(C) -> + ok = py_context:stop(C), + ok. + +flush() -> + receive _ -> flush() after 0 -> ok end. diff --git a/test/py_test_isolated.py b/test/py_test_isolated.py new file mode 100644 index 0000000..1567b47 --- /dev/null +++ b/test/py_test_isolated.py @@ -0,0 +1,374 @@ +"""Helpers for the isolated-mode suites (py_isolated_SUITE, +py_isolated_vm_SUITE, py_isolated_async_SUITE). + +Every function here runs unchanged in worker and isolated mode; the suites +run both so a divergence between modes shows up as a failing pair. +""" + +import asyncio +import threading +import time +from concurrent.futures import ThreadPoolExecutor + +import erlang + +# --------------------------------------------------------------------------- +# basics +# --------------------------------------------------------------------------- + +def add(a, b): + return a + b + + +def kwargs_probe(*args, **kwargs): + return (list(args), sorted(kwargs.items())) + + +def identity(x): + return x + + +def type_name(x): + return type(x).__name__ + + +def raise_value_error(msg): + raise ValueError(msg) + + +def big_payload(n): + return b'x' * n + + +def sleep_then(seconds, value): + time.sleep(seconds) + return value + + +def blocked_sleep(seconds): + """Sleep with every signal blocked: a soft interrupt cannot land, only + SIGKILL can end this. Exercises the kill backstop.""" + import signal + signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGUSR1, signal.SIGINT}) + time.sleep(seconds) + return 'slept' + + +def segfault(): + import ctypes + ctypes.memset(0, 0, 1) + + +def close_control_socket(): + """Break the control socket from inside the child (mid-conversation), + to exercise the fail-loud discipline.""" + import os + import sys + rt = sys.modules['_erlang_impl._isolated'] + # The runtime installed in this process + import gc + for obj in gc.get_objects(): + if isinstance(obj, rt.Runtime): + os.close(obj.sock.fileno()) + break + time.sleep(5) + return 'unreachable' + + +def allocate(n_bytes): + data = bytearray(n_bytes) + return len(data) + + +def allocate_and_touch(n_bytes): + """Allocate and write every page, so the memory is resident (a bare + bytearray may stay untouched virtual memory on some platforms).""" + data = bytearray(n_bytes) + for i in range(0, n_bytes, 4096): + data[i] = 1 + return len(data) + + +def spin(seconds): + end = time.monotonic() + seconds + n = 0 + while time.monotonic() < end: + n += 1 + return n + + +def numpy_sum(n): + import numpy + return int(numpy.arange(n).sum()) + + +# --------------------------------------------------------------------------- +# VM interaction: pids, send, whereis, callbacks +# --------------------------------------------------------------------------- + +def is_pid(x): + return isinstance(x, erlang.Pid) + + +def pid_equal(a, b): + return a == b + + +def pid_hash_equal(a, b): + return hash(a) == hash(b) + + +def pid_in_structure(pid): + return {'owner': pid, 'list': [pid, (pid, 1)]} + + +def send(pid, msg): + erlang.send(pid, msg) + return True + + +def send_many(pid, n): + for i in range(n): + erlang.send(pid, ('item', i)) + erlang.send(pid, 'done') + return n + + +def send_timing(pid, n): + t0 = time.perf_counter() + for i in range(n): + erlang.send(pid, i) + return (time.perf_counter() - t0) * 1000.0 + + +def send_to_dead(pid): + try: + erlang.send(pid, 'msg') + return 'sent' + except erlang.ProcessError: + return 'process_error' + + +def send_bad_pid(): + try: + erlang.send('not_a_pid', 'msg') + return 'sent' + except TypeError: + return 'type_error' + + +def send_from_coroutine(pid, msg): + async def go(): + erlang.send(pid, msg) + return 'sent' + return asyncio.run(go()) + + +def whereis(name): + return erlang.whereis(name) + + +def suspension_is_base_exception(): + return (issubclass(erlang.SuspensionRequired, BaseException) + and not issubclass(erlang.SuspensionRequired, Exception)) + + +def call_inside_except_exception(name, arg): + """A callback inside `except Exception` must work in every mode.""" + try: + return ('ok', erlang.call(name, arg)) + except Exception as exc: + return ('caught', type(exc).__name__) + + +def callback(name, *args): + return erlang.call(name, *args) + + +def callback_error_type(name): + try: + erlang.call(name) + return 'no_error' + except Exception as exc: + return type(exc).__name__ + + +def callback_type(name): + return type(erlang.call(name)).__name__ + + +def ping_pong(name, rounds): + """Each round calls Erlang with the round number and checks the answer.""" + for i in range(rounds): + got = erlang.call(name, i) + if got != i + 1: + return ('mismatch', i, got) + return rounds + + +def poll_feed(name, expect): + """Pull terms from Erlang through a callback until `expect` items.""" + items = [] + while len(items) < expect: + item = erlang.call(name) + if item is None: + time.sleep(0.001) + continue + items.append(item) + return items + + +# --------------------------------------------------------------------------- +# Threads calling Erlang +# --------------------------------------------------------------------------- + +def thread_calls(name, n_threads, n_calls): + results = {} + errors = [] + + def worker(tid): + try: + results[tid] = [erlang.call(name, tid, i) for i in range(n_calls)] + except Exception as exc: + errors.append(repr(exc)) + + threads = [threading.Thread(target=worker, args=(t,)) for t in range(n_threads)] + for t in threads: + t.start() + for t in threads: + t.join() + if errors: + return ('errors', errors) + ok = all(results[t] == [t * 1000 + i for i in range(n_calls)] + for t in range(n_threads)) + return ('ok', ok, n_threads * n_calls) + + +def pool_calls(name, n_workers, n_calls): + with ThreadPoolExecutor(max_workers=n_workers) as pool: + futures = [pool.submit(erlang.call, name, i) for i in range(n_calls)] + got = [f.result() for f in futures] + return got == [i * 2 for i in range(n_calls)] + + +def pool_error(name): + with ThreadPoolExecutor(max_workers=2) as pool: + fut = pool.submit(erlang.call, name) + try: + fut.result() + return 'no_error' + except Exception as exc: + return type(exc).__name__ + + +def pool_nested(name): + """A pool thread makes an erlang.call whose argument is itself the + result of an erlang.call: two round trips nested in one thread.""" + with ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(lambda: erlang.call(name, erlang.call(name, 20))).result() + + +# --------------------------------------------------------------------------- +# Actor-style state +# --------------------------------------------------------------------------- + +class Counter: + def __init__(self): + self.value = 0 + + def increment(self, by=1): + self.value += by + return self.value + + +_counter = Counter() + + +def counter_increment(by=1): + return _counter.increment(by) + + +def counter_value(): + return _counter.value + + +# --------------------------------------------------------------------------- +# asyncio +# --------------------------------------------------------------------------- + +async def async_add(a, b): + await asyncio.sleep(0) + return a + b + + +async def async_sleep_gather(n, seconds): + async def one(i): + await asyncio.sleep(seconds) + return i + return await asyncio.gather(*[one(i) for i in range(n)]) + + +async def async_raise(msg): + await asyncio.sleep(0) + raise KeyError(msg) + + +async def async_big(n): + await asyncio.sleep(0) + return b'y' * n + + +async def async_erlang_call(name, x): + return await erlang.async_call(name, x) + + +async def async_erlang_calls(name, n): + return await asyncio.gather(*[erlang.async_call(name, i) for i in range(n)]) + + +async def async_erlang_call_error(name): + try: + await erlang.async_call(name) + return 'no_error' + except Exception as exc: + return type(exc).__name__ + + +async def async_send(pid, msg): + erlang.send(pid, msg) + return 'sent' + + +async def stream_to(pid, n): + async def agen(): + for i in range(n): + await asyncio.sleep(0) + yield i + async for item in agen(): + erlang.send(pid, ('item', item)) + erlang.send(pid, 'done') + return n + + +async def task_value(i): + await asyncio.sleep(0.001 * (i % 5)) + return i * i + + +async def slow_task(seconds): + await asyncio.sleep(seconds) + return 'slow_done' + + +async def block_loop(seconds): + time.sleep(seconds) + return 'unblocked' + + +def run_helper_compat(): + """erlang.run / erlang.sleep / erlang.spawn_task behave like stdlib.""" + async def main(): + await erlang.sleep(0.001) + t = erlang.spawn_task(async_add(1, 2)) + return await t + return erlang.run(main()) From bdfb81a91f87b584eccbc4c8d64af378d5c23a50 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sat, 29 Aug 2026 13:28:51 +0200 Subject: [PATCH 04/15] Add shared memory regions and shared buffers over iommap Bulk data between Erlang and Python contexts no longer has to cross the socket: py_shm regions are files mapped MAP_SHARED on both sides (no copy on the Erlang side, one copy when Erlang writes), usable as plain terms in any context mode, and py_buffer:new(#{shared => true}) is a streaming input buffer over such a region with ring backpressure, so wsgi.input works in isolated contexts. iommap is an optional dependency. Read-only handles keep a callee from writing into a region. --- CHANGELOG.md | 9 + README.md | 4 +- c_src/py_convert.c | 43 +++ docs/isolated.md | 74 ++++- priv/_erlang_impl/_etf.py | 14 +- priv/_erlang_impl/_isolated.py | 16 +- priv/_erlang_impl/_shm.py | 369 +++++++++++++++++++++++ rebar.config | 7 + src/erlang_python_sup.erl | 13 +- src/py_buffer.erl | 28 +- src/py_shm.erl | 485 ++++++++++++++++++++++++++++++ test/py_isolated_buffer_SUITE.erl | 286 ++++++++++++++++++ test/py_isolated_shm_SUITE.erl | 362 ++++++++++++++++++++++ test/py_isolated_stress_SUITE.erl | 55 +++- test/py_test_isolated_shm.py | 159 ++++++++++ test/py_worker_loop_SUITE.erl | 6 +- 16 files changed, 1910 insertions(+), 20 deletions(-) create mode 100644 priv/_erlang_impl/_shm.py create mode 100644 src/py_shm.erl create mode 100644 test/py_isolated_buffer_SUITE.erl create mode 100644 test/py_isolated_shm_SUITE.erl create mode 100644 test/py_test_isolated_shm.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ad58ce1..f042595 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,15 @@ - **Pure-Python ETF codec** (`priv/_erlang_impl/_etf.py`) with the type mapping of `py_convert.c`; the child needs no C extension. Integers beyond 64 bits round-trip exactly in isolated mode. +- **Shared memory** - `py_shm:new/1,2`, `write/3`, `read/3`, `binary/3` + (no copy), `close/1`: fixed-size regions over + [iommap](https://hex.pm/packages/iommap) (optional dependency) that any + context mode maps as `erlang.SharedMemory` (buffer protocol, numpy + friendly). `py_buffer:new(#{shared => true})` is a streaming buffer over + such a region with ring backpressure, usable as `wsgi.input` in isolated + contexts. Handles are plain terms and travel inside any argument or result; + `py_shm:read_only/1` and `new(Size, #{writable => false})` hand Python a + read-only mapping. - `py:python_executable/0`, `py:kill/1`, `py_nif:os_kill/2`. - `py_isolated` is a `gen_statem` (states `idle`, `{busy, Id}`, `looping`, `stopping_loop`, `{restarting, Reason}`): `sys:get_state/1` and diff --git a/README.md b/README.md index 66b2ae8..aa550d5 100644 --- a/README.md +++ b/README.md @@ -623,7 +623,9 @@ When creating Python contexts, you can choose the execution mode: **Isolated mode** is the only mode with a hard bound: `py_context:interrupt/1` stops a blocking C call, and `SIGKILL` is the backstop. It costs a process per context (about 16 MB and 40 ms to start) and roughly twice the call latency. -See [Isolated Contexts](docs/isolated.md). +Bulk data crosses through shared memory (`py_shm`, with the optional +[iommap](https://hex.pm/packages/iommap) dependency). See +[Isolated Contexts](docs/isolated.md). **Worker mode is recommended** because it works with any Python version and automatically benefits from free-threaded Python (3.13t+) when available. Each context owns a dedicated pthread, providing stable thread affinity for libraries with thread-local state (numpy, torch, tensorflow). diff --git a/c_src/py_convert.c b/c_src/py_convert.c index 0d27b0e..bee01c1 100644 --- a/c_src/py_convert.c +++ b/c_src/py_convert.c @@ -404,6 +404,18 @@ static ERL_NIF_TERM py_to_term_d(ErlNifEnv *env, PyObject *obj, int depth) { return enif_make_atom(env, atom_obj->name); } + /* Shared memory wrappers (_erlang_impl._shm) travel as their handle tuple */ + if (PyObject_HasAttrString(obj, "to_term") && + PyObject_HasAttrString(obj, "_mmap")) { + PyObject *term = PyObject_CallMethod(obj, "to_term", NULL); + if (term != NULL) { + ERL_NIF_TERM result = py_to_term_d(env, term, depth + 1); + Py_DECREF(term); + return result; + } + PyErr_Clear(); + } + /* Handle NumPy arrays by converting to Python list first */ if (is_numpy_ndarray(obj)) { PyObject *tolist = PyObject_CallMethod(obj, "tolist", NULL); @@ -601,6 +613,37 @@ static PyObject *term_to_py_d(ErlNifEnv *env, ERL_NIF_TERM term, int depth) { } } + /* {'$py_shm', Id, Path, Size} / {'$py_buffer', Id, Path, Ring}: a shared + * region handle, turned into the Python wrapper (mapped once per + * interpreter, see priv/_erlang_impl/_shm.py). */ + { + int arity4; + const ERL_NIF_TERM *el; + if (enif_get_tuple(env, term, &arity4, &el) && arity4 == 4) { + char tag_buf[16]; + if (enif_get_atom(env, el[0], tag_buf, sizeof(tag_buf), ERL_NIF_LATIN1) && + (strcmp(tag_buf, "$py_shm") == 0 || strcmp(tag_buf, "$py_shm_ro") == 0 || + strcmp(tag_buf, "$py_buffer") == 0)) { + PyObject *mod = PyImport_ImportModule("_erlang_impl._shm"); + if (mod == NULL) { + return NULL; + } + PyObject *id = term_to_py_d(env, el[1], depth + 1); + PyObject *path = term_to_py_d(env, el[2], depth + 1); + PyObject *size = term_to_py_d(env, el[3], depth + 1); + PyObject *result = NULL; + if (id != NULL && path != NULL && size != NULL) { + result = PyObject_CallMethod(mod, "from_term", "sOOO", tag_buf, id, path, size); + } + Py_XDECREF(id); + Py_XDECREF(path); + Py_XDECREF(size); + Py_DECREF(mod); + return result; + } + } + } + /* Check list (must come after binary to preserve structure) */ if (enif_get_list_length(env, term, &list_len)) { PyObject *list = PyList_New(list_len); diff --git a/docs/isolated.md b/docs/isolated.md index b737bf2..817fd77 100644 --- a/docs/isolated.md +++ b/docs/isolated.md @@ -202,6 +202,73 @@ socket with `SCM_RIGHTS`: Inside a coroutine, `await erlang.async_call(name, *args)` keeps the loop running while Erlang answers. +## Bulk data with shared memory + +Arguments and results cross the socket as a copy. For large payloads use a +shared region: a file mapped `MAP_SHARED` on both sides through +[iommap](https://hex.pm/packages/iommap). Add it to your deps: + +```erlang +{deps, [{iommap, "1.1.3"}]}. +``` + +A region is a fixed-size handle you pass like any other argument, in any +context mode: + +```erlang +{ok, Shm} = py_shm:new(64 * 1024 * 1024), +ok = py_shm:write(Shm, 0, Floats), %% one copy +{ok, Sum} = py_context:call(Ctx, myapp, sum_floats, [Shm]), +Out = py_shm:binary(Shm, 0, 1024), %% no copy +ok = py_shm:close(Shm). +``` + +```python +import numpy + +def sum_floats(shm): # erlang.SharedMemory + a = numpy.frombuffer(shm.buffer, dtype=numpy.float32) # no copy + a[:1024] = 0 # Erlang sees it + return float(a.sum()) +``` + +Python-produced data is zero-copy in both directions (write into the +region, read it in Erlang with `py_shm:binary/3`); Erlang-produced data +costs one `write/3` copy instead of encode, socket, decode and copy. A +region is mapped once per interpreter and reused across calls; it is +closed by `close/1` or when its owner process exits. + +Streaming bodies use the same mechanism through `py_buffer`: + +```erlang +{ok, Buf} = py_buffer:new(#{shared => true}), %% 4 MB ring +ok = py_buffer:write(Buf, Chunk), %% blocks when full +ok = py_buffer:close(Buf), +py_context:call(Ctx, myapp, handle, [#{<<"wsgi.input">> => Buf}]). +``` + +The Python side gets `erlang.SharedBuffer` with the `read`, `readline`, +`readlines`, iteration and `read_nonblock` of the native buffer. Flow +control is a callback round trip per blocking read, so in an embedded +context the native `py_buffer:new/0,1` is still the cheaper choice; use +`shared => true` when the buffer may reach an isolated context or a pool +mixing modes. A native buffer cannot cross into a child. + +Rules: regions are never resized (a truncated file would be a `SIGBUS`, so +the size is checked when mapping); mapped pages count against the child's +`as` limit and its resident set; `/dev/shm` is used when present, else a +private directory under `TMPDIR`; while a call holds a handle the child +owns the region, and a concurrent `write/3` is a caller error. + +What sharing changes about isolation: the child can still only crash or +exhaust itself, but it can write anything into a region it holds, at any +time, and `py_shm:binary/3` sees those bytes (a binary that changes under +you; take it once the callee is done, or copy with `read/3`). Hand the child +a read-only handle when it only needs to read: `py_shm:new(Size, #{writable => false})` +or `py_shm:read_only(Shm)` map it `PROT_READ` in Python, and Erlang keeps +writing. A child that runs as your user could still truncate the region +file on purpose; sealing and syscall filtering are separate hardening work. + ## Process model - One child per context, started with `open_port` so the VM reaps it and @@ -252,12 +319,7 @@ running while Erlang answers. seccomp (Linux) or Capsicum (FreeBSD) sandbox is a separate hardening step. - Each call copies its arguments and result through the socket: a 1 MB binary round-trips in about 1.3 ms, 16 MB in about 27 ms (worker mode: - 0.2 ms and 3 ms). For bulk data prefer a file, a socket the child reads - itself, or a shared mapping: `erlang-iommap` opens a file `MAP_SHARED` and - `region_binary/3` gives a binary over it, while the child maps the same - file with `mmap`; a 64 MB region costs 3 us on the Erlang side and 5 ms to - map in Python, against 12 ms to copy it through ETF. `py_buffer` is not - ported to that yet. + 0.2 ms and 3 ms). For bulk data use shared memory (below). ## See also diff --git a/priv/_erlang_impl/_etf.py b/priv/_erlang_impl/_etf.py index 47fceec..abfe001 100644 --- a/priv/_erlang_impl/_etf.py +++ b/priv/_erlang_impl/_etf.py @@ -42,9 +42,17 @@ __all__ = [ 'Atom', 'Pid', 'Ref', 'Port', - 'encode', 'decode', 'DecodeError', + 'encode', 'decode', 'DecodeError', 'register_encoder', ] +# (predicate, to_term) pairs consulted before the generic fallback +_encoders = [] + + +def register_encoder(predicate, to_term): + """Encode objects matching `predicate` as the term `to_term(obj)` returns.""" + _encoders.append((predicate, to_term)) + VERSION = 131 # Tags @@ -234,6 +242,10 @@ def _encode(obj, out): elif isinstance(obj, (set, frozenset)): _encode(list(obj), out) else: + for predicate, to_term in _encoders: + if predicate(obj): + _encode(to_term(obj), out) + return # Same fallback as py_to_term: the string representation as a binary _encode(str(obj), out) diff --git a/priv/_erlang_impl/_isolated.py b/priv/_erlang_impl/_isolated.py index a276c36..04f6a97 100644 --- a/priv/_erlang_impl/_isolated.py +++ b/priv/_erlang_impl/_isolated.py @@ -58,8 +58,12 @@ import traceback from . import _etf +from . import _shm from ._etf import Atom, Pid, Ref, Port, DecodeError +# Shared memory wrappers returned to Erlang travel as their handle tuple +_etf.register_encoder(_shm.is_shared, lambda obj: obj.to_term()) + __all__ = ['Runtime', 'install_erlang_module'] STATUS_REQUEST = 0 @@ -379,6 +383,8 @@ def _on_control(self, term): # raises when the nested request finishes self.inbox.put(('interrupt', target)) # else: already finished (or still queued: cancel handles that) + elif isinstance(term, tuple) and len(term) == 2 and term[0] == 'shm_close': + _shm.forget(term[1]) elif isinstance(term, tuple) and len(term) == 2 and term[0] == 'cancel': with self._cancel_lock: self._cancelled.add(term[1]) @@ -425,7 +431,8 @@ def _on_submit(self, frame_id, term): def schedule(): try: fn = _resolve(module, func) - result = fn(*_as_list(args), **_as_dict(kwargs)) + result = fn(*_shm.convert_args(_as_list(args)), + **_shm.convert_args(_as_dict(kwargs))) if inspect.isawaitable(result): task = asyncio.ensure_future(result) task.add_done_callback( @@ -554,11 +561,12 @@ def _dispatch(self, tag, term): if tag == 'call': _, module, func, args, kwargs = term fn = _resolve(module, func, self.globals) - result = fn(*_as_list(args), **_as_dict(kwargs)) + result = fn(*_shm.convert_args(_as_list(args)), + **_shm.convert_args(_as_dict(kwargs))) elif tag == 'eval': _, code, locals_ = term loc = dict(self.globals) - loc.update(_as_dict(locals_)) + loc.update(_shm.convert_args(_as_dict(locals_))) result = eval(compile(_as_text(code), '', 'eval'), self.globals, loc) elif tag == 'exec': _, code = term @@ -792,12 +800,14 @@ def __getattr__(name): return Function(name) from . import _server as server + from ._shm import SharedMemory, SharedBuffer ns = dict( call=call, async_call=async_call, send=send, whereis=whereis, self=self_, atom=atom, Atom=Atom, Pid=Pid, Ref=Ref, Port=Port, ProcessError=ProcessError, SuspensionRequired=SuspensionRequired, Function=Function, is_isolated=is_isolated, run=run, + SharedMemory=SharedMemory, SharedBuffer=SharedBuffer, new_event_loop=new_event_loop, get_event_loop_policy=get_event_loop_policy, install=install, spawn_task=spawn_task, sleep=sleep, log=log, server=server, __getattr__=__getattr__, diff --git a/priv/_erlang_impl/_shm.py b/priv/_erlang_impl/_shm.py new file mode 100644 index 0000000..5b490a9 --- /dev/null +++ b/priv/_erlang_impl/_shm.py @@ -0,0 +1,369 @@ +# Copyright 2026 Benoit Chesneau +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Shared memory regions and shared streaming buffers (py_shm, py_buffer with +shared => true), seen from Python. + +A region handle arrives from Erlang as the tuple ('$py_shm', id, path, size) +and is turned into a SharedMemory; a shared buffer handle +('$py_buffer', id, path, ring_size) becomes a SharedBuffer. Both map the +region file with mmap (MAP_SHARED), so the memory is the one Erlang wrote +through iommap. The same classes serve the embedded interpreter (conversion +in c_src/py_convert.c) and the isolated child (conversion in _isolated.py). + +Flow control for buffers goes through Erlang callbacks: + erlang.call('_py_buffer_wait', id, read_pos) -> (write_pos, closed) + erlang.call('_py_buffer_consumed', id, n) -> ok + erlang.call('_py_buffer_state', id) -> (write_pos, closed) +""" + +import collections +import mmap +import os +import struct +import threading + +__all__ = ['SharedMemory', 'SharedBuffer', 'from_term', 'is_shared', 'forget'] + +SHM_TAG = '$py_shm' +SHM_RO_TAG = '$py_shm_ro' +BUFFER_TAG = '$py_buffer' +_HEADER = 4096 +_HEADER_FMT = struct.Struct('=QBQ') # write position, closed flag, ring size (Erlang) +_RPOS_OFFSET = _HEADER_FMT.size +_RPOS_FMT = struct.Struct('=Q') # read position (written by the reader) + +# id -> wrapper, bounded: mapping again is cheap next to the copy it saves, +# and Erlang's close does not reach every interpreter. +_CACHE_MAX = 64 +_cache = collections.OrderedDict() +_cache_lock = threading.Lock() + + +def _erlang(): + import erlang + return erlang + + +def _atom(name): + return _erlang().atom(name) + + +def _open_mapping(path, size, writable=True): + if isinstance(path, bytes): + path = os.fsdecode(path) + fd = os.open(path, os.O_RDWR if writable else os.O_RDONLY) + try: + actual = os.fstat(fd).st_size + if actual != size: + raise RuntimeError('shared region %s is %d bytes, handle says %d' + % (path, actual, size)) + prot = mmap.PROT_READ | (mmap.PROT_WRITE if writable else 0) + return mmap.mmap(fd, size, mmap.MAP_SHARED, prot) + finally: + os.close(fd) + + +class SharedMemory: + """A fixed-size region shared with Erlang. + + Supports the buffer protocol (memoryview, numpy.frombuffer, bytes()), + len(), slicing for read and write, and `close()`. `buffer` is the + underlying mmap object, for APIs that want a plain buffer.""" + + __slots__ = ('id', 'path', 'size', 'writable', '_mmap', '__weakref__') + + def __init__(self, id, path, size, writable=True): + self.id = id + self.path = path + self.size = size + self.writable = writable + self._mmap = _open_mapping(path, size, writable) + + # buffer protocol (Python 3.12+); older versions use .buffer / memoryview + def __buffer__(self, flags): + return memoryview(self._mmap) + + def __release_buffer__(self, view): + view.release() + + @property + def buffer(self): + return self._mmap + + @property + def closed(self): + return self._mmap.closed + + def __len__(self): + return self.size + + def __getitem__(self, key): + return self._mmap[key] + + def __setitem__(self, key, value): + if not self.writable: + raise TypeError('read-only shared region') + self._mmap[key] = value + + def view(self, offset=0, length=None): + end = self.size if length is None else offset + length + return memoryview(self._mmap)[offset:end] + + def close(self): + with _cache_lock: + for k in [k for k in _cache if k[0] == self.id]: + _cache.pop(k, None) + if not self._mmap.closed: + self._mmap.close() + + def to_term(self): + return (_atom(SHM_TAG if self.writable else SHM_RO_TAG), self.id, self.path, self.size) + + def __repr__(self): + return '' % ( + self.id, self.size, '' if self.writable else ' read-only', + ' closed' if self.closed else '') + + +class SharedBuffer: + """Streaming input buffer over a shared ring: the `wsgi.input` shape. + + Erlang appends with py_buffer:write/2 and ends with py_buffer:close/1; + reads block until data or EOF, like the embedded PyBuffer.""" + + __slots__ = ('id', 'path', 'ring', '_mmap', '_rpos', '_wpos', '_closed', + '_lock', '__weakref__') + + def __init__(self, id, path, ring): + self.id = id + self.path = path + self.ring = ring + self._mmap = _open_mapping(path, _HEADER + ring) + self._wpos = 0 + self._closed = False + self._lock = threading.Lock() + # Resume where a previous mapping (a dead child) stopped + (self._rpos,) = _RPOS_FMT.unpack_from(self._mmap, _RPOS_OFFSET) + self._refresh_header() + + # -- state ------------------------------------------------------------- + + def _refresh_header(self): + wpos, flag, _ring = _HEADER_FMT.unpack_from(self._mmap, 0) + self._wpos = wpos + self._closed = bool(flag) + + def _wait_for_data(self): + """Block until write position passes our read position or EOF.""" + wpos, closed = _erlang().call('_py_buffer_wait', self.id, self._rpos) + self._wpos = wpos + self._closed = bool(closed) + + def _consumed(self, n): + if n: + _erlang().call('_py_buffer_consumed', self.id, n) + + def _available(self): + return self._wpos - self._rpos + + def _take(self, n): + """Copy n bytes (n <= available) out of the ring and advance.""" + start = self._rpos % self.ring + end = start + n + if end <= self.ring: + data = bytes(self._mmap[_HEADER + start:_HEADER + end]) + else: + first = self.ring - start + data = (bytes(self._mmap[_HEADER + start:_HEADER + self.ring]) + + bytes(self._mmap[_HEADER:_HEADER + (n - first)])) + self._rpos += n + _RPOS_FMT.pack_into(self._mmap, _RPOS_OFFSET, self._rpos) + self._consumed(n) + return data + + @property + def closed(self): + return self._mmap.closed + + def at_eof(self): + self._refresh_header() + return self._closed and self._available() == 0 + + def readable(self): + return True + + def writable(self): + return False + + def seekable(self): + return False + + def readable_amount(self): + self._refresh_header() + return self._available() + + # -- reads ------------------------------------------------------------- + + def read(self, size=-1): + with self._lock: + if size is None or size < 0: + chunks = [] + while True: + if self._available() == 0: + if self._closed: + break + self._wait_for_data() + continue + chunks.append(self._take(self._available())) + return b''.join(chunks) + if size == 0: + return b'' + while self._available() == 0: + if self._closed: + return b'' + self._wait_for_data() + return self._take(min(size, self._available())) + + def read_nonblock(self, size=-1): + with self._lock: + self._refresh_header() + avail = self._available() + if avail == 0: + return b'' + n = avail if (size is None or size < 0) else min(size, avail) + return self._take(n) + + def readline(self, size=-1): + with self._lock: + limit = None if (size is None or size < 0) else size + out = bytearray() + while True: + if self._available() == 0: + if self._closed: + return bytes(out) + self._wait_for_data() + continue + # Search the readable range for a newline, handling the wrap + avail = self._available() + want = avail if limit is None else min(avail, limit - len(out)) + start = self._rpos % self.ring + end = start + want + if end <= self.ring: + view = self._mmap[_HEADER + start:_HEADER + end] + else: + view = (self._mmap[_HEADER + start:_HEADER + self.ring] + + self._mmap[_HEADER:_HEADER + (end - self.ring)]) + idx = view.find(b'\n') + take = want if idx < 0 else idx + 1 + out += self._take(take) + if idx >= 0 or (limit is not None and len(out) >= limit): + return bytes(out) + + def readlines(self, hint=-1): + lines = [] + total = 0 + while True: + line = self.readline() + if not line: + return lines + lines.append(line) + total += len(line) + if hint is not None and hint > 0 and total >= hint: + return lines + + def __iter__(self): + return self + + def __next__(self): + line = self.readline() + if not line: + raise StopIteration + return line + + def close(self): + with _cache_lock: + _cache.pop((self.id, BUFFER_TAG), None) + if not self._mmap.closed: + self._mmap.close() + + def to_term(self): + return (_atom(BUFFER_TAG), self.id, self.path, self.ring) + + def __repr__(self): + return '' % (self.id, self.ring) + + +# --------------------------------------------------------------------------- +# conversion entry points (used by py_convert.c and _isolated.py) +# --------------------------------------------------------------------------- + +def is_shared(obj): + return isinstance(obj, (SharedMemory, SharedBuffer)) + + +def from_term(tag, id, path, size): + """Wrapper for a handle tuple, cached per interpreter by id.""" + key = (id, tag) + with _cache_lock: + cached = _cache.get(key) + if cached is not None and not cached.closed: + _cache.move_to_end(key) + return cached + if tag == SHM_TAG: + obj = SharedMemory(id, path, size) + elif tag == SHM_RO_TAG: + obj = SharedMemory(id, path, size, writable=False) + elif tag == BUFFER_TAG: + obj = SharedBuffer(id, path, size) + else: + raise ValueError('unknown shared handle tag %r' % (tag,)) + evicted = [] + with _cache_lock: + _cache[key] = obj + while len(_cache) > _CACHE_MAX: + evicted.append(_cache.popitem(last=False)[1]) + # Unmap outside the lock: close() takes it too + for old in evicted: + try: + old.close() + except Exception: + pass + return obj + + +def forget(id): + """Drop and unmap cached wrappers of a region (Erlang closed it).""" + with _cache_lock: + objs = [_cache.pop(k) for k in list(_cache) if k[0] == id] + for obj in objs: + try: + obj.close() + except Exception: + pass + + +def convert_args(value): + """Replace handle tuples inside a decoded argument, recursively.""" + if isinstance(value, tuple): + if len(value) == 4 and value[0] in (SHM_TAG, SHM_RO_TAG, BUFFER_TAG) \ + and isinstance(value[1], int): + return from_term(value[0], value[1], value[2], value[3]) + return tuple(convert_args(v) for v in value) + if isinstance(value, list): + return [convert_args(v) for v in value] + if isinstance(value, dict): + return {k: convert_args(v) for k, v in value.items()} + return value diff --git a/rebar.config b/rebar.config index 07d555b..0010a51 100644 --- a/rebar.config +++ b/rebar.config @@ -12,6 +12,13 @@ {deps, []}. +%% iommap is optional at runtime (py_shm); the suites need it. +{profiles, [ + {test, [ + {deps, [{iommap, "1.1.3"}]} + ]} +]}. + {pre_hooks, [ {clean, "rm -f priv/*.so"}, {clean, "rm -rf _build/cmake"}, diff --git a/src/erlang_python_sup.erl b/src/erlang_python_sup.erl index e9e3ebd..1a5c459 100644 --- a/src/erlang_python_sup.erl +++ b/src/erlang_python_sup.erl @@ -66,6 +66,7 @@ init([]) -> ok = py_state:register_callbacks(), ok = py_event_loop:register_callbacks(), ok = py_channel:register_callbacks(), + ok = py_shm:register_callbacks(), %% Callback registry - must start before contexts CallbackSpec = #{ @@ -77,6 +78,16 @@ init([]) -> modules => [py_callback] }, + %% Shared memory regions and shared buffers (py_shm, needs iommap at use) + ShmSpec = #{ + id => py_shm, + start => {py_shm, start_link, []}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [py_shm] + }, + %% Thread worker coordinator (for ThreadPoolExecutor support) ThreadHandlerSpec = #{ id => py_thread_handler, @@ -167,7 +178,7 @@ init([]) -> modules => [py_event_loop_pool] }, - Children = [CallbackSpec, ThreadHandlerSpec, LoggerSpec, TracerSpec, + Children = [CallbackSpec, ShmSpec, ThreadHandlerSpec, LoggerSpec, TracerSpec, ContextSupSpec, ContextRouterInitSpec, WorkerRegistrySpec, WorkerSupSpec, EventLoopSpec, EventLoopPoolSpec], diff --git a/src/py_buffer.erl b/src/py_buffer.erl index edab628..829d703 100644 --- a/src/py_buffer.erl +++ b/src/py_buffer.erl @@ -56,6 +56,7 @@ new/0, new/1, write/2, + write/3, close/1 ]). @@ -75,11 +76,19 @@ new() -> %% %% @param ContentLength Expected total size in bytes, or `undefined' for chunked %% @returns {ok, BufferRef} | {error, Reason} --spec new(non_neg_integer() | undefined) -> {ok, reference()} | {error, term()}. +-spec new(non_neg_integer() | undefined | map()) -> + {ok, reference() | py_shm:buffer()} | {error, term()}. new(undefined) -> py_nif:py_buffer_create(undefined); new(ContentLength) when is_integer(ContentLength), ContentLength >= 0 -> - py_nif:py_buffer_create(ContentLength). + py_nif:py_buffer_create(ContentLength); +new(#{shared := true} = Opts) -> + %% Shared buffer: a py_shm ring, usable in every context mode, + %% including isolated ones. Options: size (ring bytes, default 4 MB), + %% owner (pid whose exit closes it). Needs iommap. + py_shm:buffer_new(maps:remove(shared, Opts)); +new(Opts) when is_map(Opts) -> + new(maps:get(content_length, Opts, undefined)). %% @doc Write data to the buffer. %% @@ -90,10 +99,19 @@ new(ContentLength) when is_integer(ContentLength), ContentLength >= 0 -> %% @param Ref Buffer reference from new/0 or new/1 %% @param Data Binary data to append %% @returns ok | {error, Reason} --spec write(reference(), binary()) -> ok | {error, term()}. +-spec write(reference() | py_shm:buffer(), binary()) -> ok | {error, term()}. +write({'$py_buffer', _, _, _} = Buf, Data) when is_binary(Data) -> + py_shm:buffer_write(Buf, Data); write(Ref, Data) when is_binary(Data) -> py_nif:py_buffer_write(Ref, Data). +%% @doc Write with a timeout (shared buffers block while the ring is full). +-spec write(reference() | py_shm:buffer(), binary(), timeout()) -> ok | {error, term()}. +write({'$py_buffer', _, _, _} = Buf, Data, Timeout) when is_binary(Data) -> + py_shm:buffer_write(Buf, Data, Timeout); +write(Ref, Data, _Timeout) when is_binary(Data) -> + py_nif:py_buffer_write(Ref, Data). + %% @doc Close the buffer (signal end of data). %% %% Sets the EOF flag and wakes up any Python threads waiting for data. @@ -102,6 +120,8 @@ write(Ref, Data) when is_binary(Data) -> %% %% @param Ref Buffer reference %% @returns ok --spec close(reference()) -> ok. +-spec close(reference() | py_shm:buffer()) -> ok. +close({'$py_buffer', _, _, _} = Buf) -> + py_shm:buffer_close(Buf); close(Ref) -> py_nif:py_buffer_close(Ref). diff --git a/src/py_shm.erl b/src/py_shm.erl new file mode 100644 index 0000000..d14e86f --- /dev/null +++ b/src/py_shm.erl @@ -0,0 +1,485 @@ +%% Copyright 2026 Benoit Chesneau +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%% @doc Shared memory regions between Erlang and Python contexts. +%%% +%%% A region is a fixed-size file mapped `MAP_SHARED' through +%%% iommap. Erlang reads it +%%% with no copy (`binary/3') and writes with one copy (`write/3'); Python +%%% maps the same file and sees it as a buffer (`erlang.SharedMemory'), +%%% in every context mode. The handle is a plain term, +%%% `{'$py_shm', Id, Path, Size}', so it travels inside any call argument or +%%% result. +%%% +%%% ``` +%%% {ok, Shm} = py_shm:new(64 * 1024 * 1024), +%%% ok = py_shm:write(Shm, 0, Data), +%%% {ok, Sum} = py_context:call(Ctx, myapp, sum_floats, [Shm]), +%%% Out = py_shm:binary(Shm, 0, 1024), +%%% ok = py_shm:close(Shm). +%%% ''' +%%% +%%% iommap is an optional dependency: add `{iommap, "1.1.3"}' to your deps. +%%% Without it `new/1,2' returns `{error, iommap_not_available}'. +%%% +%%% The module also backs shared `py_buffer's (`py_buffer:new(#{shared => true})'): +%%% a region used as a ring, with the write position and the closed flag in +%%% a header page and flow control through the `_py_buffer_wait' and +%%% `_py_buffer_consumed' callbacks the Python side calls. +-module(py_shm). + +-behaviour(gen_server). + +-export([ + start_link/0, + available/0, + new/1, + new/2, + read_only/1, + write/3, + read/3, + binary/3, + size/1, + close/1, + info/1, + %% Shared buffers (used by py_buffer) + buffer_new/1, + buffer_write/2, + buffer_write/3, + buffer_close/1, + buffer_info/1, + %% Callbacks the Python side uses + register_callbacks/0, + handle_buffer_wait/1, + handle_buffer_consumed/1, + handle_buffer_state/1, + %% Location of region files (also used by isolated contexts) + private_dir/0 +]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). + +-define(TABLE, py_shm_regions). +-define(HEADER, 4096). +-define(DEFAULT_RING, 4 * 1024 * 1024). +-define(DEFAULT_WRITE_TIMEOUT, 30000). +-define(IS_SHM(T), (T =:= '$py_shm' orelse T =:= '$py_shm_ro')). + +-type shm() :: {'$py_shm' | '$py_shm_ro', pos_integer(), binary(), non_neg_integer()}. +-type buffer() :: {'$py_buffer', pos_integer(), binary(), pos_integer()}. +-export_type([shm/0, buffer/0]). + +%% Ring buffer state +-record(buf, { + id :: pos_integer(), + handle :: term(), + ring :: pos_integer(), + wpos = 0 :: non_neg_integer(), %% total bytes written + rpos = 0 :: non_neg_integer(), %% total bytes consumed + closed = false :: boolean(), + readers = [] :: [{gen_server:from(), non_neg_integer()}], + %% Writers waiting for room: {From, Rest, TimerRef} + writers = [] :: [{gen_server:from(), binary(), reference()}] +}). + +-record(state, { + buffers = #{} :: #{pos_integer() => #buf{}} +}). + +%% ============================================================================ +%% API +%% ============================================================================ + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +%% @doc Whether iommap is available: shared memory needs it. +-spec available() -> boolean(). +available() -> + code:ensure_loaded(iommap) =:= {module, iommap}. + +%% @doc Create a region of `Size' bytes owned by the calling process. +-spec new(pos_integer()) -> {ok, shm()} | {error, term()}. +new(Size) -> + new(Size, #{}). + +%% @doc Create a region. Options: `owner' (pid whose exit closes the region, +%% default the caller); `writable' (default `true'): with `false' Python maps +%% the region read-only, so a buggy or hostile callee cannot change it. +-spec new(pos_integer(), map()) -> {ok, shm()} | {error, term()}. +new(Size, Opts) when is_integer(Size), Size > 0, is_map(Opts) -> + Owner = maps:get(owner, Opts, self()), + case gen_server:call(?MODULE, {new, Size, Owner}, infinity) of + {ok, {'$py_shm', Id, Path, Size}} when map_get(writable, Opts) =:= false -> + {ok, {'$py_shm_ro', Id, Path, Size}}; + Other -> + Other + end. + +%% @doc Read-only view of a handle for the Python side; Erlang keeps writing. +-spec read_only(shm()) -> shm(). +read_only({Tag, Id, Path, Size}) when ?IS_SHM(Tag) -> + {'$py_shm_ro', Id, Path, Size}. + +%% @doc Copy `Data' into the region at `Offset'. +-spec write(shm(), non_neg_integer(), binary()) -> ok | {error, term()}. +write({Tag, Id, _, Size}, Offset, Data) when ?IS_SHM(Tag), is_binary(Data) -> + case Offset + byte_size(Data) > Size of + true -> {error, out_of_bounds}; + false -> + case lookup(Id) of + {ok, Handle} -> iommap(pwrite, [Handle, Offset, Data]); + error -> {error, closed} + end + end. + +%% @doc Copy `Len' bytes out of the region. +-spec read(shm(), non_neg_integer(), non_neg_integer()) -> {ok, binary()} | {error, term()}. +read({Tag, Id, _, Size}, Offset, Len) when ?IS_SHM(Tag) -> + case Offset + Len > Size of + true -> {error, out_of_bounds}; + false -> + case lookup(Id) of + {ok, Handle} -> iommap(pread, [Handle, Offset, Len]); + error -> {error, closed} + end + end. + +%% @doc A binary over the region, no copy. It stays valid after `close/1' +%% (the mapping is kept as long as the binary is referenced). Its bytes are +%% the region's: they change when Python writes, so treat it as a snapshot +%% only once the callee is done with the handle, or use `read/3' for a copy. +-spec binary(shm(), non_neg_integer(), non_neg_integer()) -> binary(). +binary({Tag, Id, _, Size}, Offset, Len) when ?IS_SHM(Tag) -> + Offset + Len =< Size orelse error(out_of_bounds), + case lookup(Id) of + {ok, Handle} -> + case iommap(region_binary, [Handle, Offset, Len]) of + {ok, Bin} -> Bin; + {error, Reason} -> error(Reason) + end; + error -> + error(closed) + end. + +-spec size(shm()) -> non_neg_integer(). +size({Tag, _, _, Size}) when ?IS_SHM(Tag) -> + Size. + +%% @doc Close the region: the file is removed and the iommap handle closed. +%% Python mappings stay valid until the wrapper is closed or collected. +-spec close(shm() | buffer()) -> ok. +close({Tag, Id, _, _}) when ?IS_SHM(Tag) -> + gen_server:call(?MODULE, {close, Id}, infinity); +close({'$py_buffer', _, _, _} = Buf) -> + buffer_close(Buf). + +-spec info(shm()) -> {ok, map()} | {error, term()}. +info({Tag, Id, Path, Size}) when ?IS_SHM(Tag) -> + case lookup(Id) of + {ok, _} -> {ok, #{id => Id, path => Path, size => Size}}; + error -> {error, closed} + end. + +%% ---- shared buffers -------------------------------------------------------- + +%% @doc Create a shared streaming buffer (ring of `RingSize' bytes). +-spec buffer_new(map()) -> {ok, buffer()} | {error, term()}. +buffer_new(Opts) when is_map(Opts) -> + Ring = maps:get(size, Opts, ?DEFAULT_RING), + Owner = maps:get(owner, Opts, self()), + gen_server:call(?MODULE, {buffer_new, Ring, Owner}, infinity). + +-spec buffer_write(buffer(), binary()) -> ok | {error, term()}. +buffer_write(Buf, Data) -> + buffer_write(Buf, Data, ?DEFAULT_WRITE_TIMEOUT). + +%% @doc Append `Data'; blocks up to `Timeout' ms while the ring is full. +-spec buffer_write(buffer(), binary(), timeout()) -> ok | {error, term()}. +buffer_write({'$py_buffer', Id, _, _}, Data, Timeout) when is_binary(Data) -> + gen_server:call(?MODULE, {buffer_write, Id, Data, Timeout}, infinity). + +-spec buffer_close(buffer()) -> ok. +buffer_close({'$py_buffer', Id, _, _}) -> + gen_server:call(?MODULE, {buffer_close, Id}, infinity). + +-spec buffer_info(buffer()) -> {ok, map()} | {error, term()}. +buffer_info({'$py_buffer', Id, _, _}) -> + gen_server:call(?MODULE, {buffer_info, Id}, infinity). + +%% ---- callbacks used from Python -------------------------------------------- + +%% @private Registered by the supervisor once py_callback is up. +register_callbacks() -> + py_callback:register(<<"_py_buffer_wait">>, {?MODULE, handle_buffer_wait}), + py_callback:register(<<"_py_buffer_consumed">>, {?MODULE, handle_buffer_consumed}), + py_callback:register(<<"_py_buffer_state">>, {?MODULE, handle_buffer_state}), + ok. + +%% @private Block until the write position passes `ReadPos' or the buffer is +%% closed. Returns `{WPos, Closed}'. +handle_buffer_wait([Id, ReadPos]) -> + gen_server:call(?MODULE, {buffer_wait, Id, ReadPos}, infinity). + +%% @private The reader consumed `N' bytes: make room for writers. +handle_buffer_consumed([Id, N]) -> + gen_server:call(?MODULE, {buffer_consumed, Id, N}, infinity). + +%% @private Current `{WPos, Closed}' without waiting. +handle_buffer_state([Id]) -> + gen_server:call(?MODULE, {buffer_state, Id}, infinity). + +%% @doc Private directory for region files: `/dev/shm' when it exists +%% (memory backed), else a 0700 directory under `TMPDIR'. +-spec private_dir() -> string(). +private_dir() -> + Base = case filelib:is_dir("/dev/shm") of + true -> "/dev/shm"; + false -> + case os:getenv("TMPDIR") of + false -> "/tmp"; + T -> T + end + end, + Dir = filename:join(Base, "erlang_python_" ++ os:getpid()), + ok = filelib:ensure_dir(filename:join(Dir, "x")), + _ = file:change_mode(Dir, 8#700), + Dir. + +%% ============================================================================ +%% gen_server +%% ============================================================================ + +init([]) -> + ?TABLE = ets:new(?TABLE, [named_table, protected, set, {read_concurrency, true}]), + {ok, #state{}}. + +handle_call({new, Size, Owner}, _From, State) -> + {reply, create_region(Size, Owner), State}; + +handle_call({close, Id}, _From, State) -> + close_region(Id), + {reply, ok, State}; + +handle_call({buffer_new, Ring, Owner}, _From, #state{buffers = Bufs} = State) -> + case create_region(?HEADER + Ring, Owner) of + {ok, {'$py_shm', Id, Path, _}} -> + {ok, Handle} = lookup(Id), + Buf = #buf{id = Id, handle = Handle, ring = Ring}, + ok = write_header(Buf), + {reply, {ok, {'$py_buffer', Id, Path, Ring}}, State#state{buffers = Bufs#{Id => Buf}}}; + {error, _} = Err -> + {reply, Err, State} + end; + +handle_call({buffer_write, Id, Data, Timeout}, From, #state{buffers = Bufs} = State) -> + case Bufs of + #{Id := #buf{closed = true}} -> + {reply, {error, closed}, State}; + #{Id := Buf} -> + case do_write(Buf, Data) of + {ok, Buf1} -> + {reply, ok, State#state{buffers = Bufs#{Id => Buf1}}}; + {partial, Buf1, Rest} -> + Timer = erlang:send_after(Timeout, self(), {write_timeout, Id, From}), + Buf2 = Buf1#buf{writers = Buf1#buf.writers ++ [{From, Rest, Timer}]}, + {noreply, State#state{buffers = Bufs#{Id => Buf2}}} + end; + _ -> + {reply, {error, closed}, State} + end; + +handle_call({buffer_close, Id}, _From, #state{buffers = Bufs} = State) -> + case Bufs of + #{Id := Buf} -> + Buf1 = Buf#buf{closed = true}, + ok = write_header(Buf1), + %% Readers learn about EOF; writers still waiting fail + [gen_server:reply(R, {Buf1#buf.wpos, true}) || {R, _} <- Buf1#buf.readers], + [begin erlang:cancel_timer(T), gen_server:reply(W, {error, closed}) end + || {W, _, T} <- Buf1#buf.writers], + {reply, ok, State#state{buffers = Bufs#{Id => Buf1#buf{readers = [], writers = []}}}}; + _ -> + {reply, ok, State} + end; + +handle_call({buffer_wait, Id, ReadPos}, From, #state{buffers = Bufs} = State) -> + case Bufs of + #{Id := #buf{wpos = W, closed = C}} when W > ReadPos; C -> + {reply, {W, C}, State}; + #{Id := Buf} -> + Buf1 = Buf#buf{readers = [{From, ReadPos} | Buf#buf.readers]}, + {noreply, State#state{buffers = Bufs#{Id => Buf1}}}; + _ -> + {reply, {error, closed}, State} + end; + +handle_call({buffer_consumed, Id, N}, _From, #state{buffers = Bufs} = State) -> + case Bufs of + #{Id := Buf} -> + Buf1 = Buf#buf{rpos = Buf#buf.rpos + N}, + Buf2 = drain_writers(Buf1), + {reply, ok, State#state{buffers = Bufs#{Id => Buf2}}}; + _ -> + {reply, {error, closed}, State} + end; + +handle_call({buffer_state, Id}, _From, #state{buffers = Bufs} = State) -> + case Bufs of + #{Id := #buf{wpos = W, closed = C}} -> {reply, {W, C}, State}; + _ -> {reply, {error, closed}, State} + end; + +handle_call({buffer_info, Id}, _From, #state{buffers = Bufs} = State) -> + case Bufs of + #{Id := #buf{wpos = W, rpos = R, closed = C, ring = Ring}} -> + {reply, {ok, #{written => W, consumed => R, closed => C, ring => Ring, + pending_writers => length((maps:get(Id, Bufs))#buf.writers)}}, State}; + _ -> + {reply, {error, closed}, State} + end; + +handle_call(_Req, _From, State) -> + {reply, {error, badarg}, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info({write_timeout, Id, From}, #state{buffers = Bufs} = State) -> + case Bufs of + #{Id := #buf{writers = Ws} = Buf} -> + case lists:keytake(From, 1, Ws) of + {value, {From, _Rest, _T}, Rest} -> + gen_server:reply(From, {error, timeout}), + {noreply, State#state{buffers = Bufs#{Id => Buf#buf{writers = Rest}}}}; + false -> + {noreply, State} + end; + _ -> + {noreply, State} + end; +handle_info({'DOWN', _Mon, process, Owner, _Reason}, #state{buffers = Bufs} = State) -> + Ids = [Id || {Id, _, _, _, O} <- ets:tab2list(?TABLE), O =:= Owner], + [close_region(Id) || Id <- Ids], + Bufs1 = maps:without(Ids, Bufs), + {noreply, State#state{buffers = Bufs1}}; +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + [close_region(Id) || {Id, _, _, _, _} <- ets:tab2list(?TABLE)], + ok. + +%% ============================================================================ +%% Internal +%% ============================================================================ + +%% iommap is an optional dependency: call it indirectly so xref and +%% dialyzer do not require it in the default profile. +iommap(Fun, Args) -> + apply(iommap, Fun, Args). + +lookup(Id) -> + try ets:lookup(?TABLE, Id) of + [{Id, Handle, _Path, _Size, _Owner}] -> {ok, Handle}; + [] -> error + catch + error:badarg -> error + end. + +create_region(Size, Owner) -> + case available() of + false -> + {error, iommap_not_available}; + true -> + Id = erlang:unique_integer([positive]), + Path = filename:join(private_dir(), "shm_" ++ integer_to_list(Id)), + case iommap(open, [Path, read_write, [create, truncate, {size, Size}, shared]]) of + {ok, Handle} -> + _ = file:change_mode(Path, 8#600), + _ = erlang:monitor(process, Owner), + ets:insert(?TABLE, {Id, Handle, Path, Size, Owner}), + {ok, {'$py_shm', Id, unicode:characters_to_binary(Path), Size}}; + {error, Reason} -> + {error, {shm_open_failed, Reason}} + end + end. + +close_region(Id) -> + case ets:take(?TABLE, Id) of + [{Id, Handle, Path, _Size, _Owner}] -> + _ = file:delete(Path), + _ = iommap(close, [Handle]), + ok; + [] -> + ok + end. + +%% Header page: <> +write_header(#buf{handle = H, wpos = W, closed = C, ring = Ring}) -> + Flag = case C of true -> 1; false -> 0 end, + iommap(pwrite, [H, 0, <>]). + +%% Copy as much of Data as fits, advance wpos, wake readers. +do_write(#buf{ring = Ring, wpos = W, rpos = R} = Buf, Data) -> + Free = Ring - (W - R), + Size = byte_size(Data), + Take = min(Free, Size), + Buf1 = case Take > 0 of + true -> + <> = Data, + ok = ring_write(Buf, W, Chunk), + B = Buf#buf{wpos = W + Take}, + ok = write_header(B), + wake_readers(B); + false -> + Buf + end, + case Take =:= Size of + true -> {ok, Buf1}; + false -> + <<_:Take/binary, Rest/binary>> = Data, + {partial, Buf1, Rest} + end. + +ring_write(#buf{handle = H, ring = Ring}, Pos, Chunk) -> + Off = Pos rem Ring, + Size = byte_size(Chunk), + case Off + Size =< Ring of + true -> + iommap(pwrite, [H, ?HEADER + Off, Chunk]); + false -> + First = Ring - Off, + <> = Chunk, + ok = iommap(pwrite, [H, ?HEADER + Off, A]), + iommap(pwrite, [H, ?HEADER, B]) + end. + +wake_readers(#buf{readers = Readers, wpos = W, closed = C} = Buf) -> + {Ready, Waiting} = lists:partition(fun({_, Pos}) -> W > Pos end, Readers), + [gen_server:reply(From, {W, C}) || {From, _} <- Ready], + Buf#buf{readers = Waiting}. + +%% Room was made: continue pending writers in order. +drain_writers(#buf{writers = []} = Buf) -> + Buf; +drain_writers(#buf{writers = [{From, Rest, Timer} | Others]} = Buf) -> + case do_write(Buf#buf{writers = Others}, Rest) of + {ok, Buf1} -> + erlang:cancel_timer(Timer), + gen_server:reply(From, ok), + drain_writers(Buf1); + {partial, Buf1, Rest1} -> + Buf1#buf{writers = [{From, Rest1, Timer} | Others]} + end. diff --git a/test/py_isolated_buffer_SUITE.erl b/test/py_isolated_buffer_SUITE.erl new file mode 100644 index 0000000..5c09e80 --- /dev/null +++ b/test/py_isolated_buffer_SUITE.erl @@ -0,0 +1,286 @@ +%%% @doc Common Test suite for shared py_buffers (`py_buffer:new(#{shared => true})'): +%%% the streaming input buffer over shared memory, in worker and isolated +%%% contexts. Mirrors py_buffer_SUITE where the case applies. +-module(py_isolated_buffer_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([ + all/0, + groups/0, + init_per_suite/1, + end_per_suite/1, + init_per_group/2, + end_per_group/2 +]). + +-export([ + test_read_all/1, + test_read_n/1, + test_readline/1, + test_readlines_and_iter/1, + test_read_blocks_until_write/1, + test_read_nonblock_and_eof/1, + test_backpressure/1, + test_write_timeout/1, + test_large_body_throughput/1, + test_close_while_reading/1, + test_wsgi_input_in_environ/1, + test_read_with_nested_callback/1, + test_write_after_close/1, + test_restart_mid_body/1, + test_native_buffer_refused_in_isolated/1 +]). + +-define(MOD, py_test_isolated_shm). +-define(MB, (1024 * 1024)). + +all() -> + [{group, worker}, {group, isolated}, {group, isolated_only}]. + +groups() -> + Both = [ + test_read_all, + test_read_n, + test_readline, + test_readlines_and_iter, + test_read_blocks_until_write, + test_read_nonblock_and_eof, + test_backpressure, + test_write_timeout, + test_large_body_throughput, + test_close_while_reading, + test_wsgi_input_in_environ, + test_read_with_nested_callback, + test_write_after_close + ], + [{worker, [], Both}, + {isolated, [], Both}, + {isolated_only, [], [test_restart_mid_body, test_native_buffer_refused_in_isolated]}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(erlang_python), + case py_shm:available() of + true -> [{test_dir, filename:join(code:lib_dir(erlang_python), "test")} | Config]; + false -> {skip, "iommap not available"} + end. + +end_per_suite(_Config) -> + ok = application:stop(erlang_python), + ok. + +init_per_group(isolated_only, Config) -> [{mode, isolated} | Config]; +init_per_group(Mode, Config) -> [{mode, Mode} | Config]. + +end_per_group(_Group, _Config) -> + ok. + +%%% ============================================================================ + +test_read_all(Config) -> + C = new_ctx(Config), + {ok, Buf} = py_buffer:new(#{shared => true}), + {ok, <<"SharedBuffer">>} = py_context:call(C, ?MOD, buf_kind, [Buf]), + ok = py_buffer:write(Buf, <<"hello ">>), + ok = py_buffer:write(Buf, <<"world">>), + ok = py_buffer:close(Buf), + {ok, <<"hello world">>} = py_context:call(C, ?MOD, buf_read_all, [Buf]), + %% EOF: further reads return empty + {ok, <<>>} = py_context:call(C, ?MOD, buf_read_all, [Buf]), + stop(C). + +test_read_n(Config) -> + C = new_ctx(Config), + {ok, Buf} = py_buffer:new(#{shared => true}), + ok = py_buffer:write(Buf, <<"abcdefghij">>), + ok = py_buffer:close(Buf), + {ok, <<"abc">>} = py_context:call(C, ?MOD, buf_read_n, [Buf, 3]), + {ok, [<<"defg">>, <<"hij">>]} = py_context:call(C, ?MOD, buf_read_chunks, [Buf, 4]), + stop(C). + +test_readline(Config) -> + C = new_ctx(Config), + {ok, Buf} = py_buffer:new(#{shared => true}), + ok = py_buffer:write(Buf, <<"line one\nline ">>), + ok = py_buffer:write(Buf, <<"two\nno newline">>), + ok = py_buffer:close(Buf), + {ok, <<"line one\n">>} = py_context:call(C, ?MOD, buf_readline, [Buf]), + {ok, <<"line two\n">>} = py_context:call(C, ?MOD, buf_readline, [Buf]), + {ok, <<"no newline">>} = py_context:call(C, ?MOD, buf_readline, [Buf]), + {ok, <<>>} = py_context:call(C, ?MOD, buf_readline, [Buf]), + stop(C). + +test_readlines_and_iter(Config) -> + C = new_ctx(Config), + {ok, B1} = py_buffer:new(#{shared => true}), + ok = py_buffer:write(B1, <<"a\nb\nc">>), + ok = py_buffer:close(B1), + {ok, [<<"a\n">>, <<"b\n">>, <<"c">>]} = py_context:call(C, ?MOD, buf_readlines, [B1]), + {ok, B2} = py_buffer:new(#{shared => true}), + ok = py_buffer:write(B2, <<"x\ny\n">>), + ok = py_buffer:close(B2), + {ok, [<<"x\n">>, <<"y\n">>]} = py_context:call(C, ?MOD, buf_iter, [B2]), + stop(C). + +%% @doc A read issued before any data blocks, then returns once written. +test_read_blocks_until_write(Config) -> + C = new_ctx(Config), + {ok, Buf} = py_buffer:new(#{shared => true}), + Self = self(), + spawn_link(fun() -> Self ! {got, py_context:call(C, ?MOD, buf_read_n, [Buf, 5], #{}, 10000)} end), + receive {got, _} -> ct:fail(read_returned_without_data) after 300 -> ok end, + ok = py_buffer:write(Buf, <<"data!">>), + receive {got, {ok, <<"data!">>}} -> ok after 5000 -> ct:fail(read_did_not_wake) end, + ok = py_buffer:close(Buf), + stop(C). + +test_read_nonblock_and_eof(Config) -> + C = new_ctx(Config), + {ok, Buf} = py_buffer:new(#{shared => true}), + {ok, <<>>} = py_context:call(C, ?MOD, buf_read_nonblock, [Buf]), + {ok, false} = py_context:call(C, ?MOD, buf_at_eof, [Buf]), + ok = py_buffer:write(Buf, <<"ready">>), + {ok, <<"ready">>} = py_context:call(C, ?MOD, buf_read_nonblock, [Buf]), + ok = py_buffer:close(Buf), + {ok, true} = py_context:call(C, ?MOD, buf_at_eof, [Buf]), + stop(C). + +%% @doc Body larger than the ring: the writer blocks while the reader +%% catches up and everything arrives in order. +test_backpressure(Config) -> + C = new_ctx(Config), + Ring = 64 * 1024, + {ok, Buf} = py_buffer:new(#{shared => true, size => Ring}), + Total = 10 * Ring + 123, + Self = self(), + spawn_link(fun() -> + Self ! {read, py_context:call(C, ?MOD, buf_consume_checksum, [Buf, 7000], #{}, 60000)} + end), + Chunks = [crypto:strong_rand_bytes(11111) || _ <- lists:seq(1, Total div 11111)], + Last = crypto:strong_rand_bytes(Total rem 11111), + All = iolist_to_binary(Chunks ++ [Last]), + T0 = erlang:monotonic_time(millisecond), + [ok = py_buffer:write(Buf, Ch) || Ch <- Chunks ++ [Last]], + ok = py_buffer:close(Buf), + ct:log("wrote ~p bytes through a ~p ring in ~p ms", + [Total, Ring, erlang:monotonic_time(millisecond) - T0]), + Expected = {Total, checksum(All)}, + receive {read, {ok, {Got, Sum}}} -> Expected = {Got, Sum} + after 60000 -> ct:fail(reader_hung) + end, + stop(C). + +test_write_timeout(Config) -> + C = new_ctx(Config), + {ok, Buf} = py_buffer:new(#{shared => true, size => 4096}), + ok = py_buffer:write(Buf, binary:copy(<<1>>, 4096)), + %% Nobody reads: the next write cannot fit and times out + {error, timeout} = py_buffer:write(Buf, <<"more">>, 300), + %% Reading frees the ring and later writes succeed + Self = self(), + spawn_link(fun() -> Self ! {n, py_context:call(C, ?MOD, buf_consume_len, [Buf, 4096], #{}, 10000)} end), + ok = py_buffer:write(Buf, <<"more">>, 5000), + ok = py_buffer:close(Buf), + receive {n, {ok, 4100}} -> ok after 10000 -> ct:fail(reader_hung) end, + stop(C). + +test_large_body_throughput(Config) -> + C = new_ctx(Config), + Size = 64 * ?MB, + Body = crypto:strong_rand_bytes(Size), + {ok, Buf} = py_buffer:new(#{shared => true, size => 8 * ?MB}), + Self = self(), + spawn_link(fun() -> + Self ! {read, py_context:call(C, ?MOD, buf_consume_len, [Buf, ?MB], #{}, 120000)} + end), + T0 = erlang:monotonic_time(microsecond), + [ok = py_buffer:write(Buf, Chunk) || <> <= Body], + ok = py_buffer:close(Buf), + receive {read, {ok, Size}} -> ok after 120000 -> ct:fail(reader_hung) end, + Us = erlang:monotonic_time(microsecond) - T0, + ct:log("64 MB through a shared buffer (~p): ~.1f ms, ~.1f MB/s", + [?config(mode, Config), Us / 1000, Size / ?MB / (Us / 1.0e6)]), + ct:print("shared buffer 64 MB (~p): ~.1f ms", [?config(mode, Config), Us / 1000]), + stop(C). + +test_close_while_reading(Config) -> + C = new_ctx(Config), + {ok, Buf} = py_buffer:new(#{shared => true}), + Self = self(), + spawn_link(fun() -> Self ! {got, py_context:call(C, ?MOD, buf_read_all, [Buf], #{}, 10000)} end), + timer:sleep(200), + ok = py_buffer:close(Buf), + receive {got, {ok, <<>>}} -> ok after 5000 -> ct:fail(read_did_not_return_on_close) end, + stop(C). + +test_wsgi_input_in_environ(Config) -> + C = new_ctx(Config), + {ok, Buf} = py_buffer:new(#{shared => true}), + ok = py_buffer:write(Buf, <<"{\"json\": true}">>), + ok = py_buffer:close(Buf), + Environ = #{<<"method">> => <<"POST">>, <<"wsgi.input">> => Buf}, + {ok, {<<"POST">>, 14, <<"{\"json\":">>}} = py_context:call(C, ?MOD, buf_from_environ, [Environ]), + stop(C). + +%% @doc A callback re-entering the context while a read is in progress. +test_read_with_nested_callback(Config) -> + C = new_ctx(Config), + py_callback:register(<<"shm_double">>, fun([X]) -> X * 2 end), + {ok, Buf} = py_buffer:new(#{shared => true}), + ok = py_buffer:write(Buf, <<"headrest of the body">>), + ok = py_buffer:close(Buf), + {ok, {<<"head">>, 42, 16}} = py_context:call(C, ?MOD, buf_read_with_callback, [Buf, <<"shm_double">>]), + py_callback:unregister(<<"shm_double">>), + stop(C). + +test_write_after_close(Config) -> + C = new_ctx(Config), + {ok, Buf} = py_buffer:new(#{shared => true}), + ok = py_buffer:close(Buf), + {error, closed} = py_buffer:write(Buf, <<"late">>), + {ok, <<>>} = py_context:call(C, ?MOD, buf_read_all, [Buf]), + stop(C). + +%% @doc The child dies mid-body; the new child continues from the read +%% position (unread data is still in the ring). +test_restart_mid_body(Config) -> + C = new_ctx(Config), + {ok, Buf} = py_buffer:new(#{shared => true}), + ok = py_buffer:write(Buf, <<"part one|">>), + {ok, <<"part one|">>} = py_context:call(C, ?MOD, buf_read_n, [Buf, 9]), + ok = py_buffer:write(Buf, <<"part two">>), + ok = py_buffer:close(Buf), + ok = py_context:kill(C), + {ok, <<"part two">>} = py_context:call(C, ?MOD, buf_read_all, [Buf]), + stop(C). + +test_native_buffer_refused_in_isolated(Config) -> + C = new_ctx(Config), + {ok, Native} = py_buffer:new(), + ok = py_buffer:write(Native, <<"x">>), + ok = py_buffer:close(Native), + %% A NIF resource cannot cross: it does not arrive as a buffer + {ok, Kind} = py_context:call(C, ?MOD, buf_kind, [Native]), + true = Kind =/= <<"SharedBuffer">> andalso Kind =/= <<"PyBuffer">>, + stop(C). + +%%% ============================================================================ + +checksum(Bin) -> + lists:foldl(fun(B, Acc) -> (Acc + B) rem 1000003 end, 0, binary_to_list(Bin)). + +new_ctx(Config) -> + Mode = ?config(mode, Config), + TestDir = ?config(test_dir, Config), + {ok, C} = py_context:new(#{mode => Mode, paths => [TestDir]}), + case Mode of + worker -> + ok = py_context:exec(C, iolist_to_binary(io_lib:format( + "import sys\nif '~s' not in sys.path: sys.path.insert(0, '~s')", [TestDir, TestDir]))); + _ -> ok + end, + C. + +stop(C) -> + ok = py_context:stop(C), + ok. diff --git a/test/py_isolated_shm_SUITE.erl b/test/py_isolated_shm_SUITE.erl new file mode 100644 index 0000000..4385c0e --- /dev/null +++ b/test/py_isolated_shm_SUITE.erl @@ -0,0 +1,362 @@ +%%% @doc Common Test suite for py_shm: shared memory regions between Erlang +%%% and Python contexts, in worker and isolated mode. +-module(py_isolated_shm_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([ + all/0, + groups/0, + init_per_suite/1, + end_per_suite/1, + init_per_group/2, + end_per_group/2 +]). + +-export([ + test_available/1, + test_erlang_round_trip/1, + test_bounds/1, + test_close_idempotent_binary_survives/1, + test_owner_death_closes/1, + test_unknown_handle/1, + test_pass_to_python/1, + test_nested_in_structure/1, + test_python_sees_later_write/1, + test_python_writes_erlang_reads/1, + test_returned_handle/1, + test_mapped_once/1, + test_numpy/1, + test_two_contexts_share/1, + test_mixed_pool_share/1, + test_size_mismatch_refused/1, + test_closed_wrapper_raises/1, + test_read_only_handle/1, + test_restart_remaps/1, + test_churn_no_leak/1 +]). + +-define(MOD, py_test_isolated_shm). +-define(MB, (1024 * 1024)). + +all() -> + [{group, erlang}, {group, worker}, {group, isolated}, {group, isolated_only}]. + +groups() -> + ErlangOnly = [ + test_available, + test_erlang_round_trip, + test_bounds, + test_close_idempotent_binary_survives, + test_owner_death_closes, + test_unknown_handle + ], + Both = [ + test_pass_to_python, + test_nested_in_structure, + test_python_sees_later_write, + test_python_writes_erlang_reads, + test_returned_handle, + test_mapped_once, + test_numpy, + test_two_contexts_share, + test_size_mismatch_refused, + test_closed_wrapper_raises, + test_read_only_handle + ], + IsolatedOnly = [ + test_mixed_pool_share, + test_restart_remaps, + test_churn_no_leak + ], + [{erlang, [], ErlangOnly}, + {worker, [], Both}, + {isolated, [], Both}, + {isolated_only, [], IsolatedOnly}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(erlang_python), + case py_shm:available() of + true -> [{test_dir, filename:join(code:lib_dir(erlang_python), "test")} | Config]; + false -> {skip, "iommap not available"} + end. + +end_per_suite(_Config) -> + ok = application:stop(erlang_python), + ok. + +init_per_group(erlang, Config) -> Config; +init_per_group(isolated_only, Config) -> [{mode, isolated} | Config]; +init_per_group(Mode, Config) -> [{mode, Mode} | Config]. + +end_per_group(_Group, _Config) -> + ok. + +%%% ============================================================================ +%%% Erlang side only +%%% ============================================================================ + +test_available(_Config) -> + true = py_shm:available(), + ok. + +test_erlang_round_trip(_Config) -> + {ok, Shm} = py_shm:new(?MB), + ?MB = py_shm:size(Shm), + {'$py_shm', _, Path, ?MB} = Shm, + true = filelib:is_file(Path), + Data = crypto:strong_rand_bytes(4096), + ok = py_shm:write(Shm, 100, Data), + {ok, Data} = py_shm:read(Shm, 100, 4096), + Data = py_shm:binary(Shm, 100, 4096), + {ok, #{size := ?MB}} = py_shm:info(Shm), + ok = py_shm:close(Shm), + false = filelib:is_file(Path), + ok. + +test_bounds(_Config) -> + {ok, Shm} = py_shm:new(4096), + {error, out_of_bounds} = py_shm:write(Shm, 4000, <<0:(200 * 8)>>), + {error, out_of_bounds} = py_shm:read(Shm, 4000, 200), + ok = py_shm:write(Shm, 4000, <<0:(96 * 8)>>), + ok = py_shm:close(Shm), + ok. + +test_close_idempotent_binary_survives(_Config) -> + {ok, Shm} = py_shm:new(4096), + ok = py_shm:write(Shm, 0, <<"still here">>), + Bin = py_shm:binary(Shm, 0, 10), + ok = py_shm:close(Shm), + ok = py_shm:close(Shm), + <<"still here">> = Bin, + {error, closed} = py_shm:write(Shm, 0, <<"x">>), + {error, closed} = py_shm:info(Shm), + ok. + +test_owner_death_closes(_Config) -> + Self = self(), + Owner = spawn(fun() -> + {ok, Shm} = py_shm:new(4096), + Self ! {shm, Shm}, + receive die -> ok end + end), + Shm = receive {shm, S} -> S after 5000 -> ct:fail(no_shm) end, + {'$py_shm', _, Path, _} = Shm, + true = filelib:is_file(Path), + Owner ! die, + wait_until(fun() -> not filelib:is_file(Path) end, 5000), + {error, closed} = py_shm:info(Shm), + ok. + +test_unknown_handle(_Config) -> + Fake = {'$py_shm', 999999999, <<"/nonexistent">>, 10}, + {error, closed} = py_shm:read(Fake, 0, 1), + ok = py_shm:close(Fake), + ok. + +%%% ============================================================================ +%%% Erlang <-> Python (worker and isolated) +%%% ============================================================================ + +test_pass_to_python(Config) -> + C = new_ctx(Config), + {ok, Shm} = py_shm:new(?MB), + Data = binary:copy(<<7>>, 1000), + ok = py_shm:write(Shm, 0, Data), + {ok, <<"SharedMemory">>} = py_context:call(C, ?MOD, kind, [Shm]), + {ok, ?MB} = py_context:call(C, ?MOD, shm_len, [Shm]), + {ok, 7000} = py_context:call(C, ?MOD, shm_sum, [Shm, 1000]), + {ok, Data} = py_context:call(C, ?MOD, shm_read, [Shm, 0, 1000]), + ok = py_shm:close(Shm), + stop(C). + +test_nested_in_structure(Config) -> + C = new_ctx(Config), + {ok, A} = py_shm:new(4096), + {ok, B} = py_shm:new(8192), + {ok, [4096, 8192]} = py_context:call(C, ?MOD, shm_in_structure, + [#{regions => [A, B], label => x}]), + py_shm:close(A), py_shm:close(B), + stop(C). + +test_python_sees_later_write(Config) -> + C = new_ctx(Config), + {ok, Shm} = py_shm:new(4096), + ok = py_shm:write(Shm, 0, <<"first">>), + {ok, <<"first">>} = py_context:call(C, ?MOD, shm_read, [Shm, 0, 5]), + ok = py_shm:write(Shm, 0, <<"later">>), + {ok, <<"later">>} = py_context:call(C, ?MOD, shm_read, [Shm, 0, 5]), + py_shm:close(Shm), + stop(C). + +test_python_writes_erlang_reads(Config) -> + C = new_ctx(Config), + {ok, Shm} = py_shm:new(?MB), + {ok, 11} = py_context:call(C, ?MOD, shm_write, [Shm, 10, {bytes, <<"from python">>}]), + <<"from python">> = py_shm:binary(Shm, 10, 11), + {ok, ?MB} = py_context:call(C, ?MOD, shm_fill, [Shm, 42, ?MB]), + Bin = py_shm:binary(Shm, 0, ?MB), + Bin = binary:copy(<<42>>, ?MB), + py_shm:close(Shm), + stop(C). + +test_returned_handle(Config) -> + C = new_ctx(Config), + {ok, Shm} = py_shm:new(4096), + {ok, Shm} = py_context:call(C, ?MOD, shm_identity, [Shm]), + py_shm:close(Shm), + stop(C). + +test_mapped_once(Config) -> + C = new_ctx(Config), + {ok, Shm} = py_shm:new(4096), + {ok, N0} = py_context:call(C, ?MOD, map_count, []), + {ok, _} = py_context:call(C, ?MOD, shm_len, [Shm]), + {ok, N1} = py_context:call(C, ?MOD, map_count, []), + {ok, _} = py_context:call(C, ?MOD, shm_len, [Shm]), + {ok, N2} = py_context:call(C, ?MOD, map_count, []), + N1 = N0 + 1, + N1 = N2, + py_shm:close(Shm), + stop(C). + +test_numpy(Config) -> + C = new_ctx(Config), + case py_context:eval(C, <<"__import__('importlib.util').util.find_spec('numpy') is not None">>) of + {ok, true} -> + {ok, Shm} = py_shm:new(?MB), + ok = py_shm:write(Shm, 0, binary:copy(<<3>>, 1000)), + {ok, 3000} = py_context:call(C, ?MOD, shm_numpy_sum, [Shm, 1000]), + {ok, Expected} = py_context:call(C, ?MOD, shm_numpy_write, [Shm, 256]), + Expected = lists:sum(lists:seq(0, 255)), + Bin = py_shm:binary(Shm, 0, 256), + Bin = list_to_binary(lists:seq(0, 255)), + py_shm:close(Shm), + stop(C); + _ -> + stop(C), + {skip, "numpy not installed"} + end. + +test_two_contexts_share(Config) -> + C1 = new_ctx(Config), + C2 = new_ctx(Config), + {ok, Shm} = py_shm:new(4096), + {ok, 5} = py_context:call(C1, ?MOD, shm_write, [Shm, 0, {bytes, <<"hello">>}]), + {ok, <<"hello">>} = py_context:call(C2, ?MOD, shm_read, [Shm, 0, 5]), + py_shm:close(Shm), + stop(C1), stop(C2). + +test_mixed_pool_share(Config) -> + Iso = new_ctx(Config), + {ok, W} = py_context:new(#{mode => worker}), + ok = py_context:exec(W, add_path(Config)), + {ok, Shm} = py_shm:new(4096), + {ok, 6} = py_context:call(W, ?MOD, shm_write, [Shm, 0, {bytes, <<"worker">>}]), + {ok, <<"worker">>} = py_context:call(Iso, ?MOD, shm_read, [Shm, 0, 6]), + {ok, 8} = py_context:call(Iso, ?MOD, shm_write, [Shm, 0, {bytes, <<"isolated">>}]), + {ok, <<"isolated">>} = py_context:call(W, ?MOD, shm_read, [Shm, 0, 8]), + py_shm:close(Shm), + py_context:stop(W), + stop(Iso). + +%% @doc A handle whose file has a different size than it claims is refused +%% on map (no SIGBUS later). +test_size_mismatch_refused(Config) -> + C = new_ctx(Config), + {ok, {'$py_shm', Id, Path, _} = Shm} = py_shm:new(4096), + Lie = {'$py_shm', Id, Path, 8192}, + case py_context:call(C, ?MOD, shm_len, [Lie]) of + {error, {'RuntimeError', _}} -> ok; %% isolated: raised in the child + {error, arg_conversion_failed} -> ok %% embedded: conversion refused + end, + py_shm:close(Shm), + stop(C). + +test_closed_wrapper_raises(Config) -> + C = new_ctx(Config), + {ok, Shm} = py_shm:new(4096), + {ok, <<"closed">>} = py_context:call(C, ?MOD, shm_closed_access, [Shm]), + %% A fresh mapping is made on the next use + {ok, 4096} = py_context:call(C, ?MOD, shm_len, [Shm]), + py_shm:close(Shm), + stop(C). + +%% @doc A read-only handle: Python reads it, cannot write, Erlang still can. +test_read_only_handle(Config) -> + C = new_ctx(Config), + {ok, Shm} = py_shm:new(4096, #{writable => false}), + {'$py_shm_ro', _, _, 4096} = Shm, + ok = py_shm:write(Shm, 0, <<"erlang wrote">>), + {ok, <<"erlang wrote">>} = py_context:call(C, ?MOD, shm_read, [Shm, 0, 12]), + {ok, <<"read_only">>} = py_context:call(C, ?MOD, shm_write_readonly, [Shm]), + <<"erlang wrote">> = py_shm:binary(Shm, 0, 12), + %% A writable handle downgraded for one callee + {ok, Rw} = py_shm:new(4096), + Ro = py_shm:read_only(Rw), + {ok, <<"read_only">>} = py_context:call(C, ?MOD, shm_write_readonly, [Ro]), + {ok, 3} = py_context:call(C, ?MOD, shm_write, [Rw, 0, {bytes, <<"abc">>}]), + py_shm:close(Shm), py_shm:close(Rw), + stop(C). + +test_restart_remaps(Config) -> + C = new_ctx(Config), + {ok, Shm} = py_shm:new(4096), + ok = py_shm:write(Shm, 0, <<"persist">>), + {ok, <<"persist">>} = py_context:call(C, ?MOD, shm_read, [Shm, 0, 7]), + ok = py_context:kill(C), + {ok, <<"persist">>} = py_context:call(C, ?MOD, shm_read, [Shm, 0, 7]), + py_shm:close(Shm), + stop(C). + +test_churn_no_leak(Config) -> + C = new_ctx(Config), + Dir = py_shm:private_dir(), + Files0 = length(filelib:wildcard(filename:join(Dir, "shm_*"))), + Regions0 = ets:info(py_shm_regions, size), + lists:foreach(fun(I) -> + {ok, Shm} = py_shm:new(64 * 1024), + ok = py_shm:write(Shm, 0, <>), + {ok, <>} = py_context:call(C, ?MOD, shm_read, [Shm, 0, 4]), + ok = py_shm:close(Shm) + end, lists:seq(1, 200)), + Files1 = length(filelib:wildcard(filename:join(Dir, "shm_*"))), + Files0 = Files1, + Regions0 = ets:info(py_shm_regions, size), + stop(C). + +%%% ============================================================================ +%%% Helpers +%%% ============================================================================ + +new_ctx(Config) -> + Mode = ?config(mode, Config), + TestDir = ?config(test_dir, Config), + {ok, C} = py_context:new(#{mode => Mode, paths => [TestDir]}), + case Mode of + worker -> ok = py_context:exec(C, add_path(Config)); + _ -> ok + end, + C. + +add_path(Config) -> + TestDir = ?config(test_dir, Config), + iolist_to_binary(io_lib:format( + "import sys\nif '~s' not in sys.path: sys.path.insert(0, '~s')", [TestDir, TestDir])). + +stop(C) -> + ok = py_context:stop(C), + ok. + +wait_until(Fun, TimeoutMs) -> + Deadline = erlang:monotonic_time(millisecond) + TimeoutMs, + wait_loop(Fun, Deadline). + +wait_loop(Fun, Deadline) -> + case Fun() of + true -> ok; + false -> + erlang:monotonic_time(millisecond) < Deadline orelse ct:fail(condition_not_met), + timer:sleep(50), + wait_loop(Fun, Deadline) + end. diff --git a/test/py_isolated_stress_SUITE.erl b/test/py_isolated_stress_SUITE.erl index 92fbc2d..ade684f 100644 --- a/test/py_isolated_stress_SUITE.erl +++ b/test/py_isolated_stress_SUITE.erl @@ -15,7 +15,8 @@ test_context_churn_no_leak/1, test_startup_time/1, test_payload_throughput/1, - test_parallel_contexts_cpu_bound/1 + test_parallel_contexts_cpu_bound/1, + test_shared_memory_vs_copy/1 ]). all() -> [ @@ -24,7 +25,8 @@ all() -> [ test_context_churn_no_leak, test_startup_time, test_payload_throughput, - test_parallel_contexts_cpu_bound + test_parallel_contexts_cpu_bound, + test_shared_memory_vs_copy ]. init_per_suite(Config) -> @@ -140,6 +142,55 @@ test_parallel_contexts_cpu_bound(_Config) -> [py_context:stop(C) || C <- Ctxs], ok. +%% @doc Bulk data both ways: a py_shm region against the socket copy, in +%% isolated and worker mode, for 1, 16 and 64 MB. +test_shared_memory_vs_copy(_Config) -> + case py_shm:available() of + false -> {skip, "iommap not available"}; + true -> shared_memory_vs_copy() + end. + +shared_memory_vs_copy() -> + TestDir = filename:join(code:lib_dir(erlang_python), "test"), + {ok, I} = py_context:new(#{mode => isolated, paths => [TestDir]}), + {ok, W} = py_context:new(#{mode => worker}), + ok = py_context:exec(W, iolist_to_binary(io_lib:format( + "import sys\nif '~s' not in sys.path: sys.path.insert(0, '~s')", [TestDir, TestDir]))), + ok = py_context:exec(I, <<"def ident(x): return x">>), + ok = py_context:exec(W, <<"def ident(x): return x">>), + lists:foreach(fun(Mb) -> + Size = Mb * 1024 * 1024, + Bin = crypto:strong_rand_bytes(Size), + {ok, Shm} = py_shm:new(Size), + %% Erlang -> Python: copy through the socket vs write into the region + %% and sum the first 4 KB through a memoryview + CopyI = timed(fun() -> {ok, _} = py_context:call(I, '__main__', ident, [Bin]) end), + ShmI = timed(fun() -> + ok = py_shm:write(Shm, 0, Bin), + {ok, _} = py_context:call(I, py_test_isolated_shm, shm_sum, [Shm, 4096]) + end), + ShmW = timed(fun() -> + ok = py_shm:write(Shm, 0, Bin), + {ok, _} = py_context:call(W, py_test_isolated_shm, shm_sum, [Shm, 4096]) + end), + %% Python -> Erlang: result through the socket vs fill the region and + %% read it with a region binary + OutI = timed(fun() -> {ok, _} = py_context:call(I, py_test_isolated, big_payload, [Size]) end), + FillI = timed(fun() -> + {ok, _} = py_context:call(I, py_test_isolated_shm, shm_fill, [Shm, 1, Size]), + _ = py_shm:binary(Shm, 0, Size) + end), + ct:log("~p MB Erlang->Python: socket ~.1f ms, shm isolated ~.1f ms, shm worker ~.1f ms~n" + " Python->Erlang: socket ~.1f ms, shm isolated ~.1f ms", + [Mb, CopyI / 1000, ShmI / 1000, ShmW / 1000, OutI / 1000, FillI / 1000]), + ct:print("~p MB: to Python socket ~.1f ms vs shm ~.1f ms; from Python socket ~.1f ms vs shm ~.1f ms", + [Mb, CopyI / 1000, ShmI / 1000, OutI / 1000, FillI / 1000]), + ok = py_shm:close(Shm) + end, [1, 16, 64]), + py_context:stop(I), + py_context:stop(W), + ok. + %%% ============================================================================ %%% Helpers %%% ============================================================================ diff --git a/test/py_test_isolated_shm.py b/test/py_test_isolated_shm.py new file mode 100644 index 0000000..997c90d --- /dev/null +++ b/test/py_test_isolated_shm.py @@ -0,0 +1,159 @@ +"""Helpers for py_isolated_shm_SUITE and py_isolated_buffer_SUITE. They run +unchanged in worker and isolated contexts.""" + +import erlang + +maps = 0 # how many times a wrapper was constructed in this interpreter + + +def _count_map(): + global maps + maps += 1 + + +def kind(obj): + return type(obj).__name__ + + +def shm_len(shm): + return len(shm) + + +def shm_sum(shm, n): + """Sum the first n bytes through a memoryview (no copy).""" + return sum(memoryview(shm.buffer)[:n]) + + +def shm_read(shm, offset, n): + return bytes(shm[offset:offset + n]) + + +def shm_write(shm, offset, data): + shm[offset:offset + len(data)] = data + return len(data) + + +def shm_fill(shm, byte, n): + shm[0:n] = bytes([byte]) * n + return n + + +def shm_identity(shm): + return shm + + +def shm_in_structure(payload): + """payload = {'regions': [shm, ...], 'label': ...}; returns lengths.""" + return [len(s) for s in payload['regions']] + + +def shm_numpy_sum(shm, n): + import numpy + a = numpy.frombuffer(shm.buffer, dtype=numpy.uint8, count=n) + return int(a.sum()) + + +def shm_numpy_write(shm, n): + import numpy + a = numpy.frombuffer(shm.buffer, dtype=numpy.uint8, count=n) + a[:] = numpy.arange(n, dtype=numpy.uint8) + return int(a.sum()) + + +def map_count(): + from _erlang_impl import _shm + return len(_shm._cache) + + +def shm_write_readonly(shm): + try: + shm[0:3] = b'abc' + return 'wrote' + except TypeError: + return 'read_only' + + +def shm_closed_access(shm): + shm.close() + try: + shm[0] + return 'readable' + except ValueError: + return 'closed' + + +# ---- shared buffers --------------------------------------------------------- + +def buf_kind(buf): + return type(buf).__name__ + + +def buf_read_all(buf): + return buf.read() + + +def buf_read_n(buf, n): + return buf.read(n) + + +def buf_read_chunks(buf, n): + out = [] + while True: + chunk = buf.read(n) + if not chunk: + return out + out.append(chunk) + + +def buf_readline(buf): + return buf.readline() + + +def buf_readlines(buf): + return buf.readlines() + + +def buf_iter(buf): + return [line for line in buf] + + +def buf_read_nonblock(buf): + return buf.read_nonblock() + + +def buf_at_eof(buf): + return buf.at_eof() + + +def buf_consume_len(buf, chunk): + """Total bytes read in chunks of `chunk`.""" + total = 0 + while True: + data = buf.read(chunk) + if not data: + return total + total += len(data) + + +def buf_consume_checksum(buf, chunk): + total = 0 + acc = 0 + while True: + data = buf.read(chunk) + if not data: + return (total, acc) + total += len(data) + acc = (acc + sum(data)) % 1000003 + + +def buf_from_environ(environ): + body = environ['wsgi.input'].read() + return (environ['method'], len(body), body[:8]) + + +def buf_read_with_callback(buf, name): + """Read while a callback re-enters the context: must not deadlock.""" + head = buf.read(4) + nested = erlang.call(name, 21) + rest = buf.read() + return (head, nested, len(rest)) diff --git a/test/py_worker_loop_SUITE.erl b/test/py_worker_loop_SUITE.erl index db41475..3cf6e07 100644 --- a/test/py_worker_loop_SUITE.erl +++ b/test/py_worker_loop_SUITE.erl @@ -320,8 +320,10 @@ test_three_workers_one_listen_fd(Config) -> 300 = length([R || R <- Replies, binary:part(R, byte_size(R) - 4, 4) =:= <<"ok:x">>]), Tags = lists:usort([binary:part(R, 0, 3) || R <- Replies]), ct:log("workers that served: ~p", [Tags]), - %% All three workers accept on the same socket - 3 = length(Tags), + %% Which worker wins accept() is up to the kernel; a fast worker can + %% starve another over 300 connections. Two distinct workers prove the + %% socket is shared. + true = length(Tags) >= 2, [ok = py_context:stop_loop(C) || C <- Ctxs], [stop_ctx(C) || C <- Ctxs], gen_tcp:close(LSock), From c8f77f02587298e4416c17d140f6801d2929dc05 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sat, 29 Aug 2026 14:41:17 +0200 Subject: [PATCH 05/15] Add architecture, code map and glossary A map of the code: processes and threads, the life of a call in each context mode, the paths Python uses to call Erlang, which code is live and which is legacy, and one meaning per overloaded word (worker, context, pool). The stale file list in the NIF headers now points at these pages. Also completes the 4.2.0 changelog. --- CHANGELOG.md | 8 +- README.md | 1 + c_src/README.md | 71 +++++++++++++ c_src/py_nif.c | 9 +- c_src/py_nif.h | 61 +++-------- docs/architecture.md | 198 ++++++++++++++++++++++++++++++++++++ docs/code-map.md | 98 ++++++++++++++++++ docs/glossary.md | 109 ++++++++++++++++++++ priv/_erlang_impl/README.md | 29 ++++++ rebar.config | 6 ++ 10 files changed, 536 insertions(+), 54 deletions(-) create mode 100644 c_src/README.md create mode 100644 docs/architecture.md create mode 100644 docs/code-map.md create mode 100644 docs/glossary.md create mode 100644 priv/_erlang_impl/README.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f042595..a143464 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,8 @@ such a region with ring backpressure, usable as `wsgi.input` in isolated contexts. Handles are plain terms and travel inside any argument or result; `py_shm:read_only/1` and `new(Size, #{writable => false})` hand Python a - read-only mapping. + read-only mapping. `py_buffer:write/3` takes a timeout for the case where + the ring is full and nobody reads (default 30 s). - `py:python_executable/0`, `py:kill/1`, `py_nif:os_kill/2`. - `py_isolated` is a `gen_statem` (states `idle`, `{busy, Id}`, `looping`, `stopping_loop`, `{restarting, Reason}`): `sys:get_state/1` and @@ -48,6 +49,11 @@ storms, loop churn, 60 s mixed workload with resource counters checked. - Guide: `docs/isolated.md`, with what each of the three modes guarantees. +### Fixed + +- `pthread_timedjoin_np` was called without `_GNU_SOURCE`, an implicit + declaration on Linux that newer compilers reject. + ## 4.1.0 (2026-08-15) ### Added diff --git a/README.md b/README.md index aa550d5..6045ac8 100644 --- a/README.md +++ b/README.md @@ -654,6 +654,7 @@ py:execution_mode(). %% => worker | owngil ## Documentation +- [Architecture](docs/architecture.md), [Code map](docs/code-map.md), [Glossary](docs/glossary.md) - how the pieces fit - [Getting Started](docs/getting-started.md) - [Process-Bound Environments](docs/process-bound-envs.md) - Isolated Python state per Erlang process - [AI Integration Guide](docs/ai-integration.md) diff --git a/c_src/README.md b/c_src/README.md new file mode 100644 index 0000000..a8f240e --- /dev/null +++ b/c_src/README.md @@ -0,0 +1,71 @@ +# c_src + +The NIF that embeds CPython in the VM. One translation unit: `py_nif.c` +`#include`s every other `.c` file (see the "Include module implementations" +section near line 265), so build with `rebar3 compile` (CMake through +`do_cmake.sh` / `do_build.sh`), never a single file. Headers declare what +the included files share. + +Read `docs/architecture.md` first for how a call travels; this file says +where things are. + +## Files + +| File | What it owns | Notes | +|---|---|---| +| `py_nif.h` | All shared types: `py_context_t` and its request queue, request types, callback and suspension state, runtime state machine, atoms, globals, declarations | 2.4k lines. The struct comments carry the locking rules; read `py_context_t` before touching threads | +| `py_nif.c` | Runtime init and finalize, resource types, context create/destroy, the request queue, `worker_context_thread_main` and `owngil_context_thread_main`, `owngil_execute_*` (used by both thread kinds), the `nif_context_*` NIFs, process-local envs, `py_ref`, the NIF function table at the end | Sections are banner-separated; `grep -n '^ \* ===\|^/\* ==='` lists them | +| `py_convert.c` | `py_to_term`, `term_to_py`, depth limits, tagged tuples (`{bytes, B}`, `{'$py_shm', ...}`), error tuples `{error, {Type, Msg}}` | The type mapping tables in the comments are the reference for `_etf.py` | +| `py_exec.c` | Executing a call/eval/exec with suspension support; the legacy single executor thread | | +| `py_callback.c` | The `erlang` Python module: `call`, `send`, `whereis`, `schedule*`, `Atom`/`Pid`/`Ref` types, callback delivery paths (suspension, blocking pipe, async pipe), channel and shared-dict methods, callback name registry | `erlang_call_impl` documents the path precedence | +| `py_thread_worker.c` | Python threads calling Erlang through `py_thread_handler` | | +| `py_subinterp_thread.c/.h` | Thread pool of sub-interpreters (owngil contexts, loop pools) | | +| `py_event_loop.c/.h` | `ErlangEventLoop` support: `enif_select` readers and writers, timers, task injection into loops, reactor dispatch, fd registry, Python module `py_event_loop`; plus test-only fd/TCP/UDP NIFs (section "Test Helper Functions") | Largest file | +| `py_channel.c/.h`, `py_buffer.c/.h`, `py_reactor_buffer.c/.h`, `py_shared_dict.c` | Resources with a Python-facing object each | | +| `py_logging.c` | Logging and tracing NIFs | | +| `py_mem_limit.c` | obmalloc arena accounting for owngil memory caps | | +| `py_worker_pool.c/.h` | Legacy pool, no caller in `src/` | Candidate for removal | +| `py_util.c/.h` | Macros, small helpers | | + +## Where the live paths are + +- `py:call/3` in worker or owngil mode: `nif_context_call_async` (`py_nif.c`) + enqueues; `worker_context_thread_main` or `owngil_context_thread_main` + dequeues and calls `owngil_execute_request`; the reply goes out as + `{py_result, Ref, Result}`. +- `erlang.call` from Python: `erlang_call_impl` (`py_callback.c`). +- Interrupt: `nif_context_interrupt` (`py_nif.c`), `interrupt_mutex` rules on + `py_context_t`. +- Type conversion: `py_to_term` / `term_to_py` (`py_convert.c`). +- Isolated mode has no C code of its own: `os_kill` is the only NIF it uses. + +## Rules that are easy to break + +- Only the context's thread touches the context's Python objects. NIFs + called from Erlang processes enqueue requests and return; they do not run + Python for a context that has a thread. +- `Py_BEGIN_ALLOW_THREADS` around every blocking wait; never block on an + Erlang-side resource with the GIL held. +- Never call a Future method while holding `async_futures_mutex` + (`py_callback.c`, comment above the struct explains the pattern). +- `interrupt_mutex` is taken only by threads that do not hold the GIL. +- `queue_mutex` protects the request queue of a context and is taken before + a request's own mutex (`ctx_queue_cancel_all`), never the other way. +- A NIF that can block or run Python is registered with a dirty scheduler + flag in the table at the end of `py_nif.c`. + +## Adding a NIF + +1. Implement `static ERL_NIF_TERM nif_x(ErlNifEnv*, int, const ERL_NIF_TERM[])` + next to related code. +2. Add `{"x", Arity, nif_x, Flags}` to `nif_funcs[]` at the end of `py_nif.c`. +3. Add the stub and its `-spec` and doc to `src/py_nif.erl`. +4. Cover it in a suite; `rebar3 dialyzer` and `rebar3 xref` must stay clean. + +## Legacy code, for orientation + +Not on the path of contexts created today: the `worker_*` NIFs and +"Worker management" section, `async_worker_*` NIFs (return `deprecated`), +the "Legacy mode" inline branches in `nif_context_call/eval/exec`, +`py_worker_pool.c`, and the `cancel_reader/writer` aliases. When in doubt, +follow `nif_context_call_async` and ignore the rest. diff --git a/c_src/py_nif.c b/c_src/py_nif.c index cb8f459..83f83df 100644 --- a/c_src/py_nif.c +++ b/c_src/py_nif.c @@ -29,11 +29,10 @@ * - Resource types for Python objects to ensure proper cleanup * - Dirty NIF flags for GIL-holding operations * - * This file is the main entry point. It includes the following modules: - * - py_nif.h: Shared header with types and declarations - * - py_convert.c: Type conversion (Python <-> Erlang) - * - py_exec.c: Python execution and GIL management - * - py_callback.c: Callback system and asyncio support + * This file is the main entry point and the single translation unit: it + * includes the other .c files (see "Include module implementations"). The + * file map is c_src/README.md; the request lifecycle per context mode is + * docs/architecture.md. */ /* pthread_timedjoin_np (used to bound the owngil worker join on Linux) diff --git a/c_src/py_nif.h b/c_src/py_nif.h index 4f2e37c..45020cb 100644 --- a/c_src/py_nif.h +++ b/c_src/py_nif.h @@ -21,54 +21,19 @@ * * @mainpage Python-Erlang NIF Integration * - * @section intro_sec Introduction - * - * This NIF (Native Implemented Function) library provides seamless integration - * between Erlang/OTP and Python. It embeds a Python interpreter within the - * Erlang VM and provides bidirectional communication capabilities. - * - * @section arch_sec Architecture - * - * The implementation follows a modular design with four main components: - * - * - **py_nif.h** - Shared types, macros, and declarations - * - **py_convert.c** - Bidirectional type conversion (Python ↔ Erlang) - * - **py_exec.c** - Python execution engine and GIL management - * - **py_callback.c** - Erlang callback support and asyncio integration - * - * @section modes_sec Execution Modes - * - * The library supports three execution modes based on Python version: - * - * | Mode | Python Version | Description | - * |------|----------------|-------------| - * | FREE_THREADED | 3.13+ (no-GIL) | Direct execution without GIL | - * | SUBINTERP | 3.12+ | Per-interpreter GIL isolation | - * | MULTI_EXECUTOR | Any | Multiple executor threads with GIL | - * - * @section gil_sec GIL Management - * - * The GIL (Global Interpreter Lock) is managed following PyO3/Granian patterns: - * - * - `Py_BEGIN_ALLOW_THREADS` / `Py_END_ALLOW_THREADS` around blocking ops - * - Executor threads hold the GIL and process queued requests - * - Dirty I/O schedulers are used for Python-calling NIFs - * - * @section callback_sec Callback Mechanism - * - * Python code can call back to Erlang using a suspension/resume pattern: - * - * 1. Python calls `erlang.call('func', args)` - * 2. NIF raises `SuspensionRequired` exception - * 3. Dirty scheduler is released, callback sent to Erlang - * 4. Erlang processes callback, calls `resume_callback/2` - * 5. Python execution resumes with cached result - * - * @section mem_sec Memory Management - * - * - Erlang resources wrap Python objects (prevent GC) - * - Thread-local storage for callback context - * - Proper cleanup in resource destructors + * This NIF embeds CPython in the Erlang VM. The map of the code, the life + * of a call in each context mode, the callback paths and the locking rules + * are documented in docs/architecture.md, c_src/README.md and + * docs/glossary.md; keep those current instead of this comment. + * + * In one paragraph: py_nif.c is the single translation unit and includes + * the other .c files; a context (py_context_t) owns a request queue and a + * pthread that runs Python for it (worker mode: main interpreter, shared + * GIL; owngil mode: a sub-interpreter with its own GIL); Erlang enqueues + * through nif_context_call_async and receives {py_result, Ref, Result}; + * Python calls Erlang through erlang_call_impl (py_callback.c), by + * suspension in worker mode and a blocking pipe in owngil mode. Isolated + * mode runs Python in a child process and uses no C code beyond os_kill. */ #ifndef PY_NIF_H diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..53e6426 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,198 @@ +# Architecture + +This page is the map of erlang_python. It +says which processes and threads exist, how a call travels from `py:call/3` +to Python and back in each context mode, how Python calls Erlang, and which +code paths are live. Read it before opening `c_src/` or `src/py_context.erl`. +The [code map](code-map.md) lists every file; the [glossary](glossary.md) +defines the overloaded words (worker, context, pool). + +## One picture + +``` +Erlang VM child OS process ++--------------------------------------------------+ (isolated mode only) +| erlang_python_sup | +| py_callback registry of Erlang funs Python | +| may call | +| py_shm shared memory regions | +| py_thread_handler spawns a handler process | +| per Python thread that calls | +| Erlang | +| py_logger / py_tracer Python logging, tracing | +| py_context_sup ---- py_context (one per | +| context; mode worker | owngil | isolated) | +| py_context_init starts the default pool | +| py_event_worker_sup / _registry loop drivers | +| py_event_loop, py_event_loop_pool main- | +| interpreter asyncio loops | ++--------------------------------------------------+ + | NIF calls (dirty schedulers) | Unix socket, ETF frames + v v ++------------------------+ +-------------------------+ +| libpython in the VM | | python3 py_isolated_ | +| main interpreter | | child.py | +| worker contexts: one | | reader thread + main | +| pthread each, shared | | thread, own asyncio | +| GIL | | loop | +| owngil contexts: one | +-------------------------+ +| pthread + own sub- | +| interpreter each | ++------------------------+ +``` + +A **context** is the unit of work: an Erlang process (`py_context`) that owns +one Python execution environment and serves calls in order. Pools +(`py_context_router`) route `py:call/3` to a context by scheduler affinity. + +## The three context modes + +| | `worker` | `owngil` | `isolated` | +|---|---|---|---| +| Python runs in | the VM, main interpreter | the VM, a sub-interpreter with its own GIL | a child process | +| Thread | one pthread per context (`worker_context_thread_main`) | one pthread per context (`owngil_context_thread_main`) | the child's main thread | +| Erlang process loop | the receive loop in `py_context` | the receive loop in `py_context` | `py_isolated` (`gen_statem`) | +| Transport | NIF request queue on `py_context_t` | same | Unix socket, frames of the callback pipe format | +| Python -> Erlang | suspension protocol | blocking callback pipe | socket frames | +| Interrupt | `PyThreadState_SetAsyncExc`, next bytecode | same | signal in the child, `SIGKILL` backstop | +| Guide | [context-affinity](context-affinity.md), [pools](pools.md) | [owngil_internals](owngil_internals.md) | [isolated](isolated.md) | + +## Life of a call, per mode + +### worker and owngil (embedded) + +1. `py:call(M, F, A)` picks a context through `py_context_router` and sends + `{call, From, MRef, M, F, Args, Kwargs}` to the `py_context` process + (`src/py.erl`, `src/py_context.erl:call/6`). The caller waits in + `await_reply/3`; a timeout there calls `interrupt/1`. +2. The `py_context` receive loop takes it and calls `handle_call_with_suspension/5`, + which calls the `context_call_async` NIF + (`c_src/py_nif.c`, `nif_context_call_async`). The NIF converts the + arguments (`term_to_py`, `c_src/py_convert.c`) into a request, enqueues it + on the context's queue (`ctx_queue_enqueue`) and returns `{enqueued, Ref}` + at once. The Erlang process is now free to serve callbacks. +3. The context's pthread (`worker_context_thread_main` or + `owngil_context_thread_main`, `c_src/py_nif.c`) dequeues the request and + runs it through `owngil_execute_request` (despite its name it serves both + modes), which calls into Python with the GIL held. +4. The thread converts the result (`py_to_term`) and sends + `{py_result, Ref, Result}` to the `py_context` process, which replies + `From ! {MRef, Result}`. + +The older paths in `nif_context_call` (a blocking variant with an inline +"legacy" executor) are kept for the fallback +`{error, async_requires_worker_thread}` and are not taken by contexts +created today; see [code map](code-map.md) for the list of legacy code. + +### isolated + +1. Same first step: the message reaches the `py_context` pid, which in this + mode runs `py_isolated` (a `gen_statem` entered with `enter_loop`). +2. In `idle` the request is encoded with `term_to_binary` into a frame + `<>` and written to + the Unix socket; the state becomes `{busy, Id}` and other callers' + requests are postponed (served in order when the child is free). +3. In the child, the reader thread parses frames and queues the request; + the main thread runs it (`_erlang_impl/_isolated.py`, `Runtime._dispatch`) + and writes the reply frame. +4. Back in `py_isolated`, the reply for the busy id moves the state to + `idle` and answers the caller. A crash of the child is seen as the port's + `exit_status`; the state goes through `{restarting, Reason}` and a new + child is started within the restart budget. + +Protocol details: the module header of `src/py_isolated.erl` and the +docstring of `priv/_erlang_impl/_isolated.py`. + +## Python calling Erlang (`erlang.call`) + +`erlang_call_impl` in `c_src/py_callback.c` chooses one of these paths, in +this order (the comment above it is the authoritative version): + +1. **Suspension** (worker contexts). The Python call raises + `SuspensionRequired`; the context thread returns + `{suspended, CallbackId, State, {Name, Args}}` to `py_context`, which runs + the registered fun (`execute/2` in `py_callback`), possibly serving nested calls + meanwhile (`wait_for_callback/2`), and resumes with + the `resume_callback` NIF. +2. **Blocking callback pipe** (owngil contexts). The context thread writes + a request on a pipe and blocks; the `py_context` process has a dedicated + handler (`callback_handler_loop/1`) that runs the fun and writes the + response frame back with `context_write_callback_response`. +3. **Legacy worker handler** (`worker_*` NIFs): only used by + `examples/gen_test.erl`. +4. **Thread worker** (`c_src/py_thread_worker.c`): any Python thread that is + not a context thread (`threading.Thread`, executors) asks the + `py_thread_handler` coordinator for a handler process and talks to it + over a pipe. There is also an async variant (`erlang.async_call`) using a + per-interpreter async pipe. + +In isolated mode there is one path: a status-3 frame on the socket, answered +by a process the `py_isolated` state machine spawns; nested calls into the +same context are dispatched immediately because they come from that process. + +The frame format shared by the pipe and the socket, and the ETF conventions, +will get their own page (protocols); until then `c_src/py_convert.c` (type +mapping) and `priv/_erlang_impl/_etf.py` are the reference. + +## asyncio + +Three different machineries, on purpose: + +- Embedded contexts share an `ErlangEventLoop` (`priv/_erlang_impl/_loop.py`) + whose `add_reader`/`add_writer` map onto `enif_select` and `call_later` onto + `erlang:send_after`; readiness is delivered to a `py_event_worker` process + per loop (`c_src/py_event_loop.c`, `src/py_event_worker.erl`). Worker loops + (`py_context:start_loop/1`) run such a loop on the context thread. +- `py_event_loop`/`py_event_loop_pool` expose the main-interpreter loops for + `py:async_call/3` and friends. +- An isolated child runs a plain asyncio loop; the only integration is + delivering results over the socket. Nothing of the reactor is ported. + +See [event_loop_architecture](event_loop_architecture.md) for the embedded +loop and [asyncio](asyncio.md) for the API. + +## Data paths that avoid copies + +- `py_buffer` (native): a NIF resource Erlang writes into and Python reads + through the buffer protocol; embedded modes only. +- `py_shm` and `py_buffer:new(#{shared => true})`: a file mapped + `MAP_SHARED` by the VM (through iommap) and by any interpreter, embedded or + child, with flow control through callbacks (`_py_buffer_wait`, + `_py_buffer_consumed`). See [isolated](isolated.md#bulk-data-with-shared-memory). +- `py_channel`, `py_byte_channel`: message queues between Erlang and Python + coroutines, embedded modes only. + +## Where state lives + +| State | Owner | Notes | +|---|---|---| +| Registered callbacks | `py_callback` ETS `py_callbacks` | also mirrored in a C name registry so `erlang.` resolves; `py_state` exposes its store the same way (`erlang.state_get`) | +| Import and path registry | `py_import` ETS | applied to every new interpreter and to isolated children at start | +| Preload code | `py_preload` persistent_term | run once per interpreter | +| Context pid -> NIF ref | `py_context` ETS `py_context_refs` | lets `interrupt/1` reach a context blocked in a NIF; isolated contexts store the atom `isolated` | +| Per-Erlang-process Python env | `py` process dictionary + NIF env resource | `py:call(Ctx, ...)`; not applicable to isolated contexts | +| Shared regions | `py_shm` ETS `py_shm_regions` | closed on owner death | +| Python-side state | the interpreter | lost when an isolated child restarts | + +## Invariants worth knowing before editing + +- A context serves one request at a time. Embedded: the thread dequeues one + request; nested callbacks are served by the `py_context` process while the + thread waits. Isolated: enforced by the `{busy, Id}` state and `postpone`. +- Only the context's own thread touches its Python objects; NIF callers only + enqueue. `py_context_t` fields are documented in `c_src/py_nif.h`. +- Interrupts target the request executing now. Isolated mode also cancels a + queued request by id; embedded modes cannot. +- Everything that crosses the socket is a term; NIF resources (channels, + native buffers, object references) do not cross, and the API says so with + `{error, not_supported_in_isolated}`. +- `binary_to_term` on child data is not `safe`: the child can create atoms. + +## What is live and what is not + +Kept for now, not used by current contexts: the `worker_*` NIF API and its +single executor thread (`c_src/py_exec.c`), the `async_worker_*` NIFs (they +return `deprecated`), `c_src/py_worker_pool.c` (no caller), the inline +"legacy" executor branches in `nif_context_*`, and the test-only fd NIFs in +`c_src/py_event_loop.c`. They are listed in the [code map](code-map.md) so +nobody debugs them by mistake; removing them is planned. diff --git a/docs/code-map.md b/docs/code-map.md new file mode 100644 index 0000000..3139871 --- /dev/null +++ b/docs/code-map.md @@ -0,0 +1,98 @@ +# Code map + +Every source file, what it owns, and where to look for its behaviour. Status +is `live` (on the path of a context created today), `legacy` (kept for +compatibility, no current caller in `src/`), or `test` (only exercised by +suites). Guides are in `docs/`, suites in `test/`. Start with +[architecture](architecture.md). + +## Erlang (`src/`) + +| Module | Owns | Status | Guide | Suites | +|---|---|---|---|---| +| `py` | Public API facade: call/eval/exec, streams, async helpers, venvs, memory, function registration | live | README, getting-started | `py_SUITE`, `py_api_SUITE`, `py_stream_SUITE`, `py_venv_SUITE` | +| `py_context` | The context process for embedded modes and the API every mode answers (`call/eval/exec`, `interrupt`, `kill`, loops, `pass_fd`); dispatch to `py_isolated` for isolated mode | live | context-affinity, workers, interrupts | `py_context_SUITE`, `py_context_process_SUITE`, `py_interrupt_SUITE`, `py_worker_loop_SUITE` | +| `py_isolated` | `gen_statem` driving a child process over the socket; restart policy | live | isolated | `py_isolated_*_SUITE` | +| `py_context_router` | Pools and scheduler-affinity routing | live | pools, context-affinity | `py_context_router_SUITE`, `py_pool_SUITE` | +| `py_context_sup`, `py_context_init` | Supervisor of contexts; starts the default pool at boot | live | pools | (through the above) | +| `py_nif` | Erlang stubs and docs for every NIF | live | api-reference | all | +| `py_callback` | Registry of Erlang funs callable as `erlang.call('name', ...)` | live | README (callbacks) | `py_callback_encoding_SUITE`, `py_thread_callback_SUITE` | +| `py_thread_handler` | Coordinator that gives each Python thread calling Erlang a handler process and a pipe | live | threading | `py_thread_callback_SUITE`, `py_reentrant_SUITE` | +| `py_event_loop` | Main-interpreter asyncio loop: `run`, `create_task`, `await`, and the loop callbacks Python needs | live | asyncio | `py_event_loop_SUITE`, `py_async_task_SUITE` | +| `py_event_loop_pool` | Several main-interpreter loops with process affinity | live | asyncio | `py_event_loop_pool_SUITE` | +| `py_event_worker`, `_sup`, `_registry` | One process per running loop receiving `enif_select` readiness and timers | live | event_loop_architecture | `py_event_loop_SUITE`, `py_fd_ops_SUITE` | +| `py_reactor_context` | FD-owning context for the protocol-based reactor | live | reactor | `py_reactor_SUITE` | +| `py_channel`, `py_byte_channel` | Term and byte queues between Erlang and Python coroutines (NIF resources) | live | channel | `py_channel_SUITE`, `py_byte_channel_SUITE` | +| `py_buffer` | Native streaming input buffer; shared variant delegates to `py_shm` | live | buffer, isolated | `py_buffer_SUITE`, `py_isolated_buffer_SUITE` | +| `py_shm` | Shared memory regions over iommap and the ring behind shared buffers | live | isolated | `py_isolated_shm_SUITE` | +| `py_import` | Registry of imports and `sys.path` entries applied to every interpreter | live | imports | `py_import_SUITE` | +| `py_preload` | Code run once per interpreter at start | live | preload | `py_preload_SUITE` | +| `py_state` | Shared key/value store visible from Python as `erlang.state_get/set/delete/keys` | live | README (shared state) | `py_state_SUITE` | +| `py_semaphore` | ETS counting semaphore for rate limiting | live | scalability | (through `py_SUITE`) | +| `py_logger`, `py_tracer` | Python `logging` into Erlang logger; tracing hooks | live | logging | `py_logging_SUITE` | +| `erlang_python_app`, `erlang_python_sup` | Application start and the supervision tree | live | architecture | all | +| `py_util` | Small helpers | live | | | + +## C (`c_src/`) + +`py_nif.c` is the only translation unit: it `#include`s the other `.c` +files. Editing `py_convert.c` alone does not compile it alone; build with +`rebar3 compile`. See `c_src/README.md`. + +| File | Owns | Status | +|---|---|---| +| `py_nif.h` | Every shared type: `py_context_t`, request types, runtime state machine, atoms, globals | live | +| `py_nif.c` | Runtime init, context creation and destruction, the request queue and the two context thread mains, the process-per-context NIFs (`nif_context_*`), process-local envs, `py_ref`, the NIF table | live, with legacy branches | +| `py_convert.c` | `py_to_term` / `term_to_py`, the type mapping, tagged tuples (`{bytes, B}`, shared handles) | live | +| `py_exec.c` | Execution with suspension support; the legacy single executor thread | live (suspension), legacy (executor) | +| `py_callback.c` | The `erlang` Python module: `call`, `send`, `whereis`, `Atom`/`Pid`/`Ref` types, schedule markers, callback pipes, channel and shared dict methods | live | +| `py_thread_worker.c` | Python threads calling Erlang through `py_thread_handler` | live | +| `py_subinterp_thread.c` | Sub-interpreter thread pool used by owngil contexts and loop pools | live | +| `py_event_loop.c` | `ErlangEventLoop` support: `enif_select` readers/writers, timers, task injection, reactor dispatch, fd registry; also ~570 lines of test-only fd/TCP/UDP NIFs | live; test section | +| `py_channel.c`, `py_buffer.c`, `py_reactor_buffer.c`, `py_shared_dict.c` | The corresponding resources and their Python-facing methods | live | +| `py_logging.c` | Logging and tracing NIFs | live | +| `py_mem_limit.c` | Per-interpreter memory caps (owngil) | live | +| `py_worker_pool.c/.h` | An older worker pool | legacy, no caller | +| `py_util.c/.h` | Macros and helpers | live | + +Inside `py_nif.c`, these are legacy: the `worker_*` NIFs and the "Worker +management" section, the `async_worker_*` NIFs (return `deprecated`), the +inline executor branches marked "Legacy mode" in `nif_context_call`, +`nif_context_eval`, `nif_context_exec`, and the `cancel_reader/writer` +aliases. + +## Python (`priv/`) + +`priv/` is on `sys.path` of every interpreter. `_erlang_impl` is the Python +half of the `erlang` module; the embedded C module delegates to it for the +loop, channels and servers. + +| File | Owns | Used by | +|---|---|---| +| `_erlang_impl/__init__.py` | Public surface of `erlang` in embedded modes: `run`, `sleep`, `spawn_task`, loop policy, `atom`, channels, `server` | embedded | +| `_erlang_impl/_loop.py`, `_policy.py`, `_transport.py` | `ErlangEventLoop` (uvloop-compatible) over `enif_select` | embedded | +| `_erlang_impl/_reactor.py` | Protocol-based reactor over fds Erlang owns | embedded | +| `_erlang_impl/_channel.py`, `_byte_channel.py` | Python side of channels | embedded | +| `_erlang_impl/_server.py` | `serve`, `adopt`, `stop_serving` on fds handed over by Erlang; plain asyncio, works in every mode | all | +| `_erlang_impl/_sandbox.py`, `_subprocess.py` | Audit hook blocking fork/exec inside the VM | embedded | +| `_erlang_impl/_mode.py` | Detects how Python is running (embedded, free-threaded, child) | all | +| `_erlang_impl/_etf.py` | Pure-Python ETF codec with the `py_convert.c` mapping | isolated child | +| `_erlang_impl/_isolated.py` | Child runtime: socket frames, reader thread, re-entrant main loop, interrupt signal, asyncio loop, the `erlang` shim | isolated child | +| `_erlang_impl/_shm.py` | `SharedMemory` and `SharedBuffer` wrappers over mmap | all | +| `py_isolated_child.py` | Child launcher: rlimits, parent-death signal, cgroup join, connect | isolated child | +| `test_erlang_loop.py`, `tests/` | Python-side tests of the loop | test | + +## Tests (`test/`) + +Suites named `py__SUITE`. Cross-mode suites run the same cases in +`worker` and `isolated` groups (`py_isolated_SUITE`, `py_isolated_vm_SUITE`, +`py_isolated_shm_SUITE`, `py_isolated_buffer_SUITE`). Python helpers used by +suites are `test/py_test_*.py`. `test/coverage_audit.md` maps public APIs to +cases. `test/test.config` holds node-wide settings (memory limits flag). + +## Build and docs + +`rebar.config` runs `do_cmake.sh` / `do_build.sh` (CMake in `c_src/`) as +compile hooks; the NIF lands in `priv/py_nif.so`. `make lint-docs` checks +that Erlang snippets in the guides call real exports and that Python +snippets parse. `rebar3 ex_doc` builds the guides listed in `rebar.config`. diff --git a/docs/glossary.md b/docs/glossary.md new file mode 100644 index 0000000..cf5864c --- /dev/null +++ b/docs/glossary.md @@ -0,0 +1,109 @@ +# Glossary + +The same words mean different things in different files of this project. +This page fixes one meaning per term and says where the other uses come +from, so a reader can translate as they go. + +## Context + +**Context**: one Python execution environment served by one Erlang process +(`py_context`), in order, one request at a time. The unit of `py:context/0`, +pools and modes. In C it is `py_context_t` (`c_src/py_nif.h`); in +`py_context.erl` it is the process; for isolated mode the process runs +`py_isolated` and the environment is a child OS process. + +Other uses: `py_reactor_context` is a context that also owns file +descriptors for the reactor; "coordinator context" in C comments means the +`py_thread_handler` side of the thread-worker channel. + +## Mode + +**Mode**: how a context runs Python. `worker` (main interpreter, one pthread +per context, shared GIL), `owngil` (a sub-interpreter with its own GIL per +context, one pthread), `isolated` (a child process). `py_context:new(#{mode => ...})`. + +Related flags on `py_context_t`: `uses_worker_thread` (has its own pthread; +true for worker and owngil contexts created today), `is_subinterp` (has its +own sub-interpreter), `uses_own_gil` (that sub-interpreter has its own GIL). +`subinterp` in file and NIF names (`py_subinterp_thread.c`, +`subinterp_supported/0`) refers to the machinery owngil mode is built on; +there is no separate "subinterp mode" any more. + +The runtime-wide `PY_MODE_FREE_THREADED` / `PY_MODE_GIL` in `py_nif.h` is +about the Python build (free-threaded or not), not about contexts. + +## Worker + +The most overloaded word. Meanings, by file: + +| Where | Meaning | Prefer to say | +|---|---|---| +| `py_context:new(#{mode => worker})` | the context mode above | worker mode | +| `worker_context_thread_main`, `uses_worker_thread` (`py_nif.c`) | the pthread that serves a context's queue | context thread | +| `worker_new/call/eval/exec` NIFs, `py_worker_t` | the legacy per-worker API, before contexts | legacy worker API | +| `py_worker_pool.c`, `py_pool_worker_t` | an older pool, no caller | legacy pool | +| `thread_worker`, `thread_worker_call` (`py_thread_worker.c`), `py_thread_handler` | the channel a Python thread uses to call Erlang | thread callback bridge | +| `py_event_worker` | the Erlang process that drives one asyncio loop (readiness, timers) | loop driver | +| `docs/workers.md`, "worker loop" | a long-running asyncio loop on a context thread, gunicorn-style | worker loop | + +## Pool + +`py_context_router` pools (`py:call(Pool, M, F, A)`): named sets of contexts +routed by scheduler. `py_event_loop_pool`: main-interpreter asyncio loops +with process affinity. `g_thread_pool` in `py_subinterp_thread.c`: the +threads behind owngil contexts. `g_pool` in `py_worker_pool.c`: legacy. + +## Callback + +An Erlang function registered with `py:register_function/2` or +`register/2` in `py_callback` and called from Python as `erlang.call('name', ...)` +or `erlang.name(...)`. Four delivery paths exist (suspension, blocking pipe, +thread worker, socket); see [architecture](architecture.md#python-calling-erlang-erlangcall). + +**Suspension**: worker-mode delivery where the Python call raises +`SuspensionRequired`, the context thread hands control to the Erlang process, +and execution resumes with the result (`resume_callback/2`). + +**Callback pipe**: owngil-mode delivery where the context thread blocks on a +pipe until the `py_context` handler process writes the response frame. + +## Frame + +The wire unit of the callback pipe and of the isolated socket: +`<>`, body `<>`. +Status 0 request, 1 error reply, 2 ok reply, 3 request from Python, +4 event, 5 control. + +## Loop + +`ErlangEventLoop`: the asyncio loop implementation backed by `enif_select` +(`_erlang_impl/_loop.py`), used by embedded modes. "Loop ref": the NIF +handle of such a loop (`py_context:loop_ref/1`, the `submit_task` NIF). +An isolated child uses the standard asyncio loop and has no loop ref. + +## Environment, process-local env + +The Python namespace a call runs in. Every context has globals; in embedded +modes each Erlang process can additionally get its own env inside a context +(`py:call(Ctx, ...)`, [process-bound-envs](process-bound-envs.md)); isolated +contexts have one namespace, the child's `__main__`. + +## Handle + +A term that stands for something living elsewhere: `{'$py_shm', Id, Path, Size}` +(shared region), `{'$py_buffer', Id, Path, Ring}` (shared buffer), NIF +resource references (native buffers, channels, `py_ref` object references). +Only the first two cross a process boundary. + +## Child + +The OS process an isolated context runs Python in, started from +`priv/py_isolated_child.py`. It is restarted on crash within the context's +restart budget; its state does not survive a restart. + +## Interrupt, kill + +`py_context:interrupt/1` stops the request executing now (at the next +bytecode in embedded modes, immediately in the child through a signal). +`py_context:kill/1` sends `SIGKILL` to an isolated child; there is no +equivalent for embedded contexts. diff --git a/priv/_erlang_impl/README.md b/priv/_erlang_impl/README.md new file mode 100644 index 0000000..36f5066 --- /dev/null +++ b/priv/_erlang_impl/README.md @@ -0,0 +1,29 @@ +# _erlang_impl + +The Python half of the `erlang` module. `priv/` is on `sys.path` of every +interpreter erlang_python starts. In embedded modes the C module `erlang` +(`c_src/py_callback.c`) is the primary and delegates to this package for the +asyncio loop, channels, servers and helpers; in an isolated child there is +no C module and `_isolated.py` builds the whole `erlang` module from here. + +| Module | What it is | Embedded | Child | +|---|---|---|---| +| `__init__.py` | Public surface: `run`, `sleep`, `spawn_task`, `new_event_loop`, `install`, `atom`, `channel`, `byte_channel`, `server`, `reactor` | yes | partly (the child shim re-exports `server` and reimplements the rest on the stdlib loop) | +| `_loop.py` | `ErlangEventLoop`: asyncio loop whose readiness comes from `enif_select` and timers from `erlang:send_after`, through the `py_event_loop` C module | yes | no | +| `_policy.py`, `_transport.py` | Loop policy and transports for the loop above | yes | no | +| `_reactor.py` | Protocol-based reactor on fds Erlang owns | yes | no | +| `_channel.py`, `_byte_channel.py` | `Channel`, `ByteChannel` over NIF resources | yes | no | +| `_server.py` | `serve(listen_fd, factory)`, `adopt(fd, factory)`, `stop_serving`: plain asyncio on an fd handed over by Erlang | yes | yes | +| `_sandbox.py`, `_subprocess.py` | Audit hook that blocks fork/exec inside the VM | yes | no | +| `_mode.py` | Detects the execution mode | yes | yes | +| `_etf.py` | ETF codec with the `py_convert.c` type mapping; opaque `Pid`, `Ref`, `Port` keep their raw bytes | no | yes | +| `_isolated.py` | Child runtime: frame parser, reader thread, re-entrant main loop, interrupt signal, execution stack, asyncio loop, `SharedMemory` conversion, the `erlang` shim | no | yes | +| `_shm.py` | `SharedMemory` and `SharedBuffer` over `mmap`; `from_term` cache; buffer flow control through Erlang callbacks | yes | yes | + +Conventions: modules starting with `_` are internal; user code imports +`erlang`. Anything a mode cannot support raises `RuntimeError` with the +mode in the message rather than degrading silently (`_isolated.py`, +`_subprocess.py`). + +Tests for the loop live in `priv/test_erlang_loop.py` and `priv/tests/`; +the Erlang suites drive everything else. diff --git a/rebar.config b/rebar.config index 0010a51..d3ee4e1 100644 --- a/rebar.config +++ b/rebar.config @@ -76,6 +76,9 @@ <<"docs/security.md">>, <<"docs/distributed.md">>, <<"docs/testing-free-threading.md">>, + <<"docs/architecture.md">>, + <<"docs/code-map.md">>, + <<"docs/glossary.md">>, <<"docs/preload.md">>, <<"docs/owngil_internals.md">>, <<"docs/event_loop_architecture.md">> @@ -111,6 +114,9 @@ <<"docs/testing-free-threading.md">> ]}, {<<"Internals">>, [ + <<"docs/architecture.md">>, + <<"docs/code-map.md">>, + <<"docs/glossary.md">>, <<"docs/preload.md">>, <<"docs/owngil_internals.md">>, <<"docs/event_loop_architecture.md">> From 50a761063404aa305851db16ad051d1dd70c8e11 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sat, 29 Aug 2026 15:37:44 +0200 Subject: [PATCH 06/15] Remove the legacy worker API and its executor (#75) Contexts are the only execution path; the per-worker NIFs, the single executor thread they ran on, the deprecated async worker NIFs, the unused worker pool, the cancel aliases and the unreachable inline executor branches of the context NIFs are gone (about 3,000 lines). This removes public NIF functions, so the open release becomes 5.0.0. memory_stats and gc run under the GIL on the calling scheduler. The generator example uses py:stream. --- CHANGELOG.md | 15 +- c_src/README.md | 10 +- c_src/py_callback.c | 690 +-------------------------- c_src/py_event_loop.c | 14 - c_src/py_event_loop.h | 14 - c_src/py_exec.c | 793 ------------------------------- c_src/py_nif.c | 975 ++++---------------------------------- c_src/py_nif.h | 355 +------------- c_src/py_worker_pool.c | 921 ----------------------------------- c_src/py_worker_pool.h | 496 ------------------- docs/architecture.md | 22 +- docs/code-map.md | 17 +- docs/glossary.md | 4 +- docs/scalability.md | 2 +- examples/gen_test.erl | 91 ++-- src/erlang_python.app.src | 2 +- src/py_nif.erl | 231 --------- 17 files changed, 169 insertions(+), 4483 deletions(-) delete mode 100644 c_src/py_worker_pool.c delete mode 100644 c_src/py_worker_pool.h diff --git a/CHANGELOG.md b/CHANGELOG.md index a143464..437a85c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 4.2.0 (2026-08-29) +## 5.0.0 (2026-08-29) ### Added @@ -49,6 +49,19 @@ storms, loop churn, 60 s mixed workload with resource counters checked. - Guide: `docs/isolated.md`, with what each of the three modes guarantees. +### Removed + +- The legacy worker API (`py_nif:worker_new/0,1`, `worker_call`, `worker_eval`, + `worker_exec`, `worker_next`, `worker_destroy`, `import_module/2`, + `get_attr/3`, `set_callback_handler/2`, `send_callback_response/2`, + `resume_callback/2`) and the single executor thread behind it, the + `async_worker_*`/`async_call`/`async_gather`/`async_stream` NIFs that only + returned `deprecated`, the unused worker pool (`pool_*` NIFs), the + `cancel_reader/writer` aliases, and the unreachable inline executor + branches of the context NIFs. Contexts (`py_context`, `py:call/3`) are the + only execution path. `py:memory_stats/0` and `py:gc/0,1` now run on the + calling scheduler under the GIL. + ### Fixed - `pthread_timedjoin_np` was called without `_GNU_SOURCE`, an implicit diff --git a/c_src/README.md b/c_src/README.md index a8f240e..8d1dc06 100644 --- a/c_src/README.md +++ b/c_src/README.md @@ -16,7 +16,7 @@ where things are. | `py_nif.h` | All shared types: `py_context_t` and its request queue, request types, callback and suspension state, runtime state machine, atoms, globals, declarations | 2.4k lines. The struct comments carry the locking rules; read `py_context_t` before touching threads | | `py_nif.c` | Runtime init and finalize, resource types, context create/destroy, the request queue, `worker_context_thread_main` and `owngil_context_thread_main`, `owngil_execute_*` (used by both thread kinds), the `nif_context_*` NIFs, process-local envs, `py_ref`, the NIF function table at the end | Sections are banner-separated; `grep -n '^ \* ===\|^/\* ==='` lists them | | `py_convert.c` | `py_to_term`, `term_to_py`, depth limits, tagged tuples (`{bytes, B}`, `{'$py_shm', ...}`), error tuples `{error, {Type, Msg}}` | The type mapping tables in the comments are the reference for `_etf.py` | -| `py_exec.c` | Executing a call/eval/exec with suspension support; the legacy single executor thread | | +| `py_exec.c` | Execution mode detection (free-threaded or GIL build) | | | `py_callback.c` | The `erlang` Python module: `call`, `send`, `whereis`, `schedule*`, `Atom`/`Pid`/`Ref` types, callback delivery paths (suspension, blocking pipe, async pipe), channel and shared-dict methods, callback name registry | `erlang_call_impl` documents the path precedence | | `py_thread_worker.c` | Python threads calling Erlang through `py_thread_handler` | | | `py_subinterp_thread.c/.h` | Thread pool of sub-interpreters (owngil contexts, loop pools) | | @@ -24,7 +24,6 @@ where things are. | `py_channel.c/.h`, `py_buffer.c/.h`, `py_reactor_buffer.c/.h`, `py_shared_dict.c` | Resources with a Python-facing object each | | | `py_logging.c` | Logging and tracing NIFs | | | `py_mem_limit.c` | obmalloc arena accounting for owngil memory caps | | -| `py_worker_pool.c/.h` | Legacy pool, no caller in `src/` | Candidate for removal | | `py_util.c/.h` | Macros, small helpers | | ## Where the live paths are @@ -62,10 +61,3 @@ where things are. 3. Add the stub and its `-spec` and doc to `src/py_nif.erl`. 4. Cover it in a suite; `rebar3 dialyzer` and `rebar3 xref` must stay clean. -## Legacy code, for orientation - -Not on the path of contexts created today: the `worker_*` NIFs and -"Worker management" section, `async_worker_*` NIFs (return `deprecated`), -the "Legacy mode" inline branches in `nif_context_call/eval/exec`, -`py_worker_pool.c`, and the `cancel_reader/writer` aliases. When in doubt, -follow `nif_context_call_async` and ignore the rest. diff --git a/c_src/py_callback.c b/c_src/py_callback.c index ba630a2..b560673 100644 --- a/c_src/py_callback.c +++ b/c_src/py_callback.c @@ -337,325 +337,6 @@ static void cleanup_callback_registry(void) { pthread_mutex_unlock(&g_callback_registry_mutex); } -/* ============================================================================ - * Suspended state management - * ============================================================================ */ - -/** - * Source type for suspended state creation. - * Indicates whether the source is a request or an existing suspended state. - */ -typedef enum { - SUSPENDED_SOURCE_REQUEST, /* Source is py_request_t */ - SUSPENDED_SOURCE_EXISTING /* Source is suspended_state_t */ -} suspended_source_type_t; - -/** - * Source union for suspended state creation. - * Contains pointers to either request or existing suspended state. - */ -typedef struct { - suspended_source_type_t type; - union { - py_request_t *req; /* For SUSPENDED_SOURCE_REQUEST */ - suspended_state_t *existing; /* For SUSPENDED_SOURCE_EXISTING */ - } data; -} suspended_source_t; - -/** - * Internal cleanup helper for suspended state creation failure. - */ -static void cleanup_suspended_state_partial(suspended_state_t *state, PyObject *callback_args) { - if (state->orig_env != NULL) { - enif_free_env(state->orig_env); - } - if (state->callback_args != NULL) { - Py_DECREF(state->callback_args); - } else if (callback_args != NULL) { - Py_DECREF(callback_args); - } - if (state->callback_func_name != NULL) { - enif_free(state->callback_func_name); - } - enif_release_resource(state); -} - -/** - * Create a suspended state resource from exception args. - * Args tuple format: (callback_id, func_name, args) - * - * This unified function handles both: - * - Creating from a request (initial suspension) - * - Creating from an existing suspended state (nested suspension during replay) - * - * @param env NIF environment - * @param exc_args Exception args tuple from erlang.call() - * @param source Source of original request data - * @return suspended_state_t* or NULL on error - */ -static suspended_state_t *create_suspended_state_ex( - ErlNifEnv *env, PyObject *exc_args, const suspended_source_t *source) { - - (void)env; /* Only needed for future extensions */ - - if (!PyTuple_Check(exc_args) || PyTuple_Size(exc_args) != 3) { - return NULL; - } - - PyObject *callback_id_obj = PyTuple_GetItem(exc_args, 0); - PyObject *func_name_obj = PyTuple_GetItem(exc_args, 1); - PyObject *callback_args = PyTuple_GetItem(exc_args, 2); - - if (!PyLong_Check(callback_id_obj) || !PyUnicode_Check(func_name_obj)) { - return NULL; - } - - /* Allocate the suspended state resource */ - suspended_state_t *state = enif_alloc_resource( - SUSPENDED_STATE_RESOURCE_TYPE, sizeof(suspended_state_t)); - if (state == NULL) { - return NULL; - } - - /* Initialize the state */ - memset(state, 0, sizeof(suspended_state_t)); - - /* Set worker based on source type */ - if (source->type == SUSPENDED_SOURCE_REQUEST) { - state->worker = tl_current_worker; - } else { - state->worker = source->data.existing->worker; - } - /* Keep the worker resource alive for as long as the suspended state exists. - * Without this the worker can be GC'd while a callback is suspended, and - * nif_resume_callback_dirty would dereference a freed worker (use-after-free - * with the GIL held). Mirrors the enif_keep_resource(ctx) on the context path; - * suspended_state_destructor releases it. */ - if (state->worker != NULL) { - enif_keep_resource(state->worker); - } - - state->callback_id = PyLong_AsUnsignedLongLong(callback_id_obj); - - /* Copy callback function name */ - Py_ssize_t len; - const char *func_name = PyUnicode_AsUTF8AndSize(func_name_obj, &len); - if (func_name == NULL) { - enif_release_resource(state); - return NULL; - } - state->callback_func_name = enif_alloc(len + 1); - if (state->callback_func_name == NULL) { - enif_release_resource(state); - return NULL; - } - memcpy(state->callback_func_name, func_name, len); - state->callback_func_name[len] = '\0'; - state->callback_func_len = len; - - /* Store reference to callback args */ - Py_INCREF(callback_args); - state->callback_args = callback_args; - - /* Get request type and timeout based on source */ - int request_type; - unsigned long timeout_ms; - - if (source->type == SUSPENDED_SOURCE_REQUEST) { - request_type = source->data.req->type; - timeout_ms = source->data.req->timeout_ms; - } else { - request_type = source->data.existing->request_type; - timeout_ms = source->data.existing->orig_timeout_ms; - } - - state->request_type = request_type; - state->orig_timeout_ms = timeout_ms; - - /* Create environment to hold copied terms */ - state->orig_env = enif_alloc_env(); - if (state->orig_env == NULL) { - cleanup_suspended_state_partial(state, NULL); - return NULL; - } - - /* Copy request-specific data based on source type and request type */ - if (request_type == PY_REQ_CALL) { - ErlNifBinary *src_module, *src_func; - ERL_NIF_TERM src_args, src_kwargs; - ErlNifEnv *src_env; - - if (source->type == SUSPENDED_SOURCE_REQUEST) { - src_module = &source->data.req->module_bin; - src_func = &source->data.req->func_bin; - src_args = source->data.req->args_term; - src_kwargs = source->data.req->kwargs_term; - src_env = source->data.req->env; - } else { - src_module = &source->data.existing->orig_module; - src_func = &source->data.existing->orig_func; - src_args = source->data.existing->orig_args; - src_kwargs = source->data.existing->orig_kwargs; - src_env = source->data.existing->orig_env; - } - - /* Copy module binary */ - if (!enif_alloc_binary(src_module->size, &state->orig_module)) { - cleanup_suspended_state_partial(state, NULL); - return NULL; - } - memcpy(state->orig_module.data, src_module->data, src_module->size); - - /* Copy function binary */ - if (!enif_alloc_binary(src_func->size, &state->orig_func)) { - enif_release_binary(&state->orig_module); - cleanup_suspended_state_partial(state, NULL); - return NULL; - } - memcpy(state->orig_func.data, src_func->data, src_func->size); - - /* Copy args and kwargs to our environment */ - state->orig_args = enif_make_copy(state->orig_env, src_args); - state->orig_kwargs = enif_make_copy(state->orig_env, src_kwargs); - (void)src_env; /* Used implicitly by enif_make_copy */ - - } else if (request_type == PY_REQ_EVAL) { - ErlNifBinary *src_code; - ERL_NIF_TERM src_locals; - ErlNifEnv *src_env; - - if (source->type == SUSPENDED_SOURCE_REQUEST) { - src_code = &source->data.req->code_bin; - src_locals = source->data.req->locals_term; - src_env = source->data.req->env; - } else { - src_code = &source->data.existing->orig_code; - src_locals = source->data.existing->orig_locals; - src_env = source->data.existing->orig_env; - } - - /* Copy code binary */ - if (!enif_alloc_binary(src_code->size, &state->orig_code)) { - cleanup_suspended_state_partial(state, NULL); - return NULL; - } - memcpy(state->orig_code.data, src_code->data, src_code->size); - - /* Copy locals */ - state->orig_locals = enif_make_copy(state->orig_env, src_locals); - (void)src_env; /* Used implicitly by enif_make_copy */ - } - - /* Initialize synchronization primitives */ - pthread_mutex_init(&state->mutex, NULL); - pthread_cond_init(&state->cond, NULL); - - state->result_data = NULL; - state->result_len = 0; - state->has_result = false; - state->is_error = false; - - return state; -} - -/** - * Create a suspended state resource from a request. - * Wrapper for create_suspended_state_ex for initial suspension. - */ -static suspended_state_t *create_suspended_state(ErlNifEnv *env, PyObject *exc_args, - py_request_t *req) { - suspended_source_t source = { - .type = SUSPENDED_SOURCE_REQUEST, - .data.req = req - }; - return create_suspended_state_ex(env, exc_args, &source); -} - -/** - * Create a new suspended state from an existing one (for nested suspensions). - * Wrapper for create_suspended_state_ex for nested suspension during replay. - */ -static suspended_state_t *create_suspended_state_from_existing( - ErlNifEnv *env, PyObject *exc_args, suspended_state_t *existing) { - suspended_source_t source = { - .type = SUSPENDED_SOURCE_EXISTING, - .data.existing = existing - }; - return create_suspended_state_ex(env, exc_args, &source); -} - -/** - * Build exception args tuple from thread-local pending callback state. - * - * This helper extracts the common pattern of building the exc_args tuple - * (callback_id, func_name, args) from thread-local storage. - * - * @return PyObject* tuple on success, NULL on failure - * @note On failure, tl_pending_callback is cleared - * @note Caller must Py_DECREF the returned tuple when done - */ -static PyObject *build_pending_callback_exc_args(void) { - PyObject *exc_args = PyTuple_New(3); - if (exc_args == NULL) { - tl_pending_callback = false; - Py_CLEAR(tl_pending_args); - return NULL; - } - - PyObject *callback_id_obj = PyLong_FromUnsignedLongLong(tl_pending_callback_id); - PyObject *func_name_obj = PyUnicode_FromStringAndSize( - tl_pending_func_name, tl_pending_func_name_len); - - if (callback_id_obj == NULL || func_name_obj == NULL) { - Py_XDECREF(callback_id_obj); - Py_XDECREF(func_name_obj); - Py_DECREF(exc_args); - tl_pending_callback = false; - Py_CLEAR(tl_pending_args); - return NULL; - } - - PyTuple_SET_ITEM(exc_args, 0, callback_id_obj); - PyTuple_SET_ITEM(exc_args, 1, func_name_obj); - Py_INCREF(tl_pending_args); /* Tuple takes ownership */ - PyTuple_SET_ITEM(exc_args, 2, tl_pending_args); - - return exc_args; -} - -/** - * Build the {suspended, ...} result term from a suspended state. - * - * Common helper for creating the suspension result after a callback - * is detected during Python execution. - * - * @param env NIF environment - * @param suspended Suspended state (resource will be released) - * @return ERL_NIF_TERM {suspended, CallbackId, StateRef, {FuncName, Args}} - * @note Clears tl_pending_callback - */ -static ERL_NIF_TERM build_suspended_result(ErlNifEnv *env, suspended_state_t *suspended) { - ERL_NIF_TERM state_ref = enif_make_resource(env, suspended); - enif_release_resource(suspended); - - ERL_NIF_TERM callback_id_term = enif_make_uint64(env, tl_pending_callback_id); - - ERL_NIF_TERM func_name_term; - unsigned char *fn_buf = enif_make_new_binary(env, tl_pending_func_name_len, &func_name_term); - memcpy(fn_buf, tl_pending_func_name, tl_pending_func_name_len); - - ERL_NIF_TERM args_term = py_to_term(env, tl_pending_args); - - tl_pending_callback = false; - Py_CLEAR(tl_pending_args); - - return enif_make_tuple4(env, - ATOM_SUSPENDED, - callback_id_term, - state_ref, - enif_make_tuple2(env, func_name_term, args_term)); -} - /* ============================================================================ * Context suspension helpers (for process-per-context architecture) * @@ -1881,8 +1562,7 @@ static PyObject *erlang_call_impl(PyObject *self, PyObject *args) { * Priority: * 1. tl_current_context with suspension enabled (new process-per-context API) * 2. tl_current_context with callback_handler (old blocking pipe mode) - * 3. tl_current_worker (legacy worker API) - * 4. thread_worker_call (spawned threads) + * 3. thread_worker_call (spawned threads) * * NOTE: In OWN_GIL mode, erlang.call() goes through thread_worker_call() * rather than using suspension/resume. This is because OWN_GIL contexts @@ -1893,9 +1573,8 @@ static PyObject *erlang_call_impl(PyObject *self, PyObject *args) { */ bool has_context_suspension = (tl_current_context != NULL && tl_allow_suspension); bool has_context_handler = (tl_current_context != NULL && tl_current_context->has_callback_handler); - bool has_worker_handler = (tl_current_worker != NULL && tl_current_worker->has_callback_handler); - if (!has_context_suspension && !has_context_handler && !has_worker_handler) { + if (!has_context_suspension && !has_context_handler) { /* * Not an executor thread - use thread worker path. * This enables any spawned Python thread to call erlang.call(): @@ -1951,22 +1630,6 @@ static PyObject *erlang_call_impl(PyObject *self, PyObject *args) { } size_t func_name_len = strlen(func_name); - /* Check if we have a suspended state with a cached result (replay case) */ - if (tl_current_suspended != NULL && tl_current_suspended->has_result) { - /* Verify this is the same callback */ - if (tl_current_suspended->callback_func_len == func_name_len && - memcmp(tl_current_suspended->callback_func_name, func_name, func_name_len) == 0) { - /* Return the cached result - parse using ast.literal_eval */ - PyObject *result = parse_callback_response( - tl_current_suspended->result_data, - tl_current_suspended->result_len); - /* Mark result as consumed (don't clear tl_current_suspended yet, - * as we might need it for nested callbacks in the future) */ - tl_current_suspended->has_result = false; - return result; - } - } - /* Check for context-based suspended state with cached results (context replay case) */ if (tl_current_context_suspended != NULL) { /* @@ -2022,23 +1685,6 @@ static PyObject *erlang_call_impl(PyObject *self, PyObject *args) { /* If we get here, this is a NEW callback - will suspend below */ } - /* - * FIX for multiple sequential erlang.call(): - * If we're in WORKER replay context (tl_current_suspended != NULL) but didn't get - * a cache hit above, this is a SUBSEQUENT call (e.g., second erlang.call() - * in the same Python function). For WORKER mode, the callback handler process - * is still running and will handle this via blocking pipe. - * - * For CONTEXT replay (tl_current_context_suspended != NULL), we CANNOT block - * because there's no callback handler process. Instead, we must suspend again - * and let the context process handle the subsequent callback. This works because - * the context process re-replays from the beginning, and each callback result - * is returned via the cached result mechanism on subsequent replays. - */ - bool force_blocking = (tl_current_suspended != NULL); - /* Note: tl_current_context_suspended is NOT included here - context mode - * always uses suspension for callbacks, allowing unlimited nesting via replay */ - /* Build args list (remaining args) */ PyObject *call_args = PyTuple_GetSlice(args, 1, nargs); if (call_args == NULL) { @@ -2051,9 +1697,8 @@ static PyObject *erlang_call_impl(PyObject *self, PyObject *args) { * executor (PY_REQ_CALL or PY_REQ_EVAL). For PY_REQ_EXEC or nested Python * code, we must block and wait for the result. * - * Also block if force_blocking is set (replay context with no cache hit). */ - if (!tl_allow_suspension || force_blocking) { + if (!tl_allow_suspension) { /* Fall back to blocking behavior - send message and wait on pipe */ ErlNifEnv *msg_env = enif_alloc_env(); if (msg_env == NULL) { @@ -2083,16 +1728,15 @@ static PyObject *erlang_call_impl(PyObject *self, PyObject *args) { uint32_t response_len = 0; int read_result; - /* Get callback handler and pipe from context or worker */ - ErlNifPid *handler_pid; - int read_fd; - if (has_context_handler) { - handler_pid = &tl_current_context->callback_handler; - read_fd = tl_current_context->callback_pipe[0]; - } else { - handler_pid = &tl_current_worker->callback_handler; - read_fd = tl_current_worker->callback_pipe[0]; + /* Callback handler and pipe of the context */ + if (!has_context_handler) { + Py_DECREF(call_args); + enif_free_env(msg_env); + PyErr_SetString(PyExc_RuntimeError, "erlang.call: no callback handler for this context"); + return NULL; } + ErlNifPid *handler_pid = &tl_current_context->callback_handler; + int read_fd = tl_current_context->callback_pipe[0]; Py_BEGIN_ALLOW_THREADS enif_send(NULL, handler_pid, msg_env, msg); @@ -4341,318 +3985,6 @@ static int create_erlang_module(void) { * event-driven operation without pthread polling. * ============================================================================ */ -/* ============================================================================ - * Resume callback NIFs - * ============================================================================ */ - -/* Forward declaration for the dirty resume NIF */ -static ERL_NIF_TERM nif_resume_callback_dirty(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); - -/** - * Resume a suspended callback by storing the result and scheduling replay. - * - * Args: StateRef, ResultBinary - * - * This NIF stores the callback result in the suspended state and schedules - * a dirty NIF (nif_resume_callback_dirty) to replay the Python code. - */ -static ERL_NIF_TERM nif_resume_callback(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - (void)argc; - suspended_state_t *state; - ErlNifBinary result_bin; - - if (!runtime_is_running()) { - return make_error(env, "python_not_running"); - } - - if (!enif_get_resource(env, argv[0], SUSPENDED_STATE_RESOURCE_TYPE, (void **)&state)) { - return make_error(env, "invalid_state_ref"); - } - - if (!enif_inspect_binary(env, argv[1], &result_bin)) { - return make_error(env, "invalid_result"); - } - - /* Store the result in the suspended state */ - pthread_mutex_lock(&state->mutex); - - /* Copy result data. Free any prior result first: a duplicate/raced resume - * would otherwise leak the previous buffer. (has_result is not a one-shot - * flag -- it toggles during nested replay -- so result_data is the real - * pending-result indicator.) */ - if (state->result_data != NULL) { - enif_free(state->result_data); - state->result_data = NULL; - } - state->result_data = enif_alloc(result_bin.size); - if (state->result_data == NULL) { - pthread_mutex_unlock(&state->mutex); - return make_error(env, "alloc_failed"); - } - memcpy(state->result_data, result_bin.data, result_bin.size); - state->result_len = result_bin.size; - state->has_result = true; - state->is_error = false; - - pthread_mutex_unlock(&state->mutex); - - /* - * Schedule the dirty resume NIF. - * This allows the current NIF to return immediately, and the dirty NIF - * will handle the Python replay on a dirty scheduler. - */ - ERL_NIF_TERM new_argv[1] = { argv[0] }; /* Pass StateRef to dirty NIF */ - return enif_schedule_nif(env, "resume_callback_dirty", - ERL_NIF_DIRTY_JOB_IO_BOUND, nif_resume_callback_dirty, 1, new_argv); -} - -/** - * Dirty NIF that replays Python code with the cached callback result. - * - * This is scheduled by nif_resume_callback and runs on a dirty I/O scheduler. - * It sets tl_current_suspended so erlang_call_impl can return the cached result, - * then re-runs the original Python code. When Python hits erlang.call() again, - * it gets the cached result and continues normally. - */ -static ERL_NIF_TERM nif_resume_callback_dirty(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - (void)argc; - suspended_state_t *state; - - if (!runtime_is_running()) { - return make_error(env, "python_not_running"); - } - - if (!enif_get_resource(env, argv[0], SUSPENDED_STATE_RESOURCE_TYPE, (void **)&state)) { - return make_error(env, "invalid_state_ref"); - } - - /* Verify the state has a result */ - if (!state->has_result) { - return make_error(env, "no_result"); - } - - /* The worker is kept alive for the lifetime of the suspended state, but - * guard rather than dereference NULL in the replay below. */ - if (state->worker == NULL) { - return make_error(env, "no_worker"); - } - - /* Set up thread-local state for replay */ - tl_current_worker = state->worker; - tl_callback_env = env; - tl_current_suspended = state; /* erlang_call_impl will check this */ - tl_allow_suspension = true; - - ERL_NIF_TERM result; - - if (state->request_type == PY_REQ_CALL) { - /* Replay a py:call */ - char *module_name = enif_alloc(state->orig_module.size + 1); - char *func_name = enif_alloc(state->orig_func.size + 1); - - if (module_name == NULL || func_name == NULL) { - enif_free(module_name); - enif_free(func_name); - tl_current_suspended = NULL; - return make_error(env, "alloc_failed"); - } - - memcpy(module_name, state->orig_module.data, state->orig_module.size); - module_name[state->orig_module.size] = '\0'; - memcpy(func_name, state->orig_func.data, state->orig_func.size); - func_name[state->orig_func.size] = '\0'; - - PyGILState_STATE gstate = PyGILState_Ensure(); - - PyObject *func = NULL; - - /* Get the function (same logic as process_request) */ - if (strcmp(module_name, "__main__") == 0) { - func = PyDict_GetItemString(state->worker->locals, func_name); - if (func == NULL) { - func = PyDict_GetItemString(state->worker->globals, func_name); - } - if (func != NULL) { - Py_INCREF(func); - } else { - PyErr_Format(PyExc_NameError, "name '%s' is not defined", func_name); - result = make_py_error(env); - goto call_cleanup; - } - } else { - PyObject *module = PyImport_ImportModule(module_name); - if (module == NULL) { - result = make_py_error(env); - goto call_cleanup; - } - func = PyObject_GetAttrString(module, func_name); - Py_DECREF(module); - } - - if (func == NULL) { - result = make_py_error(env); - goto call_cleanup; - } - - /* Convert args */ - unsigned int args_len; - if (!enif_get_list_length(state->orig_env, state->orig_args, &args_len)) { - Py_DECREF(func); - result = make_error(env, "invalid_args"); - goto call_cleanup; - } - - PyObject *args = PyTuple_New(args_len); - if (args == NULL) { - Py_DECREF(func); - result = make_error(env, "alloc_failed"); - goto call_cleanup; - } - ERL_NIF_TERM head, tail = state->orig_args; - for (unsigned int i = 0; i < args_len; i++) { - enif_get_list_cell(state->orig_env, tail, &head, &tail); - PyObject *arg = term_to_py(state->orig_env, head); - if (arg == NULL) { - Py_DECREF(args); - Py_DECREF(func); - result = make_error(env, "arg_conversion_failed"); - goto call_cleanup; - } - PyTuple_SET_ITEM(args, i, arg); - } - - /* Convert kwargs */ - PyObject *kwargs = NULL; - if (enif_is_map(state->orig_env, state->orig_kwargs)) { - kwargs = term_to_py(state->orig_env, state->orig_kwargs); - } - - /* Call the function (this will hit erlang.call which returns cached result) */ - PyObject *py_result = PyObject_Call(func, args, kwargs); - - Py_DECREF(func); - Py_DECREF(args); - Py_XDECREF(kwargs); - - if (py_result == NULL) { - if (tl_pending_callback) { - /* - * Flag-based callback detection during replay. - * Check flag FIRST, not exception type - this works even if - * Python code caught and re-raised the exception. - */ - PyErr_Clear(); /* Clear whatever exception is set */ - - /* Build exc_args tuple from thread-local storage */ - PyObject *exc_args = build_pending_callback_exc_args(); - if (exc_args == NULL) { - result = make_error(env, "build_exc_args_failed"); - } else { - suspended_state_t *new_suspended = create_suspended_state_from_existing(env, exc_args, state); - Py_DECREF(exc_args); - if (new_suspended == NULL) { - tl_pending_callback = false; - Py_CLEAR(tl_pending_args); - result = make_error(env, "create_nested_suspended_state_failed"); - } else { - result = build_suspended_result(env, new_suspended); - } - } - } else { - result = make_py_error(env); - } - } else { - ERL_NIF_TERM term_result = py_to_term(env, py_result); - Py_DECREF(py_result); - result = enif_make_tuple2(env, ATOM_OK, term_result); - } - - call_cleanup: - PyGILState_Release(gstate); - enif_free(module_name); - enif_free(func_name); - - } else if (state->request_type == PY_REQ_EVAL) { - /* Replay a py:eval */ - char *code = enif_alloc(state->orig_code.size + 1); - if (code == NULL) { - tl_current_suspended = NULL; - return make_error(env, "alloc_failed"); - } - memcpy(code, state->orig_code.data, state->orig_code.size); - code[state->orig_code.size] = '\0'; - - PyGILState_STATE gstate = PyGILState_Ensure(); - - /* Update locals if provided */ - if (enif_is_map(state->orig_env, state->orig_locals)) { - PyObject *new_locals = term_to_py(state->orig_env, state->orig_locals); - if (new_locals != NULL && PyDict_Check(new_locals)) { - PyDict_Update(state->worker->locals, new_locals); - Py_DECREF(new_locals); - } - } - - /* Compile and evaluate */ - PyObject *compiled = Py_CompileString(code, "", Py_eval_input); - - if (compiled == NULL) { - result = make_py_error(env); - } else { - PyObject *py_result = PyEval_EvalCode(compiled, state->worker->globals, - state->worker->locals); - Py_DECREF(compiled); - - if (py_result == NULL) { - if (tl_pending_callback) { - /* - * Flag-based callback detection during eval replay. - * Check flag FIRST, not exception type - this works even if - * Python code caught and re-raised the exception. - */ - PyErr_Clear(); /* Clear whatever exception is set */ - - /* Build exc_args tuple from thread-local storage */ - PyObject *exc_args = build_pending_callback_exc_args(); - if (exc_args == NULL) { - result = make_error(env, "build_exc_args_failed"); - } else { - suspended_state_t *new_suspended = create_suspended_state_from_existing(env, exc_args, state); - Py_DECREF(exc_args); - if (new_suspended == NULL) { - tl_pending_callback = false; - Py_CLEAR(tl_pending_args); - result = make_error(env, "create_nested_suspended_state_failed"); - } else { - result = build_suspended_result(env, new_suspended); - } - } - } else { - result = make_py_error(env); - } - } else { - ERL_NIF_TERM term_result = py_to_term(env, py_result); - Py_DECREF(py_result); - result = enif_make_tuple2(env, ATOM_OK, term_result); - } - } - - PyGILState_Release(gstate); - enif_free(code); - - } else { - result = make_error(env, "unsupported_request_type"); - } - - /* Clear thread-local state */ - tl_current_worker = NULL; - tl_callback_env = NULL; - tl_current_suspended = NULL; - tl_allow_suspension = false; - - return result; -} - /* ============================================================================ * NIF functions for callback name registration * ============================================================================ */ diff --git a/c_src/py_event_loop.c b/c_src/py_event_loop.c index 198b048..91b37ca 100644 --- a/c_src/py_event_loop.c +++ b/c_src/py_event_loop.c @@ -4838,21 +4838,7 @@ ERL_NIF_TERM nif_start_writer(ErlNifEnv *env, int argc, } /* Legacy aliases for backward compatibility */ -ERL_NIF_TERM nif_cancel_reader(ErlNifEnv *env, int argc, - const ERL_NIF_TERM argv[]) { - /* cancel_reader(Loop, FdRef) -> stop_reader(FdRef) */ - (void)argc; - ERL_NIF_TERM new_argv[1] = {argv[1]}; /* Skip Loop arg */ - return nif_stop_reader(env, 1, new_argv); -} -ERL_NIF_TERM nif_cancel_writer(ErlNifEnv *env, int argc, - const ERL_NIF_TERM argv[]) { - /* cancel_writer(Loop, FdRef) -> stop_writer(FdRef) */ - (void)argc; - ERL_NIF_TERM new_argv[1] = {argv[1]}; /* Skip Loop arg */ - return nif_stop_writer(env, 1, new_argv); -} /** * close_fd(FdRef) -> ok diff --git a/c_src/py_event_loop.h b/c_src/py_event_loop.h index 2485e5b..e9e9413 100644 --- a/c_src/py_event_loop.h +++ b/c_src/py_event_loop.h @@ -922,21 +922,7 @@ ERL_NIF_TERM nif_stop_writer(ErlNifEnv *env, int argc, ERL_NIF_TERM nif_start_writer(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); -/** - * @brief Cancel read monitoring (legacy alias for stop_reader) - * - * NIF: cancel_reader(LoopRef, FdRef) -> ok | {error, Reason} - */ -ERL_NIF_TERM nif_cancel_reader(ErlNifEnv *env, int argc, - const ERL_NIF_TERM argv[]); -/** - * @brief Cancel write monitoring (legacy alias for stop_writer) - * - * NIF: cancel_writer(LoopRef, FdRef) -> ok | {error, Reason} - */ -ERL_NIF_TERM nif_cancel_writer(ErlNifEnv *env, int argc, - const ERL_NIF_TERM argv[]); /** * @brief Explicitly close an FD with proper lifecycle cleanup diff --git a/c_src/py_exec.c b/c_src/py_exec.c index 8672664..7430c0d 100644 --- a/c_src/py_exec.c +++ b/c_src/py_exec.c @@ -65,87 +65,6 @@ * @note This file is included from py_nif.c (single compilation unit) */ -/* ============================================================================ - * Timeout Support - * - * Python execution timeout is implemented using PyEval_SetTrace(), which - * installs a callback invoked at each Python instruction. This allows - * cooperative timeout checking without requiring signal handlers. - * ============================================================================ */ - -/** - * @brief Trace callback for timeout checking - * - * Called by Python at each line/call/return event when tracing is enabled. - * Checks if the deadline has passed and raises TimeoutError if so. - * - * @param obj Trace function argument (unused) - * @param frame Current execution frame (unused) - * @param what Event type (call/line/return/exception) - * @param arg Event-specific argument (unused) - * - * @return 0 to continue, -1 to abort with exception - * - * @note Called with GIL held - * @note Uses thread-local storage for deadline - */ -static int python_trace_callback(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) { - (void)obj; - (void)frame; - (void)what; - (void)arg; - if (tl_timeout_enabled && tl_timeout_deadline > 0) { - if (get_monotonic_ns() > tl_timeout_deadline) { - PyErr_SetString(PyExc_TimeoutError, "execution timeout"); - return -1; /* Abort execution */ - } - } - return 0; -} - -/** - * @brief Enable timeout monitoring for Python execution - * - * Installs a trace callback that checks elapsed time against a deadline. - * If the deadline is exceeded, TimeoutError is raised in Python. - * - * @param timeout_ms Timeout in milliseconds (0 = no timeout) - * - * @par Implementation - * - * Uses thread-local storage to avoid global state: - * - `tl_timeout_deadline`: Absolute deadline (monotonic ns) - * - `tl_timeout_enabled`: Flag to enable checking - * - * @note Must be paired with stop_timeout() - * @note GIL must be held - * - * @see stop_timeout() - * @see python_trace_callback() - */ -static void start_timeout(unsigned long timeout_ms) { - if (timeout_ms > 0) { - tl_timeout_deadline = get_monotonic_ns() + (timeout_ms * 1000000ULL); - tl_timeout_enabled = true; - PyEval_SetTrace(python_trace_callback, NULL); - } -} - -static void stop_timeout(void) { - if (tl_timeout_enabled) { - tl_timeout_enabled = false; - tl_timeout_deadline = 0; - PyEval_SetTrace(NULL, NULL); - } -} - -static bool check_timeout_error(void) { - if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_TimeoutError)) { - return true; - } - return false; -} - /* ============================================================================ * Execution mode detection * ============================================================================ */ @@ -158,715 +77,3 @@ static void detect_execution_mode(void) { #endif } -/* ============================================================================ - * Request processing - * ============================================================================ */ - -/** - * Initialize a request structure. - */ -static void request_init(py_request_t *req) { - memset(req, 0, sizeof(py_request_t)); - pthread_mutex_init(&req->mutex, NULL); - pthread_cond_init(&req->cond, NULL); - req->completed = false; -} - -/** - * Clean up a request structure. - */ -static void request_cleanup(py_request_t *req) { - pthread_mutex_destroy(&req->mutex); - pthread_cond_destroy(&req->cond); -} - -/** - * Process a single request in the executor thread (GIL held). - */ -static void process_request(py_request_t *req) { - ErlNifEnv *env = req->env; - py_worker_t *worker = req->worker; - py_context_t *context = req->context; - - /* Extract globals/locals from context or worker */ - PyObject *globals = context ? context->globals : (worker ? worker->globals : NULL); - PyObject *locals = context ? context->locals : (worker ? worker->locals : NULL); - - switch (req->type) { - case PY_REQ_CALL: { - /* Set thread-local worker/context for callbacks */ - tl_current_worker = worker; - tl_current_context = context; - tl_callback_env = env; - tl_allow_suspension = false; /* Blocking mode - code runs once, no replay */ - - char *module_name = binary_to_string(&req->module_bin); - char *func_name = binary_to_string(&req->func_bin); - if (module_name == NULL || func_name == NULL) { - enif_free(module_name); - enif_free(func_name); - req->result = make_error(env, "alloc_failed"); - break; - } - - PyObject *func = NULL; - - /* Special handling for __main__ - look in globals/locals namespace first */ - if (strcmp(module_name, "__main__") == 0) { - func = PyDict_GetItemString(locals, func_name); - if (func == NULL) { - func = PyDict_GetItemString(globals, func_name); - } - if (func != NULL) { - Py_INCREF(func); - } - /* If not found in namespace, fall through to module import below */ - } - - if (func == NULL) { - /* Import module and get attribute */ - PyObject *module = PyImport_ImportModule(module_name); - if (module == NULL) { - req->result = make_py_error(env); - goto call_cleanup; - } - func = PyObject_GetAttrString(module, func_name); - Py_DECREF(module); - } - - if (func == NULL) { - req->result = make_py_error(env); - goto call_cleanup; - } - - /* Convert args list to Python tuple */ - unsigned int args_len; - if (!enif_get_list_length(env, req->args_term, &args_len)) { - Py_DECREF(func); - req->result = make_error(env, "invalid_args"); - goto call_cleanup; - } - - PyObject *args = PyTuple_New(args_len); - if (args == NULL) { - Py_DECREF(func); - req->result = make_error(env, "alloc_failed"); - goto call_cleanup; - } - ERL_NIF_TERM head, tail = req->args_term; - for (unsigned int i = 0; i < args_len; i++) { - enif_get_list_cell(env, tail, &head, &tail); - PyObject *arg = term_to_py(env, head); - if (arg == NULL) { - Py_DECREF(args); - Py_DECREF(func); - req->result = make_error(env, "arg_conversion_failed"); - goto call_cleanup; - } - PyTuple_SET_ITEM(args, i, arg); - } - - /* Convert kwargs map to Python dict */ - PyObject *kwargs = NULL; - if (enif_is_map(env, req->kwargs_term)) { - kwargs = term_to_py(env, req->kwargs_term); - } - - /* Start timeout if specified */ - start_timeout(req->timeout_ms); - - /* Call the function */ - PyObject *py_result = PyObject_Call(func, args, kwargs); - - /* Stop timeout */ - stop_timeout(); - - Py_DECREF(func); - Py_DECREF(args); - Py_XDECREF(kwargs); - - if (py_result == NULL) { - if (check_timeout_error()) { - PyErr_Clear(); - req->result = enif_make_tuple2(env, ATOM_ERROR, ATOM_TIMEOUT); - } else if (tl_pending_callback) { - /* - * Flag-based callback detection: check flag FIRST, not exception type. - * This works even if Python code caught and re-raised the exception. - */ - PyErr_Clear(); /* Clear whatever exception is set */ - - /* Build exc_args tuple from thread-local storage */ - PyObject *exc_args = build_pending_callback_exc_args(); - if (exc_args == NULL) { - req->result = make_error(env, "build_exc_args_failed"); - } else { - suspended_state_t *suspended = create_suspended_state(env, exc_args, req); - Py_DECREF(exc_args); - if (suspended == NULL) { - clear_pending_callback_tls(); - req->result = make_error(env, "create_suspended_state_failed"); - } else { - req->result = build_suspended_result(env, suspended); - /* func_name/args are copied into the suspended state; clear - * the pending-callback TLS so a later request on this reused - * worker thread doesn't trip the stale-TLS entry invariant. */ - clear_pending_callback_tls(); - } - } - } else { - req->result = make_py_error(env); - } - } else if (PyGen_Check(py_result) || PyIter_Check(py_result)) { - py_object_t *wrapper = enif_alloc_resource(PYOBJ_RESOURCE_TYPE, sizeof(py_object_t)); - if (wrapper == NULL) { - Py_DECREF(py_result); - req->result = make_error(env, "alloc_failed"); - } else { - wrapper->obj = py_result; - wrapper->interp_id = 0; /* Main interpreter */ - atomic_fetch_add(&g_counters.pyobj_created, 1); - ERL_NIF_TERM gen_ref = enif_make_resource(env, wrapper); - enif_release_resource(wrapper); - req->result = enif_make_tuple2(env, ATOM_OK, - enif_make_tuple2(env, ATOM_GENERATOR, gen_ref)); - } - } else if (is_inline_schedule_marker(py_result)) { - /* Inline schedule marker not supported in legacy worker NIFs. - * Note: py:call() uses the context API (nif_context_call), which - * does support schedule_inline. This code path is only hit by - * direct py_nif:worker_call usage, which is rare. */ - Py_DECREF(py_result); - req->result = make_error(env, "schedule_inline_not_supported_in_worker_mode"); - } else if (is_schedule_marker(py_result)) { - /* Schedule marker: release dirty scheduler, continue via callback */ - ScheduleMarkerObject *marker = (ScheduleMarkerObject *)py_result; - ERL_NIF_TERM callback_name = py_to_term(env, marker->callback_name); - ERL_NIF_TERM callback_args = py_to_term(env, marker->args); - Py_DECREF(py_result); - req->result = enif_make_tuple3(env, ATOM_SCHEDULE, callback_name, callback_args); - } else { - ERL_NIF_TERM term_result = py_to_term(env, py_result); - Py_DECREF(py_result); - req->result = enif_make_tuple2(env, ATOM_OK, term_result); - } - - call_cleanup: - tl_current_worker = NULL; - tl_current_context = NULL; - tl_callback_env = NULL; - tl_allow_suspension = false; - enif_free(module_name); - enif_free(func_name); - break; - } - - case PY_REQ_EVAL: { - tl_current_worker = worker; - tl_current_context = context; - tl_callback_env = env; - tl_allow_suspension = true; /* Allow suspension - we replay on resume */ - - char *code = binary_to_string(&req->code_bin); - if (code == NULL) { - req->result = make_error(env, "alloc_failed"); - break; - } - - /* Update locals if provided */ - if (enif_is_map(env, req->locals_term)) { - PyObject *new_locals = term_to_py(env, req->locals_term); - if (new_locals != NULL && PyDict_Check(new_locals)) { - PyDict_Update(locals, new_locals); - Py_DECREF(new_locals); - } - } - - /* Start timeout if specified */ - start_timeout(req->timeout_ms); - - /* Compile and evaluate */ - PyObject *compiled = Py_CompileString(code, "", Py_eval_input); - - if (compiled == NULL) { - stop_timeout(); - req->result = make_py_error(env); - } else { - PyObject *py_result = PyEval_EvalCode(compiled, globals, locals); - Py_DECREF(compiled); - stop_timeout(); - - if (py_result == NULL) { - if (check_timeout_error()) { - PyErr_Clear(); - req->result = enif_make_tuple2(env, ATOM_ERROR, ATOM_TIMEOUT); - } else if (tl_pending_callback) { - /* Flag-based callback detection for eval */ - PyErr_Clear(); - - PyObject *exc_args = build_pending_callback_exc_args(); - if (exc_args == NULL) { - req->result = make_error(env, "build_exc_args_failed"); - } else { - suspended_state_t *suspended = create_suspended_state(env, exc_args, req); - Py_DECREF(exc_args); - if (suspended == NULL) { - clear_pending_callback_tls(); - req->result = make_error(env, "create_suspended_state_failed"); - } else { - req->result = build_suspended_result(env, suspended); - clear_pending_callback_tls(); - } - } - } else { - req->result = make_py_error(env); - } - } else if (PyGen_Check(py_result) || PyIter_Check(py_result)) { - py_object_t *wrapper = enif_alloc_resource(PYOBJ_RESOURCE_TYPE, sizeof(py_object_t)); - if (wrapper == NULL) { - Py_DECREF(py_result); - req->result = make_error(env, "alloc_failed"); - } else { - wrapper->obj = py_result; - wrapper->interp_id = 0; /* Main interpreter */ - atomic_fetch_add(&g_counters.pyobj_created, 1); - ERL_NIF_TERM gen_ref = enif_make_resource(env, wrapper); - enif_release_resource(wrapper); - req->result = enif_make_tuple2(env, ATOM_OK, - enif_make_tuple2(env, ATOM_GENERATOR, gen_ref)); - } - } else if (is_inline_schedule_marker(py_result)) { - /* Inline schedule marker not supported in legacy worker NIFs. - * Note: py:call() uses the context API, which supports schedule_inline. */ - Py_DECREF(py_result); - req->result = make_error(env, "schedule_inline_not_supported_in_worker_mode"); - } else if (is_schedule_marker(py_result)) { - /* Schedule marker: release dirty scheduler, continue via callback */ - ScheduleMarkerObject *marker = (ScheduleMarkerObject *)py_result; - ERL_NIF_TERM callback_name = py_to_term(env, marker->callback_name); - ERL_NIF_TERM callback_args = py_to_term(env, marker->args); - Py_DECREF(py_result); - req->result = enif_make_tuple3(env, ATOM_SCHEDULE, callback_name, callback_args); - } else { - ERL_NIF_TERM term_result = py_to_term(env, py_result); - Py_DECREF(py_result); - req->result = enif_make_tuple2(env, ATOM_OK, term_result); - } - } - - tl_current_worker = NULL; - tl_current_context = NULL; - tl_callback_env = NULL; - tl_allow_suspension = false; - enif_free(code); - break; - } - - case PY_REQ_EXEC: { - tl_current_worker = worker; - tl_current_context = context; - tl_callback_env = env; - /* Note: tl_allow_suspension stays false for exec - suspension not allowed */ - - char *code = binary_to_string(&req->code_bin); - if (code == NULL) { - req->result = make_error(env, "alloc_failed"); - break; - } - - PyObject *compiled = Py_CompileString(code, "", Py_file_input); - - if (compiled == NULL) { - req->result = make_py_error(env); - } else { - /* Use globals for both to ensure imports are visible to defined functions. - * When using separate dicts, imports go to locals but function closures - * only see globals, causing "name X is not defined" errors. */ - PyObject *py_result = PyEval_EvalCode(compiled, globals, globals); - Py_DECREF(compiled); - - if (py_result == NULL) { - req->result = make_py_error(env); - } else { - Py_DECREF(py_result); - req->result = ATOM_OK; - } - } - - tl_current_worker = NULL; - tl_current_context = NULL; - tl_callback_env = NULL; - enif_free(code); - break; - } - - case PY_REQ_NEXT: { - PyObject *item = PyIter_Next(req->gen_wrapper->obj); - - if (item == NULL) { - if (PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_StopIteration)) { - PyErr_Clear(); - req->result = enif_make_tuple2(env, ATOM_ERROR, ATOM_STOP_ITERATION); - } else { - req->result = make_py_error(env); - } - } else { - req->result = enif_make_tuple2(env, ATOM_ERROR, ATOM_STOP_ITERATION); - } - } else if (PyGen_Check(item) || PyIter_Check(item)) { - py_object_t *wrapper = enif_alloc_resource(PYOBJ_RESOURCE_TYPE, sizeof(py_object_t)); - if (wrapper == NULL) { - Py_DECREF(item); - req->result = make_error(env, "alloc_failed"); - } else { - wrapper->obj = item; - wrapper->interp_id = 0; /* Main interpreter */ - atomic_fetch_add(&g_counters.pyobj_created, 1); - ERL_NIF_TERM gen_ref = enif_make_resource(env, wrapper); - enif_release_resource(wrapper); - req->result = enif_make_tuple2(env, ATOM_OK, - enif_make_tuple2(env, ATOM_GENERATOR, gen_ref)); - } - } else { - ERL_NIF_TERM term_result = py_to_term(env, item); - Py_DECREF(item); - req->result = enif_make_tuple2(env, ATOM_OK, term_result); - } - break; - } - - case PY_REQ_IMPORT: { - char *module_name = binary_to_string(&req->module_bin); - if (module_name == NULL) { - req->result = make_error(env, "alloc_failed"); - break; - } - - PyObject *module = PyImport_ImportModule(module_name); - enif_free(module_name); - - if (module == NULL) { - req->result = make_py_error(env); - } else { - py_object_t *wrapper = enif_alloc_resource(PYOBJ_RESOURCE_TYPE, sizeof(py_object_t)); - if (wrapper == NULL) { - Py_DECREF(module); - req->result = make_error(env, "alloc_failed"); - } else { - wrapper->obj = module; - wrapper->interp_id = 0; /* Main interpreter */ - atomic_fetch_add(&g_counters.pyobj_created, 1); - ERL_NIF_TERM mod_ref = enif_make_resource(env, wrapper); - enif_release_resource(wrapper); - req->result = enif_make_tuple2(env, ATOM_OK, mod_ref); - } - } - break; - } - - case PY_REQ_GETATTR: { - char *attr_name = binary_to_string(&req->attr_bin); - if (attr_name == NULL) { - req->result = make_error(env, "alloc_failed"); - break; - } - - PyObject *attr = PyObject_GetAttrString(req->obj_wrapper->obj, attr_name); - enif_free(attr_name); - - if (attr == NULL) { - req->result = make_py_error(env); - } else { - ERL_NIF_TERM term_result = py_to_term(env, attr); - Py_DECREF(attr); - req->result = enif_make_tuple2(env, ATOM_OK, term_result); - } - break; - } - - case PY_REQ_MEMORY_STATS: { - /* Import gc module */ - PyObject *gc_module = PyImport_ImportModule("gc"); - if (gc_module == NULL) { - req->result = make_error(env, "gc_import_failed"); - break; - } - - ERL_NIF_TERM result_map = enif_make_new_map(env); - - PyObject *stats = PyObject_CallMethod(gc_module, "get_stats", NULL); - if (stats != NULL && PyList_Check(stats)) { - Py_ssize_t num_gens = PyList_Size(stats); - if (num_gens > 0) { - ERL_NIF_TERM *gen_stats = enif_alloc(sizeof(ERL_NIF_TERM) * num_gens); - if (gen_stats != NULL) { - for (Py_ssize_t i = 0; i < num_gens; i++) { - PyObject *gen = PyList_GetItem(stats, i); - gen_stats[i] = py_to_term(env, gen); - } - ERL_NIF_TERM gc_stats_list = enif_make_list_from_array(env, gen_stats, num_gens); - enif_free(gen_stats); - enif_make_map_put(env, result_map, - enif_make_atom(env, "gc_stats"), gc_stats_list, &result_map); - } - /* If gen_stats alloc failed, we skip gc_stats but continue with other stats */ - } - Py_DECREF(stats); - } - - PyObject *counts = PyObject_CallMethod(gc_module, "get_count", NULL); - if (counts != NULL && PyTuple_Check(counts)) { - ERL_NIF_TERM count_term = py_to_term(env, counts); - enif_make_map_put(env, result_map, - enif_make_atom(env, "gc_count"), count_term, &result_map); - Py_DECREF(counts); - } - - PyObject *threshold = PyObject_CallMethod(gc_module, "get_threshold", NULL); - if (threshold != NULL && PyTuple_Check(threshold)) { - ERL_NIF_TERM threshold_term = py_to_term(env, threshold); - enif_make_map_put(env, result_map, - enif_make_atom(env, "gc_threshold"), threshold_term, &result_map); - Py_DECREF(threshold); - } - - Py_DECREF(gc_module); - - /* Try to get tracemalloc stats if available */ - PyObject *tracemalloc = PyImport_ImportModule("tracemalloc"); - if (tracemalloc != NULL) { - PyObject *is_tracing = PyObject_CallMethod(tracemalloc, "is_tracing", NULL); - if (is_tracing != NULL && PyObject_IsTrue(is_tracing)) { - PyObject *current_traced = PyObject_CallMethod(tracemalloc, "get_traced_memory", NULL); - if (current_traced != NULL && PyTuple_Check(current_traced)) { - ERL_NIF_TERM current = py_to_term(env, PyTuple_GetItem(current_traced, 0)); - ERL_NIF_TERM peak = py_to_term(env, PyTuple_GetItem(current_traced, 1)); - enif_make_map_put(env, result_map, - enif_make_atom(env, "traced_memory_current"), current, &result_map); - enif_make_map_put(env, result_map, - enif_make_atom(env, "traced_memory_peak"), peak, &result_map); - Py_DECREF(current_traced); - } - } - Py_XDECREF(is_tracing); - Py_DECREF(tracemalloc); - } - PyErr_Clear(); - - req->result = enif_make_tuple2(env, ATOM_OK, result_map); - break; - } - - case PY_REQ_GC: { - PyObject *gc_module = PyImport_ImportModule("gc"); - if (gc_module == NULL) { - req->result = make_error(env, "gc_import_failed"); - break; - } - - PyObject *result = PyObject_CallMethod(gc_module, "collect", "i", req->gc_generation); - Py_DECREF(gc_module); - - if (result == NULL) { - req->result = make_py_error(env); - } else { - long collected = PyLong_AsLong(result); - Py_DECREF(result); - req->result = enif_make_tuple2(env, ATOM_OK, enif_make_long(env, collected)); - } - break; - } - - case PY_REQ_SHUTDOWN: - /* Signal to exit the loop - nothing to do here */ - break; - } -} - -/* ============================================================================ - * Single executor thread implementation - * ============================================================================ */ - -/** - * Main function for the executor thread. - * Acquires GIL and processes requests until shutdown. - */ -static void *executor_thread_main(void *arg) { - (void)arg; - - /* Acquire GIL for this thread */ - PyGILState_STATE gstate = PyGILState_Ensure(); - - atomic_store(&g_executor_running, true); - - /* - * Main processing loop. - * We continue processing until we receive a PY_REQ_SHUTDOWN request. - * The shutdown flag is used to stop waiting when the queue is empty. - */ - bool should_exit = false; - while (!should_exit) { - py_request_t *req = NULL; - - /* Release GIL while waiting for work (like PyO3 allow_threads) */ - Py_BEGIN_ALLOW_THREADS - - pthread_mutex_lock(&g_executor_mutex); - while (g_executor_queue_head == NULL && !atomic_load(&g_executor_shutdown)) { - pthread_cond_wait(&g_executor_cond, &g_executor_mutex); - } - - /* Dequeue request if available */ - if (g_executor_queue_head != NULL) { - req = g_executor_queue_head; - g_executor_queue_head = req->next; - if (g_executor_queue_head == NULL) { - g_executor_queue_tail = NULL; - } - req->next = NULL; - } else if (atomic_load(&g_executor_shutdown)) { - /* Queue is empty and shutdown requested - exit */ - should_exit = true; - } - pthread_mutex_unlock(&g_executor_mutex); - - Py_END_ALLOW_THREADS - - if (req != NULL) { - if (req->type == PY_REQ_SHUTDOWN) { - /* Signal completion and exit */ - pthread_mutex_lock(&req->mutex); - req->completed = true; - pthread_cond_signal(&req->cond); - pthread_mutex_unlock(&req->mutex); - should_exit = true; - } else { - /* Process the request with GIL held */ - process_request(req); - - /* Track completed requests */ - atomic_fetch_add(&g_counters.complete_count, 1); - - /* Signal completion */ - pthread_mutex_lock(&req->mutex); - req->completed = true; - pthread_cond_signal(&req->cond); - pthread_mutex_unlock(&req->mutex); - } - } - } - - atomic_store(&g_executor_running, false); - PyGILState_Release(gstate); - - return NULL; -} - -/** - * Enqueue a request to the appropriate executor based on execution mode. - * Routes to multi-executor pool, single executor, or executes directly. - * - * @return 0 on success, -1 if shutting down (request rejected) - */ -static int executor_enqueue(py_request_t *req) { - /* Reject work if runtime is shutting down (except shutdown requests) */ - if (runtime_is_shutting_down() && req->type != PY_REQ_SHUTDOWN) { - atomic_fetch_add(&g_counters.rejected_count, 1); - return -1; - } - - /* Track enqueued requests */ - atomic_fetch_add(&g_counters.enqueue_count, 1); - -#ifdef HAVE_FREE_THREADED - if (g_execution_mode == PY_MODE_FREE_THREADED) { - /* Execute directly in free-threaded mode - no executor needed */ - PyGILState_STATE gstate = PyGILState_Ensure(); - process_request(req); - PyGILState_Release(gstate); - /* Signal completion immediately */ - pthread_mutex_lock(&req->mutex); - req->completed = true; - pthread_cond_signal(&req->cond); - pthread_mutex_unlock(&req->mutex); - return 0; - } -#endif - - /* Single coordinator executor queue */ - pthread_mutex_lock(&g_executor_mutex); - req->next = NULL; - if (g_executor_queue_tail == NULL) { - g_executor_queue_head = req; - g_executor_queue_tail = req; - } else { - g_executor_queue_tail->next = req; - g_executor_queue_tail = req; - } - pthread_cond_signal(&g_executor_cond); - pthread_mutex_unlock(&g_executor_mutex); - return 0; -} - -/** - * Wait for a request to complete. - */ -static void executor_wait(py_request_t *req) { - pthread_mutex_lock(&req->mutex); - while (!req->completed) { - pthread_cond_wait(&req->cond, &req->mutex); - } - pthread_mutex_unlock(&req->mutex); -} - -/** - * Start the executor thread. - * Called during Python initialization. - */ -static int executor_start(void) { - atomic_store(&g_executor_shutdown, false); - g_executor_queue_head = NULL; - g_executor_queue_tail = NULL; - - if (pthread_create(&g_executor_thread, NULL, executor_thread_main, NULL) != 0) { - return -1; - } - - /* Wait for executor to be ready */ - int max_wait = 100; /* 1 second max */ - while (!atomic_load(&g_executor_running) && max_wait-- > 0) { - usleep(10000); /* 10ms */ - } - - return atomic_load(&g_executor_running) ? 0 : -1; -} - -/** - * Stop the executor thread. - * Called during Python finalization. - */ -static void executor_stop(void) { - if (!atomic_load(&g_executor_running)) { - return; - } - - /* Send shutdown request */ - py_request_t shutdown_req; - request_init(&shutdown_req); - shutdown_req.type = PY_REQ_SHUTDOWN; - - atomic_store(&g_executor_shutdown, true); - executor_enqueue(&shutdown_req); - executor_wait(&shutdown_req); - request_cleanup(&shutdown_req); - - /* Wait for thread to finish */ - pthread_join(g_executor_thread, NULL); -} - -/* - * Note: Free-threaded execution (Python 3.13+ nogil) is handled inline - * in executor_enqueue() using PyGILState_Ensure/Release which are no-ops - * in free-threaded builds but still work correctly. - */ diff --git a/c_src/py_nif.c b/c_src/py_nif.c index 83f83df..14c3b62 100644 --- a/c_src/py_nif.c +++ b/c_src/py_nif.c @@ -53,10 +53,8 @@ * Global state definitions * ============================================================================ */ -ErlNifResourceType *WORKER_RESOURCE_TYPE = NULL; ErlNifResourceType *PYOBJ_RESOURCE_TYPE = NULL; /* ASYNC_WORKER_RESOURCE_TYPE removed - async workers replaced by event loop model */ -ErlNifResourceType *SUSPENDED_STATE_RESOURCE_TYPE = NULL; /* Process-per-context resource type (no mutex) */ ErlNifResourceType *PY_CONTEXT_RESOURCE_TYPE = NULL; @@ -147,13 +145,6 @@ PyThreadState *g_main_thread_state = NULL; py_execution_mode_t g_execution_mode = PY_MODE_GIL; /* Single executor state */ -pthread_t g_executor_thread; -pthread_mutex_t g_executor_mutex = PTHREAD_MUTEX_INITIALIZER; -pthread_cond_t g_executor_cond = PTHREAD_COND_INITIALIZER; -py_request_t *g_executor_queue_head = NULL; -py_request_t *g_executor_queue_tail = NULL; -_Atomic bool g_executor_running = false; -_Atomic bool g_executor_shutdown = false; /* Global counter for callback IDs */ _Atomic uint64_t g_callback_id_counter = 1; @@ -168,10 +159,8 @@ PyObject *ProcessErrorException = NULL; PyObject *g_numpy_ndarray_type = NULL; /* Thread-local callback context */ -__thread py_worker_t *tl_current_worker = NULL; __thread py_context_t *tl_current_context = NULL; __thread ErlNifEnv *tl_callback_env = NULL; -__thread suspended_state_t *tl_current_suspended = NULL; __thread suspended_context_state_t *tl_current_context_suspended = NULL; __thread bool tl_allow_suspension = false; @@ -240,8 +229,6 @@ ERL_NIF_TERM ATOM_SPAN_EVENT; * ============================================================================ */ /* From py_callback.c - needed by py_exec.c */ -static PyObject *build_pending_callback_exc_args(void); -static ERL_NIF_TERM build_suspended_result(ErlNifEnv *env, suspended_state_t *suspended); /* Schedule marker type and helper - from py_callback.c, needed by py_exec.c */ typedef struct { @@ -276,8 +263,6 @@ static int is_inline_schedule_marker(PyObject *obj); #include "py_callback.c" #include "py_thread_worker.c" #include "py_event_loop.c" -#include "py_worker_pool.h" -#include "py_worker_pool.c" #include "py_subinterp_thread.c" #include "py_reactor_buffer.c" #include "py_channel.c" @@ -287,23 +272,6 @@ static int is_inline_schedule_marker(PyObject *obj); * Resource callbacks * ============================================================================ */ -static void worker_destructor(ErlNifEnv *env, void *obj) { - (void)env; - py_worker_t *worker = (py_worker_t *)obj; - - /* Close callback pipes */ - close_pipe_pair(worker->callback_pipe); - - /* Only clean up Python state if Python is still initialized */ - if (worker->thread_state != NULL && runtime_is_running()) { - PyEval_RestoreThread(worker->thread_state); - Py_XDECREF(worker->globals); - Py_XDECREF(worker->locals); - PyThreadState_Clear(worker->thread_state); - PyThreadState_DeleteCurrent(); - } -} - static void pyobj_destructor(ErlNifEnv *env, void *obj) { (void)env; py_object_t *wrapper = (py_object_t *)obj; @@ -524,53 +492,6 @@ static void suspended_context_state_destructor(ErlNifEnv *env, void *obj) { atomic_fetch_add(&g_counters.suspended_destroyed, 1); } -static void suspended_state_destructor(ErlNifEnv *env, void *obj) { - (void)env; - suspended_state_t *state = (suspended_state_t *)obj; - - /* Release the worker resource kept alive in create_suspended_state_ex. */ - if (state->worker != NULL) { - enif_release_resource(state->worker); - state->worker = NULL; - } - - /* Clean up Python objects if Python is still initialized. - * suspended_state_t is used with the worker-based API which runs in - * the main interpreter, so we always use PyGILState_Ensure. */ - if (runtime_is_running() && state->callback_args != NULL) { - if (PyGILState_GetThisThreadState() != NULL || PyGILState_Check()) { - Py_XDECREF(state->callback_args); - state->callback_args = NULL; - } else { - PyGILState_STATE gstate = PyGILState_Ensure(); - Py_XDECREF(state->callback_args); - state->callback_args = NULL; - PyGILState_Release(gstate); - } - } - - /* Free allocated memory */ - if (state->callback_func_name != NULL) { - enif_free(state->callback_func_name); - state->callback_func_name = NULL; - } - if (state->result_data != NULL) { - enif_free(state->result_data); - state->result_data = NULL; - } - - /* Free original context environment */ - if (state->orig_env != NULL) { - enif_free_env(state->orig_env); - state->orig_env = NULL; - } - - /* Destroy synchronization primitives */ - pthread_mutex_destroy(&state->mutex); - pthread_cond_destroy(&state->cond); - - atomic_fetch_add(&g_counters.suspended_destroyed, 1); -} /* ============================================================================ * Inline Continuation Support @@ -1142,22 +1063,6 @@ static ERL_NIF_TERM nif_py_init(ErlNifEnv *env, int argc, const ERL_NIF_TERM arg /* Save main thread state and release GIL for other threads */ g_main_thread_state = PyEval_SaveThread(); - /* Start single executor for coordinator operations. - * Context operations use per-context worker threads (see worker_context_init). - * The single executor handles legacy worker API and coordinator tasks. */ - int executor_result = 0; - if (g_execution_mode != PY_MODE_FREE_THREADED) { - executor_result = executor_start(); - } - - if (executor_result < 0) { - PyEval_RestoreThread(g_main_thread_state); - g_main_thread_state = NULL; - Py_Finalize(); - atomic_store(&g_runtime_state, PY_STATE_STOPPED); - return make_error(env, "executor_start_failed"); - } - /* Initialize thread worker system for ThreadPoolExecutor support */ if (thread_worker_init() < 0) { /* Non-fatal - thread worker support just won't be available */ @@ -1195,11 +1100,6 @@ static ERL_NIF_TERM nif_finalize(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar * 3. Then clean up caches with GIL (no active work at this point) */ - /* Step 1: Stop executor - it will finish in-flight requests and exit */ - if (g_execution_mode != PY_MODE_FREE_THREADED) { - executor_stop(); - } - /* Step 2: Clean up thread worker system */ thread_worker_cleanup(); @@ -1246,280 +1146,6 @@ static ERL_NIF_TERM nif_finalize(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar return ATOM_OK; } -/* ============================================================================ - * Worker management - * ============================================================================ */ - -static ERL_NIF_TERM nif_worker_new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - (void)argc; - (void)argv; - - if (!runtime_is_running()) { - return make_error(env, "python_not_running"); - } - - py_worker_t *worker = enif_alloc_resource(WORKER_RESOURCE_TYPE, sizeof(py_worker_t)); - if (worker == NULL) { - return make_error(env, "alloc_failed"); - } - - /* Acquire GIL to create thread state */ - PyGILState_STATE gstate = PyGILState_Ensure(); - - /* Create a new thread state for this worker */ - PyInterpreterState *interp = PyInterpreterState_Get(); - worker->thread_state = PyThreadState_New(interp); - - /* Create global/local namespaces */ - worker->globals = PyDict_New(); - worker->locals = PyDict_New(); - - /* Import __builtins__ into globals */ - PyObject *builtins = PyEval_GetBuiltins(); - PyDict_SetItemString(worker->globals, "__builtins__", builtins); - - /* Import erlang module into worker's namespace for callbacks */ - PyObject *erlang_module = PyImport_ImportModule("erlang"); - if (erlang_module != NULL) { - PyDict_SetItemString(worker->globals, "erlang", erlang_module); - Py_DECREF(erlang_module); - } - - worker->owns_gil = false; - - /* Initialize callback state */ - worker->callback_pipe[0] = -1; - worker->callback_pipe[1] = -1; - worker->has_callback_handler = false; - worker->callback_env = NULL; - - PyGILState_Release(gstate); - - ERL_NIF_TERM result = enif_make_resource(env, worker); - enif_release_resource(worker); - - return enif_make_tuple2(env, ATOM_OK, result); -} - -static ERL_NIF_TERM nif_worker_destroy(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - (void)argc; - py_worker_t *worker; - - if (!enif_get_resource(env, argv[0], WORKER_RESOURCE_TYPE, (void **)&worker)) { - return make_error(env, "invalid_worker"); - } - - /* Resource destructor will handle cleanup */ - return ATOM_OK; -} - -/* ============================================================================ - * Python execution (dirty NIFs) - * ============================================================================ */ - -static ERL_NIF_TERM nif_worker_call(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - py_worker_t *worker; - - if (!enif_get_resource(env, argv[0], WORKER_RESOURCE_TYPE, (void **)&worker)) { - return make_error(env, "invalid_worker"); - } - - /* Build request and route to executor */ - py_request_t req; - request_init(&req); - req.type = PY_REQ_CALL; - req.worker = worker; - req.env = env; - - if (!enif_inspect_binary(env, argv[1], &req.module_bin)) { - request_cleanup(&req); - return make_error(env, "invalid_module"); - } - if (!enif_inspect_binary(env, argv[2], &req.func_bin)) { - request_cleanup(&req); - return make_error(env, "invalid_func"); - } - - req.args_term = argv[3]; - req.kwargs_term = (argc > 4) ? argv[4] : 0; - req.timeout_ms = 0; - - if (argc > 5) { - enif_get_ulong(env, argv[5], &req.timeout_ms); - } - - /* Submit to executor and wait */ - if (executor_enqueue(&req) != 0) { - request_cleanup(&req); - return make_error(env, "runtime_shutting_down"); - } - executor_wait(&req); - - ERL_NIF_TERM result = req.result; - request_cleanup(&req); - return result; -} - -static ERL_NIF_TERM nif_worker_eval(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - py_worker_t *worker; - - if (!enif_get_resource(env, argv[0], WORKER_RESOURCE_TYPE, (void **)&worker)) { - return make_error(env, "invalid_worker"); - } - - py_request_t req; - request_init(&req); - req.type = PY_REQ_EVAL; - req.worker = worker; - req.env = env; - - if (!enif_inspect_binary(env, argv[1], &req.code_bin)) { - request_cleanup(&req); - return make_error(env, "invalid_code"); - } - - req.locals_term = (argc > 2) ? argv[2] : 0; - req.timeout_ms = 0; - if (argc > 3) { - enif_get_ulong(env, argv[3], &req.timeout_ms); - } - - if (executor_enqueue(&req) != 0) { - request_cleanup(&req); - return make_error(env, "runtime_shutting_down"); - } - executor_wait(&req); - - ERL_NIF_TERM result = req.result; - request_cleanup(&req); - return result; -} - -static ERL_NIF_TERM nif_worker_exec(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - (void)argc; - py_worker_t *worker; - - if (!enif_get_resource(env, argv[0], WORKER_RESOURCE_TYPE, (void **)&worker)) { - return make_error(env, "invalid_worker"); - } - - py_request_t req; - request_init(&req); - req.type = PY_REQ_EXEC; - req.worker = worker; - req.env = env; - - if (!enif_inspect_binary(env, argv[1], &req.code_bin)) { - request_cleanup(&req); - return make_error(env, "invalid_code"); - } - - if (executor_enqueue(&req) != 0) { - request_cleanup(&req); - return make_error(env, "runtime_shutting_down"); - } - executor_wait(&req); - - ERL_NIF_TERM result = req.result; - request_cleanup(&req); - return result; -} - -static ERL_NIF_TERM nif_worker_next(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - (void)argc; - py_worker_t *worker; - py_object_t *gen_wrapper; - - if (!enif_get_resource(env, argv[0], WORKER_RESOURCE_TYPE, (void **)&worker)) { - return make_error(env, "invalid_worker"); - } - if (!enif_get_resource(env, argv[1], PYOBJ_RESOURCE_TYPE, (void **)&gen_wrapper)) { - return make_error(env, "invalid_generator"); - } - - py_request_t req; - request_init(&req); - req.type = PY_REQ_NEXT; - req.worker = worker; - req.env = env; - req.gen_wrapper = gen_wrapper; - - if (executor_enqueue(&req) != 0) { - request_cleanup(&req); - return make_error(env, "runtime_shutting_down"); - } - executor_wait(&req); - - ERL_NIF_TERM result = req.result; - request_cleanup(&req); - return result; -} - -static ERL_NIF_TERM nif_import_module(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - (void)argc; - py_worker_t *worker; - - if (!enif_get_resource(env, argv[0], WORKER_RESOURCE_TYPE, (void **)&worker)) { - return make_error(env, "invalid_worker"); - } - - py_request_t req; - request_init(&req); - req.type = PY_REQ_IMPORT; - req.worker = worker; - req.env = env; - - if (!enif_inspect_binary(env, argv[1], &req.module_bin)) { - request_cleanup(&req); - return make_error(env, "invalid_module"); - } - - if (executor_enqueue(&req) != 0) { - request_cleanup(&req); - return make_error(env, "runtime_shutting_down"); - } - executor_wait(&req); - - ERL_NIF_TERM result = req.result; - request_cleanup(&req); - return result; -} - -static ERL_NIF_TERM nif_get_attr(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - (void)argc; - py_worker_t *worker; - py_object_t *obj_wrapper; - - if (!enif_get_resource(env, argv[0], WORKER_RESOURCE_TYPE, (void **)&worker)) { - return make_error(env, "invalid_worker"); - } - if (!enif_get_resource(env, argv[1], PYOBJ_RESOURCE_TYPE, (void **)&obj_wrapper)) { - return make_error(env, "invalid_object"); - } - - py_request_t req; - request_init(&req); - req.type = PY_REQ_GETATTR; - req.worker = worker; - req.env = env; - req.obj_wrapper = obj_wrapper; - - if (!enif_inspect_binary(env, argv[2], &req.attr_bin)) { - request_cleanup(&req); - return make_error(env, "invalid_attr"); - } - - if (executor_enqueue(&req) != 0) { - request_cleanup(&req); - return make_error(env, "runtime_shutting_down"); - } - executor_wait(&req); - - ERL_NIF_TERM result = req.result; - request_cleanup(&req); - return result; -} - /* ============================================================================ * Info NIFs * ============================================================================ */ @@ -1545,20 +1171,65 @@ static ERL_NIF_TERM nif_memory_stats(ErlNifEnv *env, int argc, const ERL_NIF_TER return make_error(env, "python_not_running"); } - py_request_t req; - request_init(&req); - req.type = PY_REQ_MEMORY_STATS; - req.env = env; - - if (executor_enqueue(&req) != 0) { - request_cleanup(&req); - return make_error(env, "runtime_shutting_down"); + PyGILState_STATE gstate = PyGILState_Ensure(); + PyObject *gc_module = PyImport_ImportModule("gc"); + if (gc_module == NULL) { + PyErr_Clear(); + PyGILState_Release(gstate); + return make_error(env, "gc_import_failed"); + } + ERL_NIF_TERM result_map = enif_make_new_map(env); + PyObject *stats = PyObject_CallMethod(gc_module, "get_stats", NULL); + if (stats != NULL && PyList_Check(stats)) { + Py_ssize_t num_gens = PyList_Size(stats); + if (num_gens > 0) { + ERL_NIF_TERM *gen_stats = enif_alloc(sizeof(ERL_NIF_TERM) * num_gens); + if (gen_stats != NULL) { + for (Py_ssize_t i = 0; i < num_gens; i++) { + gen_stats[i] = py_to_term(env, PyList_GetItem(stats, i)); + } + ERL_NIF_TERM gc_stats_list = enif_make_list_from_array(env, gen_stats, num_gens); + enif_free(gen_stats); + enif_make_map_put(env, result_map, + enif_make_atom(env, "gc_stats"), gc_stats_list, &result_map); + } + } + } + Py_XDECREF(stats); + PyObject *counts = PyObject_CallMethod(gc_module, "get_count", NULL); + if (counts != NULL && PyTuple_Check(counts)) { + enif_make_map_put(env, result_map, + enif_make_atom(env, "gc_count"), py_to_term(env, counts), &result_map); + } + Py_XDECREF(counts); + PyObject *threshold = PyObject_CallMethod(gc_module, "get_threshold", NULL); + if (threshold != NULL && PyTuple_Check(threshold)) { + enif_make_map_put(env, result_map, + enif_make_atom(env, "gc_threshold"), py_to_term(env, threshold), &result_map); } - executor_wait(&req); + Py_XDECREF(threshold); + Py_DECREF(gc_module); - ERL_NIF_TERM result = req.result; - request_cleanup(&req); - return result; + /* tracemalloc stats when tracing is on */ + PyObject *tracemalloc = PyImport_ImportModule("tracemalloc"); + if (tracemalloc != NULL) { + PyObject *is_tracing = PyObject_CallMethod(tracemalloc, "is_tracing", NULL); + if (is_tracing != NULL && PyObject_IsTrue(is_tracing)) { + PyObject *traced = PyObject_CallMethod(tracemalloc, "get_traced_memory", NULL); + if (traced != NULL && PyTuple_Check(traced)) { + enif_make_map_put(env, result_map, enif_make_atom(env, "traced_memory_current"), + py_to_term(env, PyTuple_GetItem(traced, 0)), &result_map); + enif_make_map_put(env, result_map, enif_make_atom(env, "traced_memory_peak"), + py_to_term(env, PyTuple_GetItem(traced, 1)), &result_map); + } + Py_XDECREF(traced); + } + Py_XDECREF(is_tracing); + Py_DECREF(tracemalloc); + } + PyErr_Clear(); + PyGILState_Release(gstate); + return enif_make_tuple2(env, ATOM_OK, result_map); } /** @@ -1620,25 +1291,30 @@ static ERL_NIF_TERM nif_gc(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) if (!runtime_is_running()) { return make_error(env, "python_not_running"); } - - py_request_t req; - request_init(&req); - req.type = PY_REQ_GC; - req.env = env; - req.gc_generation = 2; /* Full collection by default */ + int generation = 2; /* Full collection by default */ if (argc > 0) { - enif_get_int(env, argv[0], &req.gc_generation); + enif_get_int(env, argv[0], &generation); } - if (executor_enqueue(&req) != 0) { - request_cleanup(&req); - return make_error(env, "runtime_shutting_down"); + PyGILState_STATE gstate = PyGILState_Ensure(); + PyObject *gc_module = PyImport_ImportModule("gc"); + if (gc_module == NULL) { + PyErr_Clear(); + PyGILState_Release(gstate); + return make_error(env, "gc_import_failed"); } - executor_wait(&req); - - ERL_NIF_TERM result = req.result; - request_cleanup(&req); - return result; + PyObject *result = PyObject_CallMethod(gc_module, "collect", "i", generation); + Py_DECREF(gc_module); + ERL_NIF_TERM term; + if (result == NULL) { + term = make_py_error(env); + } else { + long collected = PyLong_AsLong(result); + Py_DECREF(result); + term = enif_make_tuple2(env, ATOM_OK, enif_make_long(env, collected)); + } + PyGILState_Release(gstate); + return term; } static ERL_NIF_TERM nif_tracemalloc_start(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { @@ -1719,34 +1395,6 @@ static ERL_NIF_TERM nif_execution_mode(ErlNifEnv *env, int argc, const ERL_NIF_T * Callback support NIFs * ============================================================================ */ -static ERL_NIF_TERM nif_set_callback_handler(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - (void)argc; - py_worker_t *worker; - - if (!enif_get_resource(env, argv[0], WORKER_RESOURCE_TYPE, (void **)&worker)) { - return make_error(env, "invalid_worker"); - } - - if (!enif_get_local_pid(env, argv[1], &worker->callback_handler)) { - return make_error(env, "invalid_pid"); - } - - /* Create pipe for callback responses */ - if (pipe(worker->callback_pipe) < 0) { - return make_error(env, "pipe_failed"); - } - /* Non-blocking write end so write_all_with_deadline can bound the write. */ - { - int wfl = fcntl(worker->callback_pipe[1], F_GETFL, 0); - if (wfl >= 0) (void)fcntl(worker->callback_pipe[1], F_SETFL, wfl | O_NONBLOCK); - } - - worker->has_callback_handler = true; - - /* Return the write end of the pipe as a file descriptor for Erlang to use */ - return enif_make_tuple2(env, ATOM_OK, - enif_make_int(env, worker->callback_pipe[1])); -} /* Bound for callback-response pipe writes: a stalled reader must not block a * dirty scheduler forever (the pipe write ends are set non-blocking). */ @@ -1756,71 +1404,6 @@ static ERL_NIF_TERM nif_set_callback_handler(ErlNifEnv *env, int argc, const ERL * block the dispatching dirty scheduler forever. */ #define OWNGIL_IO_TIMEOUT_MS 30000 -static ERL_NIF_TERM nif_send_callback_response(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - (void)argc; - int fd; - ErlNifBinary response; - - if (!enif_get_int(env, argv[0], &fd)) { - return make_error(env, "invalid_fd"); - } - - if (!enif_inspect_binary(env, argv[1], &response)) { - return make_error(env, "invalid_response"); - } - - /* Write length then data with a timed, non-blocking writer (the pipe write - * end is O_NONBLOCK) so a stalled reader or a large payload can't block a - * dirty scheduler forever or desync the length-framed protocol on EINTR. */ - uint32_t len = (uint32_t)response.size; - if (write_all_with_deadline(fd, &len, sizeof(len), - CALLBACK_RESPONSE_IO_TIMEOUT_MS) != WRITE_OK) { - return make_error(env, "write_length_failed"); - } - if (write_all_with_deadline(fd, response.data, response.size, - CALLBACK_RESPONSE_IO_TIMEOUT_MS) != WRITE_OK) { - return make_error(env, "write_data_failed"); - } - - return ATOM_OK; -} - -/* ============================================================================ - * Async worker NIFs (deprecated - replaced by event loop model) - * - * These NIFs are deprecated and return errors. Use py_event_loop_pool and - * py_event_loop:run_async/2 instead. - * ============================================================================ */ - -static ERL_NIF_TERM nif_async_worker_new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - (void)argc; - (void)argv; - return make_error(env, "async_workers_deprecated_use_event_loop"); -} - -static ERL_NIF_TERM nif_async_worker_destroy(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - (void)argc; - (void)argv; - return ATOM_OK; -} - -static ERL_NIF_TERM nif_async_call(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - (void)argc; - (void)argv; - return make_error(env, "async_workers_deprecated_use_event_loop"); -} - -static ERL_NIF_TERM nif_async_gather(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - (void)argc; - (void)argv; - return make_error(env, "async_workers_deprecated_use_event_loop"); -} - -static ERL_NIF_TERM nif_async_stream(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - (void)argc; - (void)argv; - return make_error(env, "async_workers_deprecated_use_event_loop"); -} /* ============================================================================ * Sub-interpreter support (Python 3.12+) @@ -5097,35 +4680,7 @@ static ERL_NIF_TERM nif_context_destroy(ErlNifEnv *env, int argc, const ERL_NIF_ return ATOM_OK; } - /* Legacy mode (should not reach here with new architecture) */ - if (runtime_is_running()) { - PyGILState_STATE gstate = PyGILState_Ensure(); - Py_XDECREF(ctx->module_cache); - ctx->module_cache = NULL; - Py_XDECREF(ctx->globals); - ctx->globals = NULL; - Py_XDECREF(ctx->locals); - ctx->locals = NULL; -#ifndef HAVE_SUBINTERPRETERS - if (ctx->thread_state != NULL) { - PyThreadState_Clear(ctx->thread_state); - PyThreadState_Delete(ctx->thread_state); - ctx->thread_state = NULL; - } -#endif - PyGILState_Release(gstate); - } - - /* Close callback pipes */ - if (ctx->callback_pipe[0] >= 0) { - close(ctx->callback_pipe[0]); - ctx->callback_pipe[0] = -1; - } - if (ctx->callback_pipe[1] >= 0) { - close(ctx->callback_pipe[1]); - ctx->callback_pipe[1] = -1; - } - + /* Every context created by nif_context_create has a thread */ atomic_fetch_add(&g_counters.ctx_destroyed, 1); return ATOM_OK; } @@ -5209,179 +4764,8 @@ static ERL_NIF_TERM nif_context_call(ErlNifEnv *env, int argc, const ERL_NIF_TER return dispatch_to_worker_thread(env, ctx, CTX_REQ_CALL, request); } - /* Legacy mode: direct execution with py_context_acquire. - * For subinterpreters, py_context_acquire handles PyThreadState_Swap - * to switch to the pool slot's interpreter. */ - ErlNifBinary module_bin, func_bin; - if (!enif_inspect_binary(env, argv[1], &module_bin)) { - return make_error(env, "invalid_module"); - } - if (!enif_inspect_binary(env, argv[2], &func_bin)) { - return make_error(env, "invalid_func"); - } - - char *module_name = binary_to_string(&module_bin); - char *func_name = binary_to_string(&func_bin); - if (module_name == NULL || func_name == NULL) { - enif_free(module_name); - enif_free(func_name); - return make_error(env, "alloc_failed"); - } - - ERL_NIF_TERM result; - - /* Acquire thread state using centralized guard (worker mode only) */ - py_context_guard_t guard = py_context_acquire(ctx); - if (!guard.acquired) { - enif_free(module_name); - enif_free(func_name); - return make_error(env, "acquire_failed"); - } - - /* Set thread-local context for callback support */ - py_context_t *prev_context = tl_current_context; - tl_current_context = ctx; - - /* Enable suspension for callback support */ - bool prev_allow_suspension = tl_allow_suspension; - tl_allow_suspension = true; - - PyObject *module = NULL; - PyObject *func = NULL; - - /* Special handling for __main__ module - check ctx->globals first */ - if (strcmp(module_name, "__main__") == 0) { - func = PyDict_GetItemString(ctx->globals, func_name); /* Borrowed ref */ - if (func != NULL) { - Py_INCREF(func); - } - } - - if (func == NULL) { - /* Get or import module */ - module = context_get_module(ctx, module_name); - if (module == NULL) { - result = make_py_error(env); - goto cleanup; - } - - /* Get function */ - func = PyObject_GetAttrString(module, func_name); - if (func == NULL) { - result = make_py_error(env); - goto cleanup; - } - } - - /* Convert args */ - unsigned int args_len; - if (!enif_get_list_length(env, argv[3], &args_len)) { - Py_DECREF(func); - result = make_error(env, "invalid_args"); - goto cleanup; - } - - PyObject *args = PyTuple_New(args_len); - if (args == NULL) { - Py_DECREF(func); - result = make_error(env, "alloc_failed"); - goto cleanup; - } - ERL_NIF_TERM head, tail = argv[3]; - for (unsigned int i = 0; i < args_len; i++) { - enif_get_list_cell(env, tail, &head, &tail); - PyObject *arg = term_to_py(env, head); - if (arg == NULL) { - Py_DECREF(args); - Py_DECREF(func); - result = make_error(env, "arg_conversion_failed"); - goto cleanup; - } - PyTuple_SET_ITEM(args, i, arg); - } - - /* Convert kwargs */ - PyObject *kwargs = NULL; - if (argc > 4 && enif_is_map(env, argv[4])) { - kwargs = term_to_py(env, argv[4]); - } - - /* Call the function */ - PyObject *py_result = PyObject_Call(func, args, kwargs); - Py_DECREF(func); - Py_DECREF(args); - Py_XDECREF(kwargs); - - if (py_result == NULL) { - /* Check for pending callback (flag-based detection) */ - if (tl_pending_callback) { - PyErr_Clear(); /* Clear whatever exception is set */ - - /* Create suspended context state */ - suspended_context_state_t *suspended = create_suspended_context_state_for_call( - env, ctx, &module_bin, &func_bin, argv[3], - argc > 4 ? argv[4] : enif_make_new_map(env)); - - if (suspended == NULL) { - tl_pending_callback = false; - Py_CLEAR(tl_pending_args); - result = make_error(env, "create_suspended_state_failed"); - } else { - result = build_suspended_context_result(env, suspended); - } - } else { - result = make_py_error(env); - } - } else if (is_inline_schedule_marker(py_result)) { - /* Inline schedule marker: chain via enif_schedule_nif without Erlang messaging */ - inline_continuation_t *cont = create_inline_continuation(ctx, NULL, py_result, 0); - Py_DECREF(py_result); - - if (cont == NULL) { - result = make_error(env, "create_continuation_failed"); - } else { - ERL_NIF_TERM cont_ref = enif_make_resource(env, cont); - enif_release_resource(cont); - - /* Restore thread-local state before scheduling */ - tl_allow_suspension = prev_allow_suspension; - tl_current_context = prev_context; - clear_pending_callback_tls(); - enif_free(module_name); - enif_free(func_name); - py_context_release(&guard); - - return enif_schedule_nif(env, "inline_continuation", - ERL_NIF_DIRTY_JOB_IO_BOUND, nif_inline_continuation, 1, &cont_ref); - } - } else if (is_schedule_marker(py_result)) { - /* Schedule marker: release dirty scheduler, continue via callback */ - ScheduleMarkerObject *marker = (ScheduleMarkerObject *)py_result; - ERL_NIF_TERM callback_name = py_to_term(env, marker->callback_name); - ERL_NIF_TERM callback_args = py_to_term(env, marker->args); - Py_DECREF(py_result); - result = enif_make_tuple3(env, ATOM_SCHEDULE, callback_name, callback_args); - } else { - ERL_NIF_TERM term_result = py_to_term(env, py_result); - Py_DECREF(py_result); - result = enif_make_tuple2(env, ATOM_OK, term_result); - } - -cleanup: - /* Restore thread-local state */ - tl_allow_suspension = prev_allow_suspension; - tl_current_context = prev_context; - - /* Clear pending callback TLS before releasing context */ - clear_pending_callback_tls(); - - enif_free(module_name); - enif_free(func_name); - - /* Release thread state using centralized guard */ - py_context_release(&guard); - - return result; + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); } /** @@ -5691,115 +5075,8 @@ static ERL_NIF_TERM nif_context_eval(ErlNifEnv *env, int argc, const ERL_NIF_TER return dispatch_to_worker_thread(env, ctx, CTX_REQ_EVAL, request); } - /* Legacy mode: direct execution with py_context_acquire. - * For subinterpreters, py_context_acquire handles PyThreadState_Swap - * to switch to the pool slot's interpreter. */ - ErlNifBinary code_bin; - if (!enif_inspect_binary(env, argv[1], &code_bin)) { - return make_error(env, "invalid_code"); - } - - char *code = binary_to_string(&code_bin); - if (code == NULL) { - return make_error(env, "alloc_failed"); - } - - ERL_NIF_TERM result; - - /* Acquire thread state using centralized guard (worker mode only) */ - py_context_guard_t guard = py_context_acquire(ctx); - if (!guard.acquired) { - enif_free(code); - return make_error(env, "acquire_failed"); - } - - /* Set thread-local context for callback support */ - py_context_t *prev_context = tl_current_context; - tl_current_context = ctx; - - /* Enable suspension for callback support */ - bool prev_allow_suspension = tl_allow_suspension; - tl_allow_suspension = true; - - /* Update locals if provided */ - ERL_NIF_TERM locals_term = argc > 2 ? argv[2] : enif_make_new_map(env); - if (argc > 2 && enif_is_map(env, argv[2])) { - PyObject *new_locals = term_to_py(env, argv[2]); - if (new_locals != NULL && PyDict_Check(new_locals)) { - PyDict_Update(ctx->locals, new_locals); - Py_DECREF(new_locals); - } - } - - /* Compile and evaluate */ - PyObject *py_result = PyRun_String(code, Py_eval_input, ctx->globals, ctx->locals); - - if (py_result == NULL) { - /* Check for pending callback (flag-based detection) */ - if (tl_pending_callback) { - PyErr_Clear(); /* Clear whatever exception is set */ - - /* Create suspended context state */ - suspended_context_state_t *suspended = create_suspended_context_state_for_eval( - env, ctx, &code_bin, locals_term); - - if (suspended == NULL) { - tl_pending_callback = false; - Py_CLEAR(tl_pending_args); - result = make_error(env, "create_suspended_state_failed"); - } else { - result = build_suspended_context_result(env, suspended); - } - } else { - result = make_py_error(env); - } - } else if (is_inline_schedule_marker(py_result)) { - /* Inline schedule marker: chain via enif_schedule_nif without Erlang messaging */ - inline_continuation_t *cont = create_inline_continuation(ctx, NULL, py_result, 0); - Py_DECREF(py_result); - - if (cont == NULL) { - result = make_error(env, "create_continuation_failed"); - } else { - ERL_NIF_TERM cont_ref = enif_make_resource(env, cont); - enif_release_resource(cont); - - /* Restore thread-local state before scheduling */ - tl_allow_suspension = prev_allow_suspension; - tl_current_context = prev_context; - clear_pending_callback_tls(); - enif_free(code); - py_context_release(&guard); - - return enif_schedule_nif(env, "inline_continuation", - ERL_NIF_DIRTY_JOB_IO_BOUND, nif_inline_continuation, 1, &cont_ref); - } - } else if (is_schedule_marker(py_result)) { - /* Schedule marker: release dirty scheduler, continue via callback */ - ScheduleMarkerObject *marker = (ScheduleMarkerObject *)py_result; - ERL_NIF_TERM callback_name = py_to_term(env, marker->callback_name); - ERL_NIF_TERM callback_args = py_to_term(env, marker->args); - Py_DECREF(py_result); - result = enif_make_tuple3(env, ATOM_SCHEDULE, callback_name, callback_args); - } else { - ERL_NIF_TERM term_result = py_to_term(env, py_result); - Py_DECREF(py_result); - result = enif_make_tuple2(env, ATOM_OK, term_result); - } - - /* Restore thread-local state */ - tl_allow_suspension = prev_allow_suspension; - tl_current_context = prev_context; - - /* Clear pending callback TLS before releasing context */ - clear_pending_callback_tls(); - - enif_free(code); - - /* Release thread state using centralized guard */ - py_context_release(&guard); - - return result; + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); } /** @@ -5833,53 +5110,8 @@ static ERL_NIF_TERM nif_context_exec(ErlNifEnv *env, int argc, const ERL_NIF_TER return dispatch_to_worker_thread(env, ctx, CTX_REQ_EXEC, argv[1]); } - /* Legacy mode: direct execution with py_context_acquire. - * For subinterpreters, py_context_acquire handles PyThreadState_Swap - * to switch to the pool slot's interpreter. */ - ErlNifBinary code_bin; - if (!enif_inspect_binary(env, argv[1], &code_bin)) { - return make_error(env, "invalid_code"); - } - - char *code = binary_to_string(&code_bin); - if (code == NULL) { - return make_error(env, "alloc_failed"); - } - - ERL_NIF_TERM result; - - /* Acquire thread state using centralized guard (worker mode only) */ - py_context_guard_t guard = py_context_acquire(ctx); - if (!guard.acquired) { - enif_free(code); - return make_error(env, "acquire_failed"); - } - - /* Set thread-local context for callback support */ - py_context_t *prev_context = tl_current_context; - tl_current_context = ctx; - - /* Execute statements. - * Use globals for both globals and locals to simulate module-level execution. - * This ensures imports are accessible from function definitions. */ - PyObject *py_result = PyRun_String(code, Py_file_input, ctx->globals, ctx->globals); - - if (py_result == NULL) { - result = make_py_error(env); - } else { - Py_DECREF(py_result); - result = ATOM_OK; - } - - /* Restore previous context */ - tl_current_context = prev_context; - - enif_free(code); - - /* Release thread state using centralized guard */ - py_context_release(&guard); - - return result; + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); } /* ============================================================================ @@ -7937,20 +7169,12 @@ static int load(ErlNifEnv *env, void **priv_data, ERL_NIF_TERM load_info) { (void)load_info; /* Create resource types */ - WORKER_RESOURCE_TYPE = enif_open_resource_type( - env, NULL, "py_worker", worker_destructor, - ERL_NIF_RT_CREATE | ERL_NIF_RT_TAKEOVER, NULL); - PYOBJ_RESOURCE_TYPE = enif_open_resource_type( env, NULL, "py_object", pyobj_destructor, ERL_NIF_RT_CREATE | ERL_NIF_RT_TAKEOVER, NULL); /* ASYNC_WORKER_RESOURCE_TYPE removed - replaced by event loop model */ - SUSPENDED_STATE_RESOURCE_TYPE = enif_open_resource_type( - env, NULL, "py_suspended_state", suspended_state_destructor, - ERL_NIF_RT_CREATE | ERL_NIF_RT_TAKEOVER, NULL); - /* Process-per-context resource type (no mutex) */ PY_CONTEXT_RESOURCE_TYPE = enif_open_resource_type( env, NULL, "py_context", context_destructor, @@ -7984,8 +7208,8 @@ static int load(ErlNifEnv *env, void **priv_data, ERL_NIF_TERM load_info) { env, NULL, "py_shared_dict", shared_dict_destructor, ERL_NIF_RT_CREATE | ERL_NIF_RT_TAKEOVER, NULL); - if (WORKER_RESOURCE_TYPE == NULL || PYOBJ_RESOURCE_TYPE == NULL || - SUSPENDED_STATE_RESOURCE_TYPE == NULL || + if (PYOBJ_RESOURCE_TYPE == NULL || + PY_CONTEXT_RESOURCE_TYPE == NULL || PY_REF_RESOURCE_TYPE == NULL || PY_CONTEXT_SUSPENDED_RESOURCE_TYPE == NULL || PY_ENV_RESOURCE_TYPE == NULL || @@ -8023,7 +7247,6 @@ static int load(ErlNifEnv *env, void **priv_data, ERL_NIF_TERM load_info) { ATOM_SPAN_EVENT = enif_make_atom(env, "span_event"); /* Worker pool atoms */ - pool_atoms_init(env); /* Reactor buffer resource type for zero-copy read handling */ REACTOR_BUFFER_RESOURCE_TYPE = enif_open_resource_type( @@ -8112,22 +7335,10 @@ static ErlNifFunc nif_funcs[] = { {"init", 1, nif_py_init, 0}, {"finalize", 0, nif_finalize, 0}, - /* Worker management */ - {"worker_new", 0, nif_worker_new, 0}, - {"worker_new", 1, nif_worker_new, 0}, - {"worker_destroy", 1, nif_worker_destroy, 0}, /* Python execution - dirty I/O NIFs */ - {"worker_call", 5, nif_worker_call, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"worker_call", 6, nif_worker_call, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"worker_eval", 3, nif_worker_eval, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"worker_eval", 4, nif_worker_eval, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"worker_exec", 2, nif_worker_exec, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"worker_next", 2, nif_worker_next, ERL_NIF_DIRTY_JOB_IO_BOUND}, /* Module operations */ - {"import_module", 2, nif_import_module, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"get_attr", 3, nif_get_attr, ERL_NIF_DIRTY_JOB_IO_BOUND}, /* Info */ {"version", 0, nif_version, 0}, @@ -8142,18 +7353,10 @@ static ErlNifFunc nif_funcs[] = { {"tracemalloc_stop", 0, nif_tracemalloc_stop, 0}, /* Callback support */ - {"set_callback_handler", 2, nif_set_callback_handler, 0}, - {"send_callback_response", 2, nif_send_callback_response, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"resume_callback", 2, nif_resume_callback, 0}, /* Async worker management */ - {"async_worker_new", 0, nif_async_worker_new, 0}, - {"async_worker_destroy", 1, nif_async_worker_destroy, 0}, /* Async execution - dirty I/O NIFs */ - {"async_call", 6, nif_async_call, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"async_gather", 3, nif_async_gather, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"async_stream", 6, nif_async_stream, ERL_NIF_DIRTY_JOB_IO_BOUND}, /* Subinterpreter capability probes */ {"subinterp_supported", 0, nif_subinterp_supported, 0}, @@ -8240,8 +7443,6 @@ static ErlNifFunc nif_funcs[] = { {"start_reader", 1, nif_start_reader, 0}, {"stop_writer", 1, nif_stop_writer, 0}, {"start_writer", 1, nif_start_writer, 0}, - {"cancel_reader", 2, nif_cancel_reader, 0}, /* Legacy alias */ - {"cancel_writer", 2, nif_cancel_writer, 0}, /* Legacy alias */ {"close_fd", 1, nif_close_fd, 0}, /* Test helpers for fd monitoring (using pipes) */ {"create_test_pipe", 0, nif_create_test_pipe, 0}, @@ -8265,10 +7466,6 @@ static ErlNifFunc nif_funcs[] = { {"set_shared_worker", 1, nif_set_shared_worker, 0}, /* Worker pool */ - {"pool_start", 1, nif_pool_start, 0}, - {"pool_stop", 0, nif_pool_stop, 0}, - {"pool_submit", 5, nif_pool_submit, 0}, - {"pool_stats", 0, nif_pool_stats, 0}, /* Process-per-context API (no mutex) */ {"context_create", 1, nif_context_create, 0}, diff --git a/c_src/py_nif.h b/c_src/py_nif.h index 45020cb..7dd3b18 100644 --- a/c_src/py_nif.h +++ b/c_src/py_nif.h @@ -146,7 +146,7 @@ typedef enum { /** * @brief Conventional GIL mode (every other supported build) * - * Coordinator-side work runs through the single executor thread. + * Coordinator-side work (thread callbacks) runs on the thread-worker bridge. * Per-context worker / OWN_GIL pthreads handle the public context * APIs directly; this mode label only governs the coordinator path. */ @@ -300,52 +300,6 @@ extern py_invariant_counters_t g_counters; * @{ */ -/** - * @struct py_worker_t - * @brief Represents a Python worker with its own namespace - * - * A worker encapsulates a Python execution context with isolated - * global and local namespaces. Workers are created per-process in - * Erlang and can execute Python code independently. - * - * @note Workers should be created via `py:worker_new/0` and destroyed - * via `py:worker_destroy/1` or automatically via GC. - * - * @see nif_worker_new - * @see nif_worker_destroy - */ -typedef struct { - /** @brief Python thread state for this worker */ - PyThreadState *thread_state; - - /** @brief Global namespace dictionary (`__globals__`) */ - PyObject *globals; - - /** @brief Local namespace dictionary (`__locals__`) */ - PyObject *locals; - - /** @brief Whether this worker currently owns the GIL */ - bool owns_gil; - - /* Callback support fields */ - - /** - * @brief Pipe file descriptors for callback IPC - * - * - `callback_pipe[0]` - Read end (Python reads responses) - * - `callback_pipe[1]` - Write end (Erlang writes responses) - */ - int callback_pipe[2]; - - /** @brief PID of the Erlang callback handler process */ - ErlNifPid callback_handler; - - /** @brief Whether a callback handler is registered */ - bool has_callback_handler; - - /** @brief Environment for building callback messages */ - ErlNifEnv *callback_env; -} py_worker_t; /* async_pending_t and py_async_worker_t removed - async workers replaced by event loop model */ @@ -394,7 +348,7 @@ typedef struct { /** * @defgroup requests Request Handling - * @brief Structures for executor request processing + * @brief Request kinds shared by the context executors and callback replay * @{ */ @@ -403,7 +357,7 @@ typedef struct py_context py_context_t; /** * @enum py_request_type_t - * @brief Types of requests that can be submitted to the executor + * @brief Kinds of Python work a context can run */ typedef enum { PY_REQ_CALL, /**< Call a Python function */ @@ -414,98 +368,9 @@ typedef enum { PY_REQ_GETATTR, /**< Get attribute from Python object */ PY_REQ_MEMORY_STATS, /**< Get Python memory statistics */ PY_REQ_GC, /**< Trigger Python garbage collection */ - PY_REQ_SHUTDOWN /**< Signal executor shutdown */ + PY_REQ_SHUTDOWN /**< Shutdown marker */ } py_request_type_t; -/** - * @struct py_request_t - * @brief Request submitted to the executor thread for processing - * - * Encapsulates all information needed to execute a Python operation. - * The caller thread blocks on the condition variable until the - * executor signals completion. - * - * @note Requests are allocated on the stack by the caller NIF and - * passed to the executor. The executor processes them with - * the GIL held. - */ -typedef struct py_request { - /** @brief Type of operation to perform */ - py_request_type_t type; - - /* Synchronization primitives */ - - /** @brief Mutex for condition variable */ - pthread_mutex_t mutex; - - /** @brief Condition variable for completion signaling */ - pthread_cond_t cond; - - /** @brief Flag set when processing is complete */ - volatile bool completed; - - /* Common parameters */ - - /** @brief Worker context (may be NULL for global ops) */ - py_worker_t *worker; - - /** @brief Context for process-owned operations (may be NULL) */ - py_context_t *context; - - /** @brief Caller's NIF environment for term creation */ - ErlNifEnv *env; - - /* Call/Import parameters */ - - /** @brief Module name as binary */ - ErlNifBinary module_bin; - - /** @brief Function name as binary */ - ErlNifBinary func_bin; - - /** @brief Code string for eval/exec */ - ErlNifBinary code_bin; - - /** @brief Arguments list term */ - ERL_NIF_TERM args_term; - - /** @brief Keyword arguments map term */ - ERL_NIF_TERM kwargs_term; - - /** @brief Local variables map for eval */ - ERL_NIF_TERM locals_term; - - /** @brief Execution timeout in milliseconds (0 = no timeout) */ - unsigned long timeout_ms; - - /* Iterator parameters */ - - /** @brief Generator/iterator wrapper for PY_REQ_NEXT */ - py_object_t *gen_wrapper; - - /* Getattr parameters */ - - /** @brief Object wrapper for PY_REQ_GETATTR */ - py_object_t *obj_wrapper; - - /** @brief Attribute name as binary */ - ErlNifBinary attr_bin; - - /* GC parameters */ - - /** @brief Generation to collect (0, 1, or 2) */ - int gc_generation; - - /* Result */ - - /** @brief Result term set by executor */ - ERL_NIF_TERM result; - - /* Queue linkage */ - - /** @brief Next request in executor queue */ - struct py_request *next; -} py_request_t; /** @} */ @@ -519,94 +384,6 @@ typedef struct py_request { * @{ */ -/** - * @struct suspended_state_t - * @brief State for a suspended Python execution awaiting callback result - * - * When Python code calls `erlang.call()`, execution is suspended and - * this structure captures all state needed to resume after Erlang - * processes the callback. - * - * @par Suspension Flow: - * 1. Python calls `erlang.call('func', args)` - * 2. `erlang_call_impl` raises `SuspensionRequired` exception - * 3. `process_request` catches exception, creates `suspended_state_t` - * 4. Returns `{suspended, CallbackId, StateRef, {Func, Args}}` to Erlang - * 5. Erlang executes callback, calls `resume_callback(StateRef, Result)` - * 6. `nif_resume_callback_dirty` replays Python with cached result - * - * @see erlang_call_impl - * @see nif_resume_callback - */ -typedef struct { - /** @brief Worker context for replay */ - py_worker_t *worker; - - /** @brief Unique identifier for this callback */ - uint64_t callback_id; - - /* Callback invocation info */ - - /** @brief Name of Erlang function being called */ - char *callback_func_name; - - /** @brief Length of callback_func_name */ - size_t callback_func_len; - - /** @brief Arguments passed to the callback */ - PyObject *callback_args; - - /* Original request context for replay */ - - /** @brief Original module name binary */ - ErlNifBinary orig_module; - - /** @brief Original function name binary */ - ErlNifBinary orig_func; - - /** @brief Original arguments (copied to orig_env) */ - ERL_NIF_TERM orig_args; - - /** @brief Original keyword arguments */ - ERL_NIF_TERM orig_kwargs; - - /** @brief Environment owning copied terms */ - ErlNifEnv *orig_env; - - /** @brief Original timeout setting */ - int orig_timeout_ms; - - /** @brief Original request type (PY_REQ_CALL, PY_REQ_EVAL) */ - int request_type; - - /** @brief Original code for eval/exec replay */ - ErlNifBinary orig_code; - - /** @brief Original locals map for eval replay */ - ERL_NIF_TERM orig_locals; - - /* Callback result */ - - /** @brief Raw result data from Erlang callback */ - unsigned char *result_data; - - /** @brief Length of result_data */ - size_t result_len; - - /** @brief Flag: result is available for replay */ - _Atomic bool has_result; - - /** @brief Flag: result represents an error */ - _Atomic bool is_error; - - /* Synchronization */ - - /** @brief Mutex for result access */ - pthread_mutex_t mutex; - - /** @brief Condition for blocking callback mode */ - pthread_cond_t cond; -} suspended_state_t; /** @} */ @@ -1403,16 +1180,12 @@ typedef struct { * @{ */ -/** @brief Resource type for py_worker_t */ -extern ErlNifResourceType *WORKER_RESOURCE_TYPE; /** @brief Resource type for py_object_t */ extern ErlNifResourceType *PYOBJ_RESOURCE_TYPE; /* ASYNC_WORKER_RESOURCE_TYPE removed - async workers replaced by event loop model */ -/** @brief Resource type for suspended_state_t */ -extern ErlNifResourceType *SUSPENDED_STATE_RESOURCE_TYPE; /** @brief Resource type for py_context_t (process-per-context) */ extern ErlNifResourceType *PY_CONTEXT_RESOURCE_TYPE; @@ -1495,28 +1268,13 @@ extern PyThreadState *g_main_thread_state; /** @brief Current execution mode */ extern py_execution_mode_t g_execution_mode; -/* Single executor state */ -/** @brief Single executor thread handle */ -extern pthread_t g_executor_thread; -/** @brief Single executor queue mutex */ -extern pthread_mutex_t g_executor_mutex; -/** @brief Single executor queue condition */ -extern pthread_cond_t g_executor_cond; -/** @brief Single executor queue head */ -extern py_request_t *g_executor_queue_head; -/** @brief Single executor queue tail */ -extern py_request_t *g_executor_queue_tail; -/** @brief Single executor running flag (atomic for thread-safe access) */ -extern _Atomic bool g_executor_running; -/** @brief Single executor shutdown flag (atomic for thread-safe access) */ -extern _Atomic bool g_executor_shutdown; /** @brief Global counter for unique callback IDs */ extern _Atomic uint64_t g_callback_id_counter; @@ -1551,8 +1309,6 @@ extern PyObject *g_numpy_ndarray_type; /* Thread-local state */ -/** @brief Current worker for callback context (legacy) */ -extern __thread py_worker_t *tl_current_worker; /** @brief Current context for callback context (new process-per-context API) */ extern __thread py_context_t *tl_current_context; @@ -1560,8 +1316,6 @@ extern __thread py_context_t *tl_current_context; /** @brief Current NIF environment for callbacks */ extern __thread ErlNifEnv *tl_callback_env; -/** @brief Current suspended state (for replay) */ -extern __thread suspended_state_t *tl_current_suspended; /** @brief Flag: suspension is allowed in current context */ extern __thread bool tl_allow_suspension; @@ -1987,33 +1741,8 @@ static inline uint64_t get_monotonic_ns(void) { return (uint64_t)ts.tv_sec * 1000000000ULL + (uint64_t)ts.tv_nsec; } -/** - * @brief Start timeout monitoring for Python execution - * - * Sets up a trace callback that checks elapsed time and raises - * `TimeoutError` if the deadline is exceeded. - * - * @param timeout_ms Timeout in milliseconds (0 = no timeout) - * - * @see stop_timeout - */ -static void start_timeout(unsigned long timeout_ms); -/** - * @brief Stop timeout monitoring - * - * Removes the trace callback and resets timeout state. - * - * @see start_timeout - */ -static void stop_timeout(void); -/** - * @brief Check if current Python exception is a timeout error - * - * @return true if TimeoutError is pending, false otherwise - */ -static bool check_timeout_error(void); /** @} */ @@ -2027,75 +1756,12 @@ static bool check_timeout_error(void); * @{ */ -/** - * @brief Process a single request with GIL held - * - * Main dispatch function called by executor threads. Handles all - * request types and stores results in the request structure. - * - * @param req Request to process (must not be NULL) - * - * @note Caller must hold the GIL - * @note Sets req->result on completion - */ -static void process_request(py_request_t *req); -/** - * @brief Submit a request to the executor - * - * Routes the request based on execution mode: - * - FREE_THREADED: Execute directly - * - MULTI_EXECUTOR: Route to executor pool - * - SUBINTERP: Use single executor - * - * @param req Request to submit - */ -static int executor_enqueue(py_request_t *req); -/** - * @brief Wait for a request to complete - * - * Blocks until the executor signals completion by setting - * req->completed and signaling req->cond. - * - * @param req Request to wait for - */ -static void executor_wait(py_request_t *req); -/** - * @brief Initialize a request structure - * - * Zeroes the structure and initializes mutex/condvar. - * - * @param req Request to initialize - */ -static void request_init(py_request_t *req); -/** - * @brief Clean up a request structure - * - * Destroys mutex and condvar. Does not free the request itself. - * - * @param req Request to clean up - */ -static void request_cleanup(py_request_t *req); -/** - * @brief Start the single executor thread - * - * Creates and starts the executor thread, waiting for it to - * become ready before returning. - * - * @return 0 on success, -1 on failure - */ -static int executor_start(void); -/** - * @brief Stop the single executor thread - * - * Sends shutdown request and waits for thread to terminate. - */ -static void executor_stop(void); /** @} */ @@ -2148,19 +1814,6 @@ static PyObject *erlang_module_getattr(PyObject *module, PyObject *name); /* async_event_loop_thread removed - replaced by event loop model */ -/** - * @brief Create suspended state for callback handling - * - * Captures all state needed to resume Python execution after - * Erlang processes the callback. - * - * @param env NIF environment - * @param exc_args Exception args tuple (callback_id, func_name, args) - * @param req Original request being processed - * @return New suspended state resource, or NULL on error - */ -static suspended_state_t *create_suspended_state(ErlNifEnv *env, PyObject *exc_args, - py_request_t *req); /** * @brief Parse callback response from Erlang diff --git a/c_src/py_worker_pool.c b/c_src/py_worker_pool.c deleted file mode 100644 index 7452c19..0000000 --- a/c_src/py_worker_pool.c +++ /dev/null @@ -1,921 +0,0 @@ -/* - * Copyright 2026 Benoit Chesneau - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * @file py_worker_pool.c - * @brief Worker thread pool implementation - * - * Implements a pool of worker threads for processing Python operations. - * Each worker can have its own subinterpreter (Python 3.12+) for true - * parallelism, or share the GIL with batching for older Python versions. - */ - -/* ============================================================================ - * Global Pool Instance - * ============================================================================ */ - -py_worker_pool_t g_pool = { - .num_workers = 0, - .initialized = false, - .shutting_down = false, - .request_id_counter = 1, - .use_subinterpreters = false, - .free_threaded = false -}; - -/* Atom for response message */ -static ERL_NIF_TERM ATOM_PY_RESPONSE; - -/* ============================================================================ - * Queue Operations - * ============================================================================ */ - -static void queue_init(py_pool_queue_t *queue) { - queue->head = NULL; - queue->tail = NULL; - atomic_store(&queue->pending_count, 0); - atomic_store(&queue->total_enqueued, 0); - pthread_mutex_init(&queue->mutex, NULL); - pthread_cond_init(&queue->cond, NULL); -} - -static void queue_destroy(py_pool_queue_t *queue) { - pthread_mutex_destroy(&queue->mutex); - pthread_cond_destroy(&queue->cond); -} - -static void queue_enqueue(py_pool_queue_t *queue, py_pool_request_t *req) { - pthread_mutex_lock(&queue->mutex); - - req->next = NULL; - if (queue->tail == NULL) { - queue->head = req; - queue->tail = req; - } else { - queue->tail->next = req; - queue->tail = req; - } - - atomic_fetch_add(&queue->pending_count, 1); - atomic_fetch_add(&queue->total_enqueued, 1); - - pthread_cond_signal(&queue->cond); - pthread_mutex_unlock(&queue->mutex); -} - -static py_pool_request_t *queue_dequeue(py_pool_queue_t *queue, bool wait) { - pthread_mutex_lock(&queue->mutex); - - while (queue->head == NULL) { - if (!wait) { - pthread_mutex_unlock(&queue->mutex); - return NULL; - } - pthread_cond_wait(&queue->cond, &queue->mutex); - - /* Check for shutdown after wakeup */ - if (g_pool.shutting_down) { - pthread_mutex_unlock(&queue->mutex); - return NULL; - } - } - - py_pool_request_t *req = queue->head; - queue->head = req->next; - if (queue->head == NULL) { - queue->tail = NULL; - } - req->next = NULL; - - atomic_fetch_sub(&queue->pending_count, 1); - pthread_mutex_unlock(&queue->mutex); - - return req; -} - -/* Wake up all workers waiting on the queue */ -static void queue_broadcast(py_pool_queue_t *queue) { - pthread_mutex_lock(&queue->mutex); - pthread_cond_broadcast(&queue->cond); - pthread_mutex_unlock(&queue->mutex); -} - -/* ============================================================================ - * Request Management - * ============================================================================ */ - -static py_pool_request_t *py_pool_request_new(py_pool_request_type_t type, - ErlNifPid caller_pid) { - py_pool_request_t *req = enif_alloc(sizeof(py_pool_request_t)); - if (req == NULL) { - return NULL; - } - - memset(req, 0, sizeof(py_pool_request_t)); - req->type = type; - req->caller_pid = caller_pid; - req->request_id = atomic_fetch_add(&g_pool.request_id_counter, 1); - req->msg_env = enif_alloc_env(); - - if (req->msg_env == NULL) { - enif_free(req); - return NULL; - } - - return req; -} - -static void py_pool_request_free(py_pool_request_t *req) { - if (req == NULL) { - return; - } - - if (req->module_name) { - enif_free(req->module_name); - } - if (req->func_name) { - enif_free(req->func_name); - } - if (req->code) { - enif_free(req->code); - } - if (req->msg_env) { - enif_free_env(req->msg_env); - } - - enif_free(req); -} - -/* ============================================================================ - * Module Caching - * ============================================================================ */ - -static PyObject *py_pool_get_module(py_pool_worker_t *worker, - const char *module_name) { - /* Check cache first */ - if (worker->module_cache != NULL) { - PyObject *key = PyUnicode_FromString(module_name); - if (key != NULL) { - PyObject *module = PyDict_GetItem(worker->module_cache, key); - Py_DECREF(key); - if (module != NULL) { - return module; /* Borrowed reference */ - } - } - } - - /* Import module */ - PyObject *module = PyImport_ImportModule(module_name); - if (module == NULL) { - return NULL; - } - - /* Cache it */ - if (worker->module_cache != NULL) { - PyObject *key = PyUnicode_FromString(module_name); - if (key != NULL) { - PyDict_SetItem(worker->module_cache, key, module); - Py_DECREF(key); - } - } - - Py_DECREF(module); /* Dict now owns it, return borrowed ref */ - return PyDict_GetItemString(worker->module_cache, module_name); -} - -/* ============================================================================ - * Response Sending - * ============================================================================ */ - -/* Debug: track sent responses */ -static _Atomic uint64_t g_responses_sent = 0; -static _Atomic uint64_t g_responses_failed = 0; - -static void py_pool_send_response(py_pool_request_t *req, ERL_NIF_TERM result) { - /* Build message: {py_response, RequestId, Result} */ - ERL_NIF_TERM request_id_term = enif_make_uint64(req->msg_env, req->request_id); - ERL_NIF_TERM msg = enif_make_tuple3(req->msg_env, - ATOM_PY_RESPONSE, - request_id_term, - result); - - int send_result = enif_send(NULL, &req->caller_pid, req->msg_env, msg); - if (send_result) { - atomic_fetch_add(&g_responses_sent, 1); - /* IMPORTANT: enif_send consumes/invalidates the msg_env on success. - * Set to NULL to prevent double-free in py_pool_request_free. */ - req->msg_env = NULL; - } else { - /* enif_send fails normally when the caller has already died; the - * g_responses_failed counter records it (no stderr spam). */ - atomic_fetch_add(&g_responses_failed, 1); - /* On failure, msg_env is still valid and will be freed in request_free */ - } -} - -/* ============================================================================ - * Request Processing - CALL/APPLY - * ============================================================================ */ - -static ERL_NIF_TERM py_pool_process_call(py_pool_worker_t *worker, - py_pool_request_t *req) { - ErlNifEnv *env = req->msg_env; - - /* Get module */ - PyObject *module = py_pool_get_module(worker, req->module_name); - if (module == NULL) { - ERL_NIF_TERM err = make_py_error(env); - return err; - } - - /* Get function */ - PyObject *func = PyObject_GetAttrString(module, req->func_name); - if (func == NULL) { - ERL_NIF_TERM err = make_py_error(env); - return err; - } - - /* Convert args to Python */ - PyObject *args = term_to_py(env, req->args_term); - if (args == NULL) { - Py_DECREF(func); - return make_error(env, "args_conversion_failed"); - } - - /* Ensure args is a tuple */ - if (!PyTuple_Check(args)) { - if (PyList_Check(args)) { - PyObject *tuple = PyList_AsTuple(args); - Py_DECREF(args); - args = tuple; - } else { - PyObject *tuple = PyTuple_Pack(1, args); - Py_DECREF(args); - args = tuple; - } - } - - /* Call function */ - PyObject *result = PyObject_Call(func, args, NULL); - Py_DECREF(func); - Py_DECREF(args); - - if (result == NULL) { - return make_py_error(env); - } - - /* Convert result to Erlang */ - ERL_NIF_TERM result_term = py_to_term(env, result); - Py_DECREF(result); - - return enif_make_tuple2(env, ATOM_OK, result_term); -} - -static ERL_NIF_TERM py_pool_process_apply(py_pool_worker_t *worker, - py_pool_request_t *req) { - ErlNifEnv *env = req->msg_env; - - /* Get module */ - PyObject *module = py_pool_get_module(worker, req->module_name); - if (module == NULL) { - return make_py_error(env); - } - - /* Get function */ - PyObject *func = PyObject_GetAttrString(module, req->func_name); - if (func == NULL) { - return make_py_error(env); - } - - /* Convert args to Python */ - PyObject *args = term_to_py(env, req->args_term); - if (args == NULL) { - Py_DECREF(func); - return make_error(env, "args_conversion_failed"); - } - - /* Ensure args is a tuple */ - if (!PyTuple_Check(args)) { - if (PyList_Check(args)) { - PyObject *tuple = PyList_AsTuple(args); - Py_DECREF(args); - args = tuple; - } else { - PyObject *tuple = PyTuple_Pack(1, args); - Py_DECREF(args); - args = tuple; - } - } - - /* Convert kwargs to Python dict */ - PyObject *kwargs = NULL; - if (enif_is_map(env, req->kwargs_term)) { - kwargs = term_to_py(env, req->kwargs_term); - if (kwargs != NULL && !PyDict_Check(kwargs)) { - Py_DECREF(kwargs); - kwargs = NULL; - } - } - - /* Call function with kwargs */ - PyObject *result = PyObject_Call(func, args, kwargs); - Py_DECREF(func); - Py_DECREF(args); - Py_XDECREF(kwargs); - - if (result == NULL) { - return make_py_error(env); - } - - /* Convert result to Erlang */ - ERL_NIF_TERM result_term = py_to_term(env, result); - Py_DECREF(result); - - return enif_make_tuple2(env, ATOM_OK, result_term); -} - -/* ============================================================================ - * Request Processing - EVAL/EXEC - * ============================================================================ */ - -static ERL_NIF_TERM py_pool_process_eval(py_pool_worker_t *worker, - py_pool_request_t *req) { - ErlNifEnv *env = req->msg_env; - - /* Compile code as expression */ - PyObject *code = Py_CompileString(req->code, "", Py_eval_input); - if (code == NULL) { - return make_py_error(env); - } - - /* Prepare locals if provided */ - PyObject *locals = worker->locals; - /* Check if locals_term was set (non-zero) before checking if it's a map */ - if (req->locals_term != 0 && enif_is_map(env, req->locals_term)) { - PyObject *new_locals = term_to_py(env, req->locals_term); - if (new_locals != NULL && PyDict_Check(new_locals)) { - /* Merge with existing locals */ - PyDict_Update(locals, new_locals); - Py_DECREF(new_locals); - } - } - - /* Evaluate */ - PyObject *result = PyEval_EvalCode(code, worker->globals, locals); - Py_DECREF(code); - - if (result == NULL) { - return make_py_error(env); - } - - /* Convert result to Erlang */ - ERL_NIF_TERM result_term = py_to_term(env, result); - Py_DECREF(result); - - return enif_make_tuple2(env, ATOM_OK, result_term); -} - -static ERL_NIF_TERM py_pool_process_exec(py_pool_worker_t *worker, - py_pool_request_t *req) { - ErlNifEnv *env = req->msg_env; - - /* Compile code as statements */ - PyObject *code = Py_CompileString(req->code, "", Py_file_input); - if (code == NULL) { - return make_py_error(env); - } - - /* Execute */ - PyObject *result = PyEval_EvalCode(code, worker->globals, worker->locals); - Py_DECREF(code); - - if (result == NULL) { - return make_py_error(env); - } - - Py_DECREF(result); - return enif_make_tuple2(env, ATOM_OK, ATOM_NONE); -} - -/* ============================================================================ - * Request Processing Dispatcher - * ============================================================================ */ - -static void py_pool_process_request(py_pool_worker_t *worker, - py_pool_request_t *req) { - uint64_t start_ns = get_monotonic_ns(); - ERL_NIF_TERM result; - - switch (req->type) { - case PY_POOL_REQ_CALL: - result = py_pool_process_call(worker, req); - break; - case PY_POOL_REQ_APPLY: - result = py_pool_process_apply(worker, req); - break; - case PY_POOL_REQ_EVAL: - result = py_pool_process_eval(worker, req); - break; - case PY_POOL_REQ_EXEC: - result = py_pool_process_exec(worker, req); - break; - case PY_POOL_REQ_SHUTDOWN: - /* Shutdown handled by worker thread */ - return; - default: - result = make_error(req->msg_env, "unknown_request_type"); - break; - } - - /* Send response */ - py_pool_send_response(req, result); - - /* Update stats */ - uint64_t elapsed_ns = get_monotonic_ns() - start_ns; - atomic_fetch_add(&worker->requests_processed, 1); - atomic_fetch_add(&worker->total_processing_ns, elapsed_ns); -} - -/* ============================================================================ - * Worker Thread - * ============================================================================ */ - -static void *py_pool_worker_thread(void *arg) { - py_pool_worker_t *worker = (py_pool_worker_t *)arg; - - /* Initialize Python state */ - gil_guard_t guard = {0}; - -#ifdef HAVE_SUBINTERPRETERS - if (g_pool.use_subinterpreters) { - /* Acquire GIL in main interpreter first */ - guard = gil_acquire(); - - /* Create sub-interpreter */ - PyInterpreterConfig config = { - .use_main_obmalloc = 0, - .allow_fork = 0, - .allow_exec = 0, - .allow_threads = 1, - .allow_daemon_threads = 0, - .check_multi_interp_extensions = 1, - .gil = PyInterpreterConfig_OWN_GIL, - }; - - PyStatus status = Py_NewInterpreterFromConfig(&worker->tstate, &config); - if (PyStatus_Exception(status)) { - gil_release(guard); - worker->running = false; - return NULL; - } - - worker->interp = PyThreadState_GetInterpreter(worker->tstate); - - /* Initialize event loop for this subinterpreter */ - if (init_subinterpreter_event_loop(NULL) < 0) { - gil_release(guard); - worker->running = false; - return NULL; - } - - /* Release main GIL - we now have our own */ - gil_release(guard); - - /* We're now attached to our sub-interpreter */ - } else -#endif - { - /* Non-subinterpreter mode: acquire the shared GIL */ - guard = gil_acquire(); - } - - /* Create per-worker state */ - worker->module_cache = PyDict_New(); - worker->globals = PyDict_New(); - worker->locals = PyDict_New(); - - if (worker->module_cache == NULL || - worker->globals == NULL || - worker->locals == NULL) { - goto cleanup; - } - - /* Add builtins to globals */ - PyObject *builtins = PyEval_GetBuiltins(); - if (builtins != NULL) { - PyDict_SetItemString(worker->globals, "__builtins__", builtins); - } - - worker->running = true; - - /* Main processing loop */ - while (!worker->shutdown) { - py_pool_request_t *req = NULL; - -#ifdef HAVE_SUBINTERPRETERS - if (g_pool.use_subinterpreters) { - /* Subinterpreter mode: we own our GIL, just dequeue and process */ - req = queue_dequeue(&g_pool.queue, true); - } else -#endif - { - /* Release GIL while waiting for work */ - Py_BEGIN_ALLOW_THREADS - req = queue_dequeue(&g_pool.queue, true); - Py_END_ALLOW_THREADS - } - - if (req == NULL || req->type == PY_POOL_REQ_SHUTDOWN) { - if (req != NULL) { - py_pool_request_free(req); - } - break; - } - - /* Process with GIL held (or in subinterpreter with own GIL) */ - py_pool_process_request(worker, req); - py_pool_request_free(req); - } - -cleanup: - /* Clean up Python state */ - Py_XDECREF(worker->module_cache); - Py_XDECREF(worker->globals); - Py_XDECREF(worker->locals); - worker->module_cache = NULL; - worker->globals = NULL; - worker->locals = NULL; - -#ifdef HAVE_SUBINTERPRETERS - if (g_pool.use_subinterpreters && worker->tstate != NULL) { - Py_EndInterpreter(worker->tstate); - worker->tstate = NULL; - worker->interp = NULL; - } else -#endif - { - gil_release(guard); - } - - worker->running = false; - return NULL; -} - -/* ============================================================================ - * Pool Lifecycle - * ============================================================================ */ - -static int py_pool_init(int num_workers) { - /* Init/shutdown are serialized by the single Erlang gen_server that owns the - * pool, so this check-then-init runs without a concurrent caller and needs no - * extra lock. */ - if (g_pool.initialized) { - return 0; /* Already initialized */ - } - - /* Determine number of workers */ - if (num_workers <= 0) { - /* Auto-detect: use number of CPUs */ - long ncpus = sysconf(_SC_NPROCESSORS_ONLN); - num_workers = (ncpus > 0) ? (int)ncpus : 4; - } - if (num_workers > POOL_MAX_WORKERS) { - num_workers = POOL_MAX_WORKERS; - } - - /* Detect execution mode */ -#ifdef HAVE_FREE_THREADED - g_pool.free_threaded = true; - g_pool.use_subinterpreters = false; -#elif defined(HAVE_SUBINTERPRETERS) - g_pool.free_threaded = false; - g_pool.use_subinterpreters = true; -#else - g_pool.free_threaded = false; - g_pool.use_subinterpreters = false; -#endif - - /* Initialize queue */ - queue_init(&g_pool.queue); - - /* Initialize workers */ - g_pool.num_workers = num_workers; - for (int i = 0; i < num_workers; i++) { - py_pool_worker_t *worker = &g_pool.workers[i]; - memset(worker, 0, sizeof(py_pool_worker_t)); - worker->worker_id = i; - worker->shutdown = false; - atomic_store(&worker->requests_processed, 0); - atomic_store(&worker->total_processing_ns, 0); - } - - /* Start worker threads */ - for (int i = 0; i < num_workers; i++) { - py_pool_worker_t *worker = &g_pool.workers[i]; - int rc = pthread_create(&worker->thread, NULL, - py_pool_worker_thread, worker); - if (rc != 0) { - /* Failed to create thread - shut down already created ones */ - g_pool.shutting_down = true; - queue_broadcast(&g_pool.queue); - for (int j = 0; j < i; j++) { - pthread_join(g_pool.workers[j].thread, NULL); - } - queue_destroy(&g_pool.queue); - return -1; - } - } - - /* Wait for workers to start */ - for (int i = 0; i < num_workers; i++) { - while (!g_pool.workers[i].running && !g_pool.workers[i].shutdown) { - usleep(1000); /* 1ms */ - } - } - - g_pool.initialized = true; - return 0; -} - -static void py_pool_shutdown(void) { - if (!g_pool.initialized) { - return; - } - - g_pool.shutting_down = true; - - /* Send shutdown requests to all workers */ - for (int i = 0; i < g_pool.num_workers; i++) { - g_pool.workers[i].shutdown = true; - - /* Enqueue shutdown request to wake up workers */ - py_pool_request_t *shutdown_req = py_pool_request_new( - PY_POOL_REQ_SHUTDOWN, (ErlNifPid){0}); - if (shutdown_req != NULL) { - queue_enqueue(&g_pool.queue, shutdown_req); - } - } - - /* Wake up all waiting workers */ - queue_broadcast(&g_pool.queue); - - /* Wait for workers to terminate */ - for (int i = 0; i < g_pool.num_workers; i++) { - pthread_join(g_pool.workers[i].thread, NULL); - } - - /* Drain and free remaining requests */ - py_pool_request_t *req; - while ((req = queue_dequeue(&g_pool.queue, false)) != NULL) { - /* Send error response for abandoned requests */ - if (req->type != PY_POOL_REQ_SHUTDOWN && req->msg_env != NULL) { - ERL_NIF_TERM error = make_error(req->msg_env, "pool_shutdown"); - py_pool_send_response(req, error); - } - py_pool_request_free(req); - } - - queue_destroy(&g_pool.queue); - g_pool.initialized = false; - g_pool.shutting_down = false; -} - -static int py_pool_enqueue(py_pool_request_t *req) { - if (!g_pool.initialized || g_pool.shutting_down) { - return -1; - } - - queue_enqueue(&g_pool.queue, req); - return 0; -} - -/* ============================================================================ - * Statistics - * ============================================================================ */ - -static void py_pool_get_stats(py_pool_stats_t *stats) { - memset(stats, 0, sizeof(py_pool_stats_t)); - - stats->num_workers = g_pool.num_workers; - stats->initialized = g_pool.initialized; - stats->use_subinterpreters = g_pool.use_subinterpreters; - stats->free_threaded = g_pool.free_threaded; - stats->pending_count = atomic_load(&g_pool.queue.pending_count); - stats->total_enqueued = atomic_load(&g_pool.queue.total_enqueued); - - for (int i = 0; i < g_pool.num_workers && i < POOL_MAX_WORKERS; i++) { - stats->worker_stats[i].requests_processed = - atomic_load(&g_pool.workers[i].requests_processed); - stats->worker_stats[i].total_processing_ns = - atomic_load(&g_pool.workers[i].total_processing_ns); - } -} - -/* ============================================================================ - * NIF Functions - * ============================================================================ */ - -static ERL_NIF_TERM nif_pool_start(ErlNifEnv *env, int argc, - const ERL_NIF_TERM argv[]) { - if (argc != 1) { - return enif_make_badarg(env); - } - - int num_workers; - if (!enif_get_int(env, argv[0], &num_workers)) { - return enif_make_badarg(env); - } - - if (py_pool_init(num_workers) != 0) { - return make_error(env, "failed_to_start_pool"); - } - - return ATOM_OK; -} - -static ERL_NIF_TERM nif_pool_stop(ErlNifEnv *env, int argc, - const ERL_NIF_TERM argv[]) { - (void)argc; - (void)argv; - - py_pool_shutdown(); - return ATOM_OK; -} - -static ERL_NIF_TERM nif_pool_submit(ErlNifEnv *env, int argc, - const ERL_NIF_TERM argv[]) { - if (argc != 5) { - return enif_make_badarg(env); - } - - if (!g_pool.initialized) { - return make_error(env, "pool_not_started"); - } - - /* Get request type atom */ - char type_buf[32]; - if (!enif_get_atom(env, argv[0], type_buf, sizeof(type_buf), ERL_NIF_LATIN1)) { - return enif_make_badarg(env); - } - - py_pool_request_type_t type; - if (strcmp(type_buf, "call") == 0) { - type = PY_POOL_REQ_CALL; - } else if (strcmp(type_buf, "apply") == 0) { - type = PY_POOL_REQ_APPLY; - } else if (strcmp(type_buf, "eval") == 0) { - type = PY_POOL_REQ_EVAL; - } else if (strcmp(type_buf, "exec") == 0) { - type = PY_POOL_REQ_EXEC; - } else { - return make_error(env, "unknown_request_type"); - } - - /* Get caller PID */ - ErlNifPid caller_pid; - if (!enif_self(env, &caller_pid)) { - return make_error(env, "cannot_get_self_pid"); - } - - /* Create request */ - py_pool_request_t *req = py_pool_request_new(type, caller_pid); - if (req == NULL) { - return make_error(env, "request_allocation_failed"); - } - - /* Parse arguments based on type */ - switch (type) { - case PY_POOL_REQ_CALL: - case PY_POOL_REQ_APPLY: { - /* argv[1] = Module, argv[2] = Func, argv[3] = Args, argv[4] = Kwargs/undefined */ - ErlNifBinary module_bin, func_bin; - if (!enif_inspect_binary(env, argv[1], &module_bin) || - !enif_inspect_binary(env, argv[2], &func_bin)) { - py_pool_request_free(req); - return enif_make_badarg(env); - } - - req->module_name = enif_alloc(module_bin.size + 1); - req->func_name = enif_alloc(func_bin.size + 1); - if (req->module_name == NULL || req->func_name == NULL) { - py_pool_request_free(req); - return make_error(env, "allocation_failed"); - } - - memcpy(req->module_name, module_bin.data, module_bin.size); - req->module_name[module_bin.size] = '\0'; - memcpy(req->func_name, func_bin.data, func_bin.size); - req->func_name[func_bin.size] = '\0'; - - req->args_term = enif_make_copy(req->msg_env, argv[3]); - - if (type == PY_POOL_REQ_APPLY && !enif_is_atom(env, argv[4])) { - req->kwargs_term = enif_make_copy(req->msg_env, argv[4]); - } - break; - } - - case PY_POOL_REQ_EVAL: - case PY_POOL_REQ_EXEC: { - /* argv[1] = Code, argv[2-4] = unused */ - ErlNifBinary code_bin; - if (!enif_inspect_binary(env, argv[1], &code_bin)) { - py_pool_request_free(req); - return enif_make_badarg(env); - } - - req->code = enif_alloc(code_bin.size + 1); - if (req->code == NULL) { - py_pool_request_free(req); - return make_error(env, "allocation_failed"); - } - - memcpy(req->code, code_bin.data, code_bin.size); - req->code[code_bin.size] = '\0'; - - if (!enif_is_atom(env, argv[2])) { - req->locals_term = enif_make_copy(req->msg_env, argv[2]); - } - break; - } - - default: - py_pool_request_free(req); - return make_error(env, "unknown_request_type"); - } - - /* IMPORTANT: Save request_id BEFORE enqueueing. - * Once enqueued, a worker can process and free the request at any time. - * Accessing req->request_id after enqueue is use-after-free. */ - uint64_t request_id = req->request_id; - - /* Enqueue request */ - if (py_pool_enqueue(req) != 0) { - py_pool_request_free(req); - return make_error(env, "enqueue_failed"); - } - - /* Return {ok, RequestId} - using saved ID to avoid use-after-free */ - return enif_make_tuple2(env, ATOM_OK, - enif_make_uint64(env, request_id)); -} - -static ERL_NIF_TERM nif_pool_stats(ErlNifEnv *env, int argc, - const ERL_NIF_TERM argv[]) { - (void)argc; - (void)argv; - - py_pool_stats_t stats; - py_pool_get_stats(&stats); - - /* Build result map */ - ERL_NIF_TERM keys[8], values[8]; - - keys[0] = enif_make_atom(env, "num_workers"); - values[0] = enif_make_int(env, stats.num_workers); - - keys[1] = enif_make_atom(env, "initialized"); - values[1] = stats.initialized ? ATOM_TRUE : ATOM_FALSE; - - keys[2] = enif_make_atom(env, "use_subinterpreters"); - values[2] = stats.use_subinterpreters ? ATOM_TRUE : ATOM_FALSE; - - keys[3] = enif_make_atom(env, "free_threaded"); - values[3] = stats.free_threaded ? ATOM_TRUE : ATOM_FALSE; - - keys[4] = enif_make_atom(env, "pending_count"); - values[4] = enif_make_uint64(env, stats.pending_count); - - keys[5] = enif_make_atom(env, "total_enqueued"); - values[5] = enif_make_uint64(env, stats.total_enqueued); - - keys[6] = enif_make_atom(env, "responses_sent"); - values[6] = enif_make_uint64(env, atomic_load(&g_responses_sent)); - - keys[7] = enif_make_atom(env, "responses_failed"); - values[7] = enif_make_uint64(env, atomic_load(&g_responses_failed)); - - ERL_NIF_TERM result; - enif_make_map_from_arrays(env, keys, values, 8, &result); - - return result; -} - -/* Initialize pool-specific atoms */ -static int pool_atoms_init(ErlNifEnv *env) { - ATOM_PY_RESPONSE = enif_make_atom(env, "py_response"); - return 0; -} diff --git a/c_src/py_worker_pool.h b/c_src/py_worker_pool.h deleted file mode 100644 index 2ca6147..0000000 --- a/c_src/py_worker_pool.h +++ /dev/null @@ -1,496 +0,0 @@ -/* - * Copyright 2026 Benoit Chesneau - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * @file py_worker_pool.h - * @brief Worker thread pool for Python operations - * @author Benoit Chesneau - * - * @section overview Overview - * - * This module implements a general-purpose worker thread pool for Python - * calls (py:call, py:eval). Each worker has its own subinterpreter - * (Python 3.12+) or dedicated GIL-holding thread, processing requests from a - * shared queue. - * - * @section architecture Architecture - * - * ``` - * Erlang Processes Lock-free Queue Python Workers - * [P1]--enqueue--+ +---------+ +-------------------------+ - * [P2]--enqueue--+---->| Request |<--poll---| Worker 0 (Subinterp+GIL)| - * [P3]--enqueue--+ | Queue | | - Holds GIL | - * ... | | (MPSC) |<--poll---| Worker 1 (Subinterp+GIL)| - * [PN]--enqueue--+ +---------+ +-------------------------+ - * ``` - * - * @section benefits Key Benefits - * - * - No GIL acquire/release per request (workers hold GIL) - * - Module/callable cached per worker (no reimport) - * - True parallelism with subinterpreters (each has OWN_GIL) - * - * @section modes Python Mode Support - * - * | Mode | Python Version | Strategy | - * |------|----------------|----------| - * | FREE_THREADED | 3.13+ (no-GIL) | N workers, no GIL needed | - * | SUBINTERP | 3.12+ | N subinterpreters, each OWN_GIL | - * | FALLBACK | <3.12 | N workers share GIL, batching reduces overhead | - */ - -#ifndef PY_WORKER_POOL_H -#define PY_WORKER_POOL_H - -#include "py_nif.h" - -/* ============================================================================ - * Configuration - * ============================================================================ */ - -/** - * @def POOL_MAX_WORKERS - * @brief Maximum number of workers in the pool - */ -#define POOL_MAX_WORKERS 32 - -/** - * @def POOL_QUEUE_SIZE - * @brief Size of the request queue (power of 2 for efficient modulo) - */ -#define POOL_QUEUE_SIZE 4096 - -/** - * @def POOL_DEFAULT_WORKERS - * @brief Default number of workers (0 = use CPU count) - */ -#define POOL_DEFAULT_WORKERS 0 - -/* ============================================================================ - * Request Types - * ============================================================================ */ - -/** - * @enum py_pool_request_type_t - * @brief Types of requests that can be submitted to the worker pool - */ -typedef enum { - PY_POOL_REQ_CALL, /**< py:call(Module, Func, Args) */ - PY_POOL_REQ_APPLY, /**< py:apply(Module, Func, Args, Kwargs) */ - PY_POOL_REQ_EVAL, /**< py:eval(Code) */ - PY_POOL_REQ_EXEC, /**< py:exec(Code) */ - PY_POOL_REQ_SHUTDOWN /**< Shutdown signal */ -} py_pool_request_type_t; - -/* ============================================================================ - * Request Structure - * ============================================================================ */ - -/** - * @struct py_pool_request_t - * @brief Request submitted to the worker pool - * - * Contains all information needed to process a Python operation. - * The result is sent back to the caller via enif_send(). - */ -typedef struct py_pool_request { - /** @brief Unique request ID for correlation */ - uint64_t request_id; - - /** @brief Type of operation to perform */ - py_pool_request_type_t type; - - /** @brief PID of the calling Erlang process */ - ErlNifPid caller_pid; - - /** @brief Environment for building result terms (thread-safe copy) */ - ErlNifEnv *msg_env; - - /* ========== CALL/APPLY parameters ========== */ - - /** @brief Module name (heap-allocated, NULL-terminated) */ - char *module_name; - - /** @brief Function name (heap-allocated, NULL-terminated) */ - char *func_name; - - /** @brief Arguments list term (copied to msg_env) */ - ERL_NIF_TERM args_term; - - /** @brief Keyword arguments map term (copied to msg_env, optional) */ - ERL_NIF_TERM kwargs_term; - - /* ========== EVAL/EXEC parameters ========== */ - - /** @brief Python code to evaluate/execute (heap-allocated, NULL-terminated) */ - char *code; - - /** @brief Local variables for eval (copied to msg_env) */ - ERL_NIF_TERM locals_term; - - /* ========== Timeout ========== */ - - /** @brief Timeout in milliseconds (0 = no timeout) */ - unsigned long timeout_ms; - - /* ========== Queue linkage ========== */ - - /** @brief Next request in queue (for linked list) */ - struct py_pool_request *next; -} py_pool_request_t; - -/* ============================================================================ - * Worker Structure - * ============================================================================ */ - -/** - * @struct py_pool_worker_t - * @brief Single worker thread in the pool - * - * Each worker runs in its own thread and optionally has its own - * subinterpreter (Python 3.12+) for true parallelism. - */ -typedef struct { - /** @brief Worker thread handle */ - pthread_t thread; - - /** @brief Worker ID (0 to num_workers-1) */ - int worker_id; - - /** @brief Flag: worker is running */ - volatile bool running; - - /** @brief Flag: worker should shut down */ - volatile bool shutdown; - -#ifdef HAVE_SUBINTERPRETERS - /** @brief Python interpreter for this worker */ - PyInterpreterState *interp; - - /** @brief Thread state in this interpreter */ - PyThreadState *tstate; -#endif - - /* ========== Cached state per worker ========== */ - - /** @brief Module cache (Dict: module_name -> PyModule) */ - PyObject *module_cache; - - /** @brief Global namespace for eval/exec */ - PyObject *globals; - - /** @brief Local namespace for eval/exec */ - PyObject *locals; - - /* ========== Statistics ========== */ - - /** @brief Total requests processed by this worker */ - _Atomic uint64_t requests_processed; - - /** @brief Total processing time in nanoseconds */ - _Atomic uint64_t total_processing_ns; -} py_pool_worker_t; - -/* ============================================================================ - * Request Queue Structure - * ============================================================================ */ - -/** - * @struct py_pool_queue_t - * @brief MPSC (Multi-Producer Single-Consumer) queue for requests - * - * Uses a simple linked list with mutex protection. Workers dequeue - * using condition variable waits. - */ -typedef struct { - /** @brief Queue head (oldest request) */ - py_pool_request_t *head; - - /** @brief Queue tail (newest request) */ - py_pool_request_t *tail; - - /** @brief Number of pending requests */ - _Atomic uint64_t pending_count; - - /** @brief Total requests enqueued */ - _Atomic uint64_t total_enqueued; - - /** @brief Mutex protecting the queue */ - pthread_mutex_t mutex; - - /** @brief Condition variable for worker notification */ - pthread_cond_t cond; -} py_pool_queue_t; - -/* ============================================================================ - * Worker Pool Structure - * ============================================================================ */ - -/** - * @struct py_worker_pool_t - * @brief The main worker pool structure - */ -typedef struct { - /** @brief Array of workers */ - py_pool_worker_t workers[POOL_MAX_WORKERS]; - - /** @brief Number of active workers */ - int num_workers; - - /** @brief Request queue */ - py_pool_queue_t queue; - - /** @brief Flag: pool is initialized */ - volatile bool initialized; - - /** @brief Flag: pool is shutting down */ - volatile bool shutting_down; - - /** @brief Request ID counter */ - _Atomic uint64_t request_id_counter; - - /** @brief Mode: use subinterpreters */ - bool use_subinterpreters; - - /** @brief Mode: free-threaded Python */ - bool free_threaded; -} py_worker_pool_t; - -/* ============================================================================ - * Global Pool Instance - * ============================================================================ */ - -/** @brief Global worker pool instance */ -extern py_worker_pool_t g_pool; - -/* ============================================================================ - * Pool Lifecycle Functions - * ============================================================================ */ - -/** - * @brief Initialize the worker pool - * - * Creates and starts num_workers worker threads. If num_workers is 0, - * uses the number of CPU cores. - * - * @param num_workers Number of workers (0 = auto-detect CPU count) - * @return 0 on success, -1 on failure - */ -static int py_pool_init(int num_workers); - -/** - * @brief Shut down the worker pool - * - * Signals all workers to stop and waits for them to terminate. - * Processes any remaining requests with error responses. - */ -static void py_pool_shutdown(void); - -/* ============================================================================ - * Request Submission Functions - * ============================================================================ */ - -/** - * @brief Submit a request to the pool - * - * Thread-safe enqueue operation. The request is processed by an - * available worker and the result is sent to caller_pid. - * - * @param req Request to submit (ownership transferred to pool) - * @return 0 on success, -1 if pool not initialized - */ -static int py_pool_enqueue(py_pool_request_t *req); - -/** - * @brief Create a new pool request - * - * Allocates and initializes a request structure. - * - * @param type Request type - * @param caller_pid Calling process PID - * @return New request, or NULL on allocation failure - */ -static py_pool_request_t *py_pool_request_new(py_pool_request_type_t type, - ErlNifPid caller_pid); - -/** - * @brief Free a pool request - * - * Releases all resources associated with the request. - * - * @param req Request to free - */ -static void py_pool_request_free(py_pool_request_t *req); - -/* ============================================================================ - * Worker Functions - * ============================================================================ */ - -/** - * @brief Worker thread main function - * - * Entry point for worker threads. Processes requests until shutdown. - * - * @param arg Pointer to py_pool_worker_t - * @return NULL - */ -static void *py_pool_worker_thread(void *arg); - -/** - * @brief Process a single request - * - * Dispatches based on request type and sends result to caller. - * - * @param worker Worker processing the request - * @param req Request to process - */ -static void py_pool_process_request(py_pool_worker_t *worker, - py_pool_request_t *req); - -/** - * @brief Send response to caller - * - * Uses enif_send() to send result back to calling process. - * - * @param req Request with caller info - * @param result Result term to send - */ -static void py_pool_send_response(py_pool_request_t *req, ERL_NIF_TERM result); - -/* ============================================================================ - * Request Processing Functions - * ============================================================================ */ - -/** - * @brief Process CALL request - * - * @param worker Worker processing request - * @param req Request with module, func, args - * @return Result term - */ -static ERL_NIF_TERM py_pool_process_call(py_pool_worker_t *worker, - py_pool_request_t *req); - -/** - * @brief Process APPLY request - * - * @param worker Worker processing request - * @param req Request with module, func, args, kwargs - * @return Result term - */ -static ERL_NIF_TERM py_pool_process_apply(py_pool_worker_t *worker, - py_pool_request_t *req); - -/** - * @brief Process EVAL request - * - * @param worker Worker processing request - * @param req Request with code - * @return Result term - */ -static ERL_NIF_TERM py_pool_process_eval(py_pool_worker_t *worker, - py_pool_request_t *req); - -/** - * @brief Process EXEC request - * - * @param worker Worker processing request - * @param req Request with code - * @return Result term - */ -static ERL_NIF_TERM py_pool_process_exec(py_pool_worker_t *worker, - py_pool_request_t *req); - -/* ============================================================================ - * Module Caching - * ============================================================================ */ - -/** - * @brief Get or import a Python module - * - * Checks the worker's module cache first, imports if not cached. - * - * @param worker Worker with module cache - * @param module_name Module name to get - * @return Borrowed reference to module, or NULL on error - */ -static PyObject *py_pool_get_module(py_pool_worker_t *worker, - const char *module_name); - -/* ============================================================================ - * Statistics - * ============================================================================ */ - -/** - * @brief Pool statistics structure - */ -typedef struct { - int num_workers; - bool initialized; - bool use_subinterpreters; - bool free_threaded; - uint64_t pending_count; - uint64_t total_enqueued; - struct { - uint64_t requests_processed; - uint64_t total_processing_ns; - } worker_stats[POOL_MAX_WORKERS]; -} py_pool_stats_t; - -/** - * @brief Get pool statistics - * - * @param stats Output structure for statistics - */ -static void py_pool_get_stats(py_pool_stats_t *stats); - -/* ============================================================================ - * NIF Functions - * ============================================================================ */ - -/** - * @brief NIF: Start the worker pool - * - * py_nif:pool_start(NumWorkers) -> ok | {error, Reason} - */ -static ERL_NIF_TERM nif_pool_start(ErlNifEnv *env, int argc, - const ERL_NIF_TERM argv[]); - -/** - * @brief NIF: Stop the worker pool - * - * py_nif:pool_stop() -> ok - */ -static ERL_NIF_TERM nif_pool_stop(ErlNifEnv *env, int argc, - const ERL_NIF_TERM argv[]); - -/** - * @brief NIF: Submit a request to the pool - * - * py_nif:pool_submit(Type, Arg1, Arg2, Arg3, Arg4) -> {ok, RequestId} | {error, Reason} - */ -static ERL_NIF_TERM nif_pool_submit(ErlNifEnv *env, int argc, - const ERL_NIF_TERM argv[]); - -/** - * @brief NIF: Get pool statistics - * - * py_nif:pool_stats() -> StatsMap - */ -static ERL_NIF_TERM nif_pool_stats(ErlNifEnv *env, int argc, - const ERL_NIF_TERM argv[]); - -#endif /* PY_WORKER_POOL_H */ diff --git a/docs/architecture.md b/docs/architecture.md index 53e6426..841a2a7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -79,10 +79,9 @@ one Python execution environment and serves calls in order. Pools `{py_result, Ref, Result}` to the `py_context` process, which replies `From ! {MRef, Result}`. -The older paths in `nif_context_call` (a blocking variant with an inline -"legacy" executor) are kept for the fallback -`{error, async_requires_worker_thread}` and are not taken by contexts -created today; see [code map](code-map.md) for the list of legacy code. +`nif_context_call` and friends also have a blocking variant used as a +fallback when a context has no thread (`{error, async_requires_worker_thread}`), +which never happens for contexts created today. ### isolated @@ -118,9 +117,7 @@ this order (the comment above it is the authoritative version): a request on a pipe and blocks; the `py_context` process has a dedicated handler (`callback_handler_loop/1`) that runs the fun and writes the response frame back with `context_write_callback_response`. -3. **Legacy worker handler** (`worker_*` NIFs): only used by - `examples/gen_test.erl`. -4. **Thread worker** (`c_src/py_thread_worker.c`): any Python thread that is +3. **Thread worker** (`c_src/py_thread_worker.c`): any Python thread that is not a context thread (`threading.Thread`, executors) asks the `py_thread_handler` coordinator for a handler process and talks to it over a pipe. There is also an async variant (`erlang.async_call`) using a @@ -190,9 +187,8 @@ loop and [asyncio](asyncio.md) for the API. ## What is live and what is not -Kept for now, not used by current contexts: the `worker_*` NIF API and its -single executor thread (`c_src/py_exec.c`), the `async_worker_*` NIFs (they -return `deprecated`), `c_src/py_worker_pool.c` (no caller), the inline -"legacy" executor branches in `nif_context_*`, and the test-only fd NIFs in -`c_src/py_event_loop.c`. They are listed in the [code map](code-map.md) so -nobody debugs them by mistake; removing them is planned. +Every code path in `src/` and `c_src/` is on the path of a context created +today, with one exception: the test-only fd/TCP/UDP NIFs in +`c_src/py_event_loop.c` ("Test Helper Functions"), which the suites use. +The legacy worker API, its executor thread, the deprecated async worker +NIFs and the unused worker pool were removed in 5.0.0. diff --git a/docs/code-map.md b/docs/code-map.md index 3139871..5b237d8 100644 --- a/docs/code-map.md +++ b/docs/code-map.md @@ -1,9 +1,8 @@ # Code map Every source file, what it owns, and where to look for its behaviour. Status -is `live` (on the path of a context created today), `legacy` (kept for -compatibility, no current caller in `src/`), or `test` (only exercised by -suites). Guides are in `docs/`, suites in `test/`. Start with +is `live` (on the path of a context created today) or `test` (only +exercised by suites). Guides are in `docs/`, suites in `test/`. Start with [architecture](architecture.md). ## Erlang (`src/`) @@ -42,9 +41,9 @@ files. Editing `py_convert.c` alone does not compile it alone; build with | File | Owns | Status | |---|---|---| | `py_nif.h` | Every shared type: `py_context_t`, request types, runtime state machine, atoms, globals | live | -| `py_nif.c` | Runtime init, context creation and destruction, the request queue and the two context thread mains, the process-per-context NIFs (`nif_context_*`), process-local envs, `py_ref`, the NIF table | live, with legacy branches | +| `py_nif.c` | Runtime init, context creation and destruction, the request queue and the two context thread mains, the process-per-context NIFs (`nif_context_*`), process-local envs, `py_ref`, the NIF table | live | | `py_convert.c` | `py_to_term` / `term_to_py`, the type mapping, tagged tuples (`{bytes, B}`, shared handles) | live | -| `py_exec.c` | Execution with suspension support; the legacy single executor thread | live (suspension), legacy (executor) | +| `py_exec.c` | Execution mode detection and GIL helpers | live | | `py_callback.c` | The `erlang` Python module: `call`, `send`, `whereis`, `Atom`/`Pid`/`Ref` types, schedule markers, callback pipes, channel and shared dict methods | live | | `py_thread_worker.c` | Python threads calling Erlang through `py_thread_handler` | live | | `py_subinterp_thread.c` | Sub-interpreter thread pool used by owngil contexts and loop pools | live | @@ -52,14 +51,10 @@ files. Editing `py_convert.c` alone does not compile it alone; build with | `py_channel.c`, `py_buffer.c`, `py_reactor_buffer.c`, `py_shared_dict.c` | The corresponding resources and their Python-facing methods | live | | `py_logging.c` | Logging and tracing NIFs | live | | `py_mem_limit.c` | Per-interpreter memory caps (owngil) | live | -| `py_worker_pool.c/.h` | An older worker pool | legacy, no caller | | `py_util.c/.h` | Macros and helpers | live | -Inside `py_nif.c`, these are legacy: the `worker_*` NIFs and the "Worker -management" section, the `async_worker_*` NIFs (return `deprecated`), the -inline executor branches marked "Legacy mode" in `nif_context_call`, -`nif_context_eval`, `nif_context_exec`, and the `cancel_reader/writer` -aliases. +The only code not on a live path is the "Test Helper Functions" section of +`py_event_loop.c` (fd, pipe, TCP and UDP helpers the suites use). ## Python (`priv/`) diff --git a/docs/glossary.md b/docs/glossary.md index cf5864c..439622e 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -40,8 +40,6 @@ The most overloaded word. Meanings, by file: |---|---|---| | `py_context:new(#{mode => worker})` | the context mode above | worker mode | | `worker_context_thread_main`, `uses_worker_thread` (`py_nif.c`) | the pthread that serves a context's queue | context thread | -| `worker_new/call/eval/exec` NIFs, `py_worker_t` | the legacy per-worker API, before contexts | legacy worker API | -| `py_worker_pool.c`, `py_pool_worker_t` | an older pool, no caller | legacy pool | | `thread_worker`, `thread_worker_call` (`py_thread_worker.c`), `py_thread_handler` | the channel a Python thread uses to call Erlang | thread callback bridge | | `py_event_worker` | the Erlang process that drives one asyncio loop (readiness, timers) | loop driver | | `docs/workers.md`, "worker loop" | a long-running asyncio loop on a context thread, gunicorn-style | worker loop | @@ -51,7 +49,7 @@ The most overloaded word. Meanings, by file: `py_context_router` pools (`py:call(Pool, M, F, A)`): named sets of contexts routed by scheduler. `py_event_loop_pool`: main-interpreter asyncio loops with process affinity. `g_thread_pool` in `py_subinterp_thread.c`: the -threads behind owngil contexts. `g_pool` in `py_worker_pool.c`: legacy. +threads behind owngil contexts. ## Callback diff --git a/docs/scalability.md b/docs/scalability.md index a725af7..4968bc7 100644 --- a/docs/scalability.md +++ b/docs/scalability.md @@ -495,7 +495,7 @@ def process(x): │ 2. Erlang executes the registered callback │ │ └──► May call py:call() to run Python (on different worker) │ │ │ -│ 3. Erlang calls resume_callback with result │ +│ 3. Erlang calls context_resume with result │ │ └──► Schedules dirty NIF to return result to Python │ │ │ │ 4. Python continues with the callback result │ diff --git a/examples/gen_test.erl b/examples/gen_test.erl index d3c75af..27b5ad9 100644 --- a/examples/gen_test.erl +++ b/examples/gen_test.erl @@ -1,58 +1,37 @@ -#!/usr/bin/env escript -%%% Generator iteration test --mode(compile). - -main(_) -> - code:add_patha("_build/default/lib/erlang_python/ebin"), - +%% Iterating Python generators from Erlang. +%% +%% A generator object cannot cross into Erlang, so the values are streamed: +%% py:stream/3 collects everything a generator yields, py:stream_start/3 +%% delivers them one message at a time. Run with: +%% +%% rebar3 shell +%% > c("examples/gen_test.erl"), gen_test:run(). +-module(gen_test). +-export([run/0]). + +run() -> {ok, _} = application:ensure_all_started(erlang_python), - - io:format("=== Generator Iteration Test ===~n~n"), - - %% Get a worker directly - ok = py_nif:init(), - {ok, Worker} = py_nif:worker_new(), - - %% Create a generator via eval - io:format("Creating generator (x**2 for x in range(5))...~n"), - {ok, {generator, Gen}} = py_nif:worker_eval(Worker, <<"(x**2 for x in range(5))">>, #{}), - io:format("Got generator ref~n~n"), - - %% Iterate manually - io:format("Iterating: "), - iterate(Worker, Gen), - io:format("~n~n"), - - %% Test with range - io:format("Range(10): "), - {ok, {generator, Gen2}} = py_nif:worker_eval(Worker, <<"iter(range(10))">>, #{}), - iterate(Worker, Gen2), - io:format("~n~n"), - - %% Test Fibonacci generator defined inline - io:format("Fibonacci via exec + call:~n"), - ok = py_nif:worker_exec(Worker, <<" -def fib(n): - a, b = 0, 1 - for _ in range(n): - yield a - a, b = b, a + b -">>), - {ok, {generator, Gen3}} = py_nif:worker_call(Worker, <<"__main__">>, <<"fib">>, [10], #{}), - io:format(" fib(10) = "), - iterate(Worker, Gen3), - io:format("~n~n"), - - io:format("=== Done ===~n"), - ok = application:stop(erlang_python). - -iterate(Worker, Gen) -> - case py_nif:worker_next(Worker, Gen) of - {ok, Value} -> - io:format("~p ", [Value]), - iterate(Worker, Gen); - {error, stop_iteration} -> - ok; - {error, Error} -> - io:format("Error: ~p", [Error]) + %% A generator expression, collected in one go + {ok, Squares} = py:stream_eval(<<"(x**2 for x in range(5))">>), + io:format("squares: ~p~n", [Squares]), + + %% Any iterable a module function returns + {ok, Range} = py:stream(builtins, range, [5]), + io:format("range: ~p~n", [Range]), + + %% One value per message, as the generator yields them + {ok, Ref} = py:stream_start(builtins, iter, [[1, 2, 3]]), + receive_all(Ref). + +receive_all(Ref) -> + receive + {py_stream, Ref, {data, V}} -> + io:format("got ~p~n", [V]), + receive_all(Ref); + {py_stream, Ref, done} -> + io:format("done~n"); + {py_stream, Ref, {error, Reason}} -> + io:format("error: ~p~n", [Reason]) + after 5000 -> + timeout end. diff --git a/src/erlang_python.app.src b/src/erlang_python.app.src index 70a95fa..c478e29 100644 --- a/src/erlang_python.app.src +++ b/src/erlang_python.app.src @@ -1,6 +1,6 @@ {application, erlang_python, [ {description, "Execute Python applications from Erlang using dirty NIFs"}, - {vsn, "4.2.0"}, + {vsn, "5.0.0"}, {registered, []}, {mod, {erlang_python_app, []}}, {applications, [ diff --git a/src/py_nif.erl b/src/py_nif.erl index e868dc8..f4dddc3 100644 --- a/src/py_nif.erl +++ b/src/py_nif.erl @@ -24,17 +24,6 @@ init/0, init/1, finalize/0, - worker_new/0, - worker_new/1, - worker_destroy/1, - worker_call/5, - worker_call/6, - worker_eval/3, - worker_eval/4, - worker_exec/2, - worker_next/2, - import_module/2, - get_attr/3, version/0, memory_stats/0, get_debug_counters/0, @@ -43,15 +32,7 @@ tracemalloc_start/0, tracemalloc_start/1, tracemalloc_stop/0, - set_callback_handler/2, - send_callback_response/2, - resume_callback/2, %% Async workers - async_worker_new/0, - async_worker_destroy/1, - async_call/6, - async_gather/3, - async_stream/6, %% Subinterpreter capability probes (Python 3.12+ / 3.14+) subinterp_supported/0, owngil_supported/0, @@ -126,8 +107,6 @@ start_reader/1, stop_writer/1, start_writer/1, - cancel_reader/2, %% Legacy alias for stop_reader - cancel_writer/2, %% Legacy alias for stop_writer close_fd/1, %% File descriptor utilities dup_fd/1, @@ -151,10 +130,6 @@ set_isolation_mode/1, set_shared_worker/1, %% Worker pool - pool_start/1, - pool_stop/0, - pool_submit/5, - pool_stats/0, %% Process-per-context API (no mutex) context_create/1, context_destroy/1, @@ -286,82 +261,10 @@ init(_Opts) -> finalize() -> ?NIF_STUB. -%%% ============================================================================ -%%% Worker Management -%%% ============================================================================ - -%% @doc Create a new Python worker context. -%% Returns an opaque reference to be used with other worker functions. --spec worker_new() -> {ok, reference()} | {error, term()}. -worker_new() -> - worker_new(#{}). - -%% @doc Create a worker with options. -%% Options: -%% use_subinterpreter => boolean() - Use a separate sub-interpreter (Python 3.12+) --spec worker_new(map()) -> {ok, reference()} | {error, term()}. -worker_new(_Opts) -> - ?NIF_STUB. - -%% @doc Destroy a worker context. --spec worker_destroy(reference()) -> ok. -worker_destroy(_WorkerRef) -> - ?NIF_STUB. - -%% @doc Call a Python function from a worker. -%% This is a dirty NIF that acquires the GIL. -%% May return {suspended, ...} if Python calls erlang.call() (reentrant callback). --spec worker_call(reference(), binary(), binary(), list(), map()) -> - {ok, term()} | {error, term()} | {suspended, term(), reference(), {binary(), term()}}. -worker_call(_WorkerRef, _Module, _Func, _Args, _Kwargs) -> - ?NIF_STUB. - -%% @doc Call a Python function from a worker with timeout. -%% May return {suspended, ...} if Python calls erlang.call() (reentrant callback). --spec worker_call(reference(), binary(), binary(), list(), map(), non_neg_integer()) -> - {ok, term()} | {error, term()} | {suspended, term(), reference(), {binary(), term()}}. -worker_call(_WorkerRef, _Module, _Func, _Args, _Kwargs, _TimeoutMs) -> - ?NIF_STUB. - -%% @doc Evaluate a Python expression in a worker. -%% May return {suspended, ...} if Python calls erlang.call() (reentrant callback). --spec worker_eval(reference(), binary(), map()) -> - {ok, term()} | {error, term()} | {suspended, term(), reference(), {binary(), term()}}. -worker_eval(_WorkerRef, _Code, _Locals) -> - ?NIF_STUB. - -%% @doc Evaluate a Python expression in a worker with timeout. -%% May return {suspended, ...} if Python calls erlang.call() (reentrant callback). --spec worker_eval(reference(), binary(), map(), non_neg_integer()) -> - {ok, term()} | {error, term()} | {suspended, term(), reference(), {binary(), term()}}. -worker_eval(_WorkerRef, _Code, _Locals, _TimeoutMs) -> - ?NIF_STUB. - -%% @doc Execute Python statements in a worker. --spec worker_exec(reference(), binary()) -> ok | {error, term()}. -worker_exec(_WorkerRef, _Code) -> - ?NIF_STUB. - -%% @doc Get next item from a generator/iterator. -%% Returns {ok, Value} | {error, stop_iteration} | {error, Error} --spec worker_next(reference(), reference()) -> {ok, term()} | {error, term()}. -worker_next(_WorkerRef, _GeneratorRef) -> - ?NIF_STUB. - %%% ============================================================================ %%% Module Operations %%% ============================================================================ -%% @doc Import a Python module in a worker context. --spec import_module(reference(), binary()) -> {ok, reference()} | {error, term()}. -import_module(_WorkerRef, _ModuleName) -> - ?NIF_STUB. - -%% @doc Get an attribute from a Python object. --spec get_attr(reference(), reference(), binary()) -> {ok, term()} | {error, term()}. -get_attr(_WorkerRef, _ObjRef, _AttrName) -> - ?NIF_STUB. - %%% ============================================================================ %%% Info %%% ============================================================================ @@ -420,65 +323,10 @@ tracemalloc_stop() -> %%% Callback Support %%% ============================================================================ -%% @doc Set callback handler process for a worker. -%% Returns {ok, Fd} where Fd is the file descriptor for sending responses. --spec set_callback_handler(reference(), pid()) -> {ok, integer()} | {error, term()}. -set_callback_handler(_WorkerRef, _HandlerPid) -> - ?NIF_STUB. - -%% @doc Send a callback response to a worker via file descriptor. --spec send_callback_response(integer(), binary()) -> ok | {error, term()}. -send_callback_response(_Fd, _Response) -> - ?NIF_STUB. - -%% @doc Resume a suspended Python callback with the result. -%% StateRef is the reference returned in the {suspended, ...} tuple. -%% Result is the callback result as a binary (status byte + data). -%% Returns {ok, FinalResult}, {error, Reason}, or another {suspended, ...} for nested callbacks. --spec resume_callback(reference(), binary()) -> - {ok, term()} | {error, term()} | {suspended, term(), reference(), {binary(), term()}}. -resume_callback(_StateRef, _Result) -> - ?NIF_STUB. - %%% ============================================================================ %%% Async Worker Support %%% ============================================================================ -%% @doc Create a new async worker with background event loop. -%% Returns an opaque reference to be used with async functions. --spec async_worker_new() -> {ok, reference()} | {error, term()}. -async_worker_new() -> - ?NIF_STUB. - -%% @doc Destroy an async worker. --spec async_worker_destroy(reference()) -> ok. -async_worker_destroy(_WorkerRef) -> - ?NIF_STUB. - -%% @doc Submit an async call to the event loop. -%% Args: AsyncWorkerRef, Module, Func, Args, Kwargs, CallerPid -%% Returns: {ok, AsyncId} | {ok, {immediate, Result}} | {error, term()} --spec async_call(reference(), binary(), binary(), list(), map(), pid()) -> - {ok, non_neg_integer() | {immediate, term()}} | {error, term()}. -async_call(_WorkerRef, _Module, _Func, _Args, _Kwargs, _CallerPid) -> - ?NIF_STUB. - -%% @doc Execute multiple async calls concurrently using asyncio.gather. -%% Args: AsyncWorkerRef, CallsList (list of {Module, Func, Args}), CallerPid -%% Returns: {ok, AsyncId} | {ok, {immediate, Results}} | {error, term()} --spec async_gather(reference(), [{binary(), binary(), list()}], pid()) -> - {ok, non_neg_integer() | {immediate, list()}} | {error, term()}. -async_gather(_WorkerRef, _Calls, _CallerPid) -> - ?NIF_STUB. - -%% @doc Stream from an async generator. -%% Args: AsyncWorkerRef, Module, Func, Args, Kwargs, CallerPid -%% Returns: {ok, AsyncId} | {error, term()} --spec async_stream(reference(), binary(), binary(), list(), map(), pid()) -> - {ok, non_neg_integer()} | {error, term()}. -async_stream(_WorkerRef, _Module, _Func, _Args, _Kwargs, _CallerPid) -> - ?NIF_STUB. - %%% ============================================================================ %%% Sub-interpreter Support (Python 3.12+) %%% ============================================================================ @@ -958,18 +806,6 @@ stop_writer(_FdRef) -> start_writer(_FdRef) -> ?NIF_STUB. -%% @doc Cancel read monitoring (legacy alias for stop_reader). -%% Kept for backward compatibility. --spec cancel_reader(reference(), reference()) -> ok | {error, term()}. -cancel_reader(_LoopRef, _FdRef) -> - ?NIF_STUB. - -%% @doc Cancel write monitoring (legacy alias for stop_writer). -%% Kept for backward compatibility. --spec cancel_writer(reference(), reference()) -> ok | {error, term()}. -cancel_writer(_LoopRef, _FdRef) -> - ?NIF_STUB. - %% @doc Explicitly close an FD with proper lifecycle cleanup. %% Transfers ownership and triggers proper cleanup via ERL_NIF_SELECT_STOP. %% Safe to call multiple times (idempotent). @@ -1092,73 +928,6 @@ set_shared_worker(_WorkerPid) -> %%% Worker Pool %%% ============================================================================ -%% @doc Start the worker pool with the specified number of workers. -%% -%% Creates a pool of worker threads that process Python operations. -%% Each worker may have its own subinterpreter (Python 3.12+) for true -%% parallelism, or share the GIL with optimized batching. -%% -%% If NumWorkers is 0, the pool will use the number of CPU cores. -%% -%% @param NumWorkers Number of worker threads (0 = auto-detect) -%% @returns ok on success, or {error, Reason} --spec pool_start(non_neg_integer()) -> ok | {error, term()}. -pool_start(_NumWorkers) -> - ?NIF_STUB. - -%% @doc Stop the worker pool. -%% -%% Signals all workers to shut down and waits for them to terminate. -%% Any pending requests will receive {error, pool_shutdown}. -%% -%% @returns ok --spec pool_stop() -> ok. -pool_stop() -> - ?NIF_STUB. - -%% @doc Submit a request to the worker pool. -%% -%% Submits an asynchronous request to the pool. The caller will receive -%% a {py_response, RequestId, Result} message when the request completes. -%% -%% Request types and arguments: -%%
    -%%
  • `call' - Module, Func, Args, undefined (or Timeout)
    • -%%
  • `apply' - Module, Func, Args, Kwargs
    • -%%
  • `eval' - Code, Locals, undefined, undefined
    • -%%
  • `exec' - Code, undefined, undefined, undefined
    • -%%
  • `asgi' - Runner, Module, Callable, {Scope, Body}
    • -%%
  • `wsgi' - Module, Callable, Environ, undefined
    • -%%
-%% -%% @param Type Request type atom -%% @param Arg1 First argument (varies by type) -%% @param Arg2 Second argument (varies by type) -%% @param Arg3 Third argument (varies by type) -%% @param Arg4 Fourth argument (varies by type) -%% @returns {ok, RequestId} on success, or {error, Reason} --spec pool_submit(atom(), term(), term(), term(), term()) -> - {ok, non_neg_integer()} | {error, term()}. -pool_submit(_Type, _Arg1, _Arg2, _Arg3, _Arg4) -> - ?NIF_STUB. - -%% @doc Get worker pool statistics. -%% -%% Returns a map with the following keys: -%%
    -%%
  • `num_workers' - Number of worker threads
    • -%%
  • `initialized' - Whether the pool is started
    • -%%
  • `use_subinterpreters' - Whether using subinterpreters (Python 3.12+)
    • -%%
  • `free_threaded' - Whether using free-threaded Python (3.13+)
    • -%%
  • `pending_count' - Number of pending requests in queue
    • -%%
  • `total_enqueued' - Total requests submitted
    • -%%
-%% -%% @returns Stats map --spec pool_stats() -> map(). -pool_stats() -> - ?NIF_STUB. - %%% ============================================================================ %%% Process-per-context API (no mutex) %%% From 20705f776c7f58f365f42bb150e3d909755ad9dd Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sat, 29 Aug 2026 16:14:56 +0200 Subject: [PATCH 07/15] Document lock and ownership contracts on the shared structs (#76) The py_context_t header still described dirty-scheduler execution over a shared-GIL pool. It now says which thread owns each field group, what each mutex guards, the lock order, and why a stuck context is leaked rather than freed. The event loop struct gets the same treatment, and the Erlang modules that own a resource state in three lines what they own, who they talk to and what they never do. --- c_src/py_event_loop.h | 53 ++++++++++++++++++++--- c_src/py_nif.h | 91 +++++++++++++++++++++++++++++---------- src/py_buffer.erl | 4 ++ src/py_callback.erl | 6 +++ src/py_context.erl | 32 ++++++++------ src/py_context_router.erl | 4 ++ src/py_event_loop.erl | 7 +++ src/py_event_worker.erl | 7 +++ src/py_isolated.erl | 6 +++ src/py_shm.erl | 8 ++++ src/py_thread_handler.erl | 7 +++ 11 files changed, 183 insertions(+), 42 deletions(-) diff --git a/c_src/py_event_loop.h b/c_src/py_event_loop.h index e9e9413..d8adb75 100644 --- a/c_src/py_event_loop.h +++ b/c_src/py_event_loop.h @@ -259,13 +259,52 @@ typedef struct { /** * @struct erlang_event_loop_t - * @brief Main state for the Erlang-backed asyncio event loop - * - * This structure maintains all state needed for the event loop: - * - Reference to the Erlang worker process (scalable I/O model) - * - Reference to the Erlang router process (legacy) - * - Pending events queue - * - Synchronization primitives + * @brief State of one ErlangEventLoop (asyncio loop backed by enif_select) + * + * Three kinds of thread touch a loop: the loop thread (the context thread + * running `run_forever`, or a scheduler for main-interpreter loops driven by + * py_event_worker), scheduler threads running NIFs (`submit_task`, + * readiness and timer callbacks from py_event_worker), and the interpreter + * thread tearing the loop down. + * + * Lock and ownership contract: + * + * - mutex guards the pending event queue (pending_head/tail, + * pending_capacity, event_freelist, freelist_count, the pending_hash_* + * set), interp and external_attached, and event_cond. Never acquire a + * GIL while holding it: the loop thread can hold the GIL while waiting + * for mutex (loop_gil_acquire attaches under mutex, then takes the GIL + * after releasing it). + * + * - task_queue_mutex guards task_queue (an ErlNifIOQueue of serialized + * task tuples). Producers are scheduler threads in the submit NIFs; the + * consumer is process_ready_tasks on the loop thread, which holds the + * GIL while decoding. task_count and task_wake_pending are atomics used + * to coalesce wakeups. + * + * - env_pool_mutex guards env_pool and env_pool_count only. + * + * - namespaces_mutex guards namespaces_head and pid_env_head. Lock order: + * GIL first, then namespaces_mutex; the Python dicts in a namespace are + * touched only under the GIL. + * + * - py_loop, cached_* and callable_cache are Python objects owned by the + * loop thread and used only under its GIL. msg_env is allocated with the + * loop and freed in the destructor; the notification paths do not use + * it (each builds a local env per message so they need no lock). + * + * - worker_pid/has_worker, self_pid/has_self, loop_id and interp_id are + * set once at creation or by the setter NIFs before the loop runs, then + * read-only. router_pid/has_router are kept for layout compatibility + * only. shutdown is set once by the stop NIF. + * + * - interp becomes NULL in event_loop_detach_interpreter, which then waits + * for external_attached to drop to zero before Py_EndInterpreter; a + * scheduler that finds interp NULL must not attach. + * + * @see loop_gil_acquire + * @see process_ready_tasks + * @see event_loop_detach_interpreter */ typedef struct erlang_event_loop { /** @brief Legacy field - kept for binary compatibility */ diff --git a/c_src/py_nif.h b/c_src/py_nif.h index 7dd3b18..3b1fcf5 100644 --- a/c_src/py_nif.h +++ b/c_src/py_nif.h @@ -642,24 +642,72 @@ typedef struct { /** * @struct py_context_t - * @brief Process-owned Python context with shared-GIL subinterpreter pool - * - * A py_context_t is owned by a single Erlang process, which serializes - * all access to it. For subinterpreters, contexts reference a slot in - * the pre-created subinterpreter pool (shared GIL model). - * - * Execution happens directly on dirty schedulers using PyThreadState_Swap() - * to switch to the subinterpreter's thread state. This avoids: - * - Dedicated pthread per context - * - Mutex/condvar dispatch overhead - * - Term copying between environments - * - * @note Python 3.12+ uses shared-GIL subinterpreters via pool slots - * @note Older Python uses worker mode with main interpreter namespace + * @brief One Python execution environment served by one Erlang process + * + * A context has exactly one pthread that runs Python for it: the context + * thread (worker_context_thread_main for worker mode, + * owngil_context_thread_main for owngil mode). Erlang processes never run + * Python on a context; NIFs enqueue a ctx_request_t and return, the + * context thread dequeues, executes and replies with `{py_result, Id, R}` + * through msg_env. Isolated mode does not use this struct at all. + * + * Lock and ownership contract, by field group: + * + * - Identity and lifecycle (interp_id, is_subinterp, uses_worker_thread, + * uses_own_gil): written once by nif_context_create before the thread + * starts, read-only afterwards. destroyed, leaked, worker_running, + * shutdown_requested and init_error are atomics; any thread may read + * them, the writers are nif_context_destroy (destroyed, leaked), the + * shutdown helpers (shutdown_requested) and the context thread + * (worker_running, init_error). + * + * - Callback handler (has_callback_handler, callback_handler, + * callback_pipe): set by the owning Erlang process through + * nif_context_set_callback_handler before the first request; read by the + * context thread inside erlang_call_impl. The pipe is closed by + * nif_context_destroy only when the thread joined (`!leaked`): a stuck + * thread still reads the fds, and closing them would let the kernel hand + * the numbers to another file. + * + * - Request queue (queue_head, queue_tail, queue_not_empty): guarded by + * queue_mutex. Producers are NIFs on scheduler threads, the consumer is + * the context thread. Lock order: queue_mutex before req->mutex + * (ctx_queue_cancel_all takes both in that order); nothing takes + * queue_mutex while holding req->mutex, the GIL or interrupt_mutex. + * + * - msg_env: used only by the context thread, one message at a time + * (enif_clear_env, build, enif_send). Freed by the shutdown helper after + * the thread joined; never freed on the leak path. + * + * - Current request (shared_env, request_type, request_term, + * response_term, response_ok, reactor_buffer_ptr, local_env_ptr): a + * mirror of the ctx_request_t being executed, written and read by the + * context thread only, cleared after each request. No lock: no other + * thread may touch them. The execute functions take the context rather + * than the request, which is why the mirror exists. + * + * - Python state (globals, locals, module_cache, own_gil_tstate, + * own_gil_interp, thread_state, event_loop): created and destroyed on the + * context thread while it holds the GIL (the sub-interpreter's own GIL in + * owngil mode). Other threads may read own_gil_interp and event_loop as + * opaque pointers (nif_context_interrupt, nif_context_get_event_loop) + * but never dereference the Python objects. Refcounts belong to the + * context thread. + * + * - Interrupt (interrupt_mutex, exec_in_flight, exec_thread_id, + * interrupt_pending): see the invariant on interrupt_mutex below. It is + * the only lock a scheduler thread holds while acquiring a GIL, so it + * must never be taken by a thread that already holds one. + * + * Shutdown: nif_context_destroy marks destroyed, cancels the queue, wakes + * the thread and joins it with a timeout. If the thread does not exit the + * context is leaked on purpose (enif_keep_resource) rather than freed + * under a running pthread. * * @see nif_context_create - * @see nif_context_call - * @see subinterp_pool_alloc + * @see nif_context_call_async + * @see nif_context_interrupt + * @see nif_context_destroy */ struct py_context { /** @brief Unique interpreter ID for routing (0 = main, >0 = subinterp) */ @@ -683,7 +731,7 @@ struct py_context { /** @brief Pipe for callback responses [read, write] */ int callback_pipe[2]; - /* ========== Worker thread fields (used by both worker and owngil modes) ========== */ + /* ========== Context thread (worker and owngil modes) ========== */ /** @brief Dedicated pthread for this context */ pthread_t worker_thread; @@ -700,7 +748,7 @@ struct py_context { /** @brief True if thread initialization failed */ _Atomic bool init_error; - /* ========== Request queue (replaces single-slot pattern) ========== */ + /* ========== Request queue (queue_mutex) ========== */ /** @brief Mutex protecting the request queue */ pthread_mutex_t queue_mutex; @@ -717,10 +765,9 @@ struct py_context { /** @brief Environment for sending messages back to Erlang */ ErlNifEnv *msg_env; - /* ========== Legacy compatibility fields (populated from queue request) ========== */ - /* These fields are populated by the worker thread from the current request - * for compatibility with existing execute functions. They will be removed - * once all execute functions are refactored to use ctx_request_t directly. */ + /* ========== Current request (mirror of the ctx_request_t in flight) ========== */ + /* Written and cleared by the context thread around each request; the + * execute functions read the request from here. Context-thread only. */ /** @brief Shared env for current request (points to current req->request_env) */ ErlNifEnv *shared_env; diff --git a/src/py_buffer.erl b/src/py_buffer.erl index 829d703..17d7956 100644 --- a/src/py_buffer.erl +++ b/src/py_buffer.erl @@ -49,6 +49,10 @@ %%% process(line) %%% ''' %%% +%%% +%%% Owns: the native buffer resource (NIF) and the dispatch on handle shape. +%%% Talks to: `py_buffer.c' for native buffers, `py_shm' for `shared => true'. +%%% Never: reads on the Erlang side; Python is the only reader. %%% @end -module(py_buffer). diff --git a/src/py_callback.erl b/src/py_callback.erl index 0eed9ad..ce25fe2 100644 --- a/src/py_callback.erl +++ b/src/py_callback.erl @@ -18,6 +18,12 @@ %%% from Python code via the erlang.call() function. %%% %%% @private +%%% +%%% Owns: the ETS registry name to fun. +%%% Talks to: `py_context' and `py_thread_handler', which look functions up +%%% when Python calls `erlang.call'. +%%% Never: runs the function itself; the caller process does, with the context +%%% blocked on the callback pipe. -module(py_callback). -behaviour(gen_server). diff --git a/src/py_context.erl b/src/py_context.erl index d857b87..385dff4 100644 --- a/src/py_context.erl +++ b/src/py_context.erl @@ -12,23 +12,29 @@ %% See the License for the specific language governing permissions and %% limitations under the License. -%%% @doc Python context process. +%%% @doc The context process: one Python execution environment, one request +%%% at a time. %%% -%%% A py_context process owns a Python context (subinterpreter or worker). -%%% Each process has exclusive access to its context, eliminating mutex -%%% contention and enabling true N-way parallelism. +%%% Every mode goes through this module. In `worker' and `owngil' mode the +%%% process holds a NIF context resource, forwards each request to the +%%% context thread in C (`nif_context_call_async') and waits for its +%%% `{py_result, Ref, Result}'. In `isolated' mode `init/4' hands the +%%% process to `py_isolated', which speaks the same messages to a child OS +%%% process. Callers do not see the difference. %%% -%%% The context is created when the process starts and destroyed when it -%%% stops. All Python operations are serialized through message passing. +%%% == Callbacks == %%% -%%% == Callback Handling == +%%% When Python calls `erlang.call', the context thread blocks on the +%%% callback pipe and sends `{erlang_callback, Id, Fun, Args}' to this +%%% process, which runs the registered function and writes the reply frame +%%% back. Nested requests from the callback are served inline, so callbacks +%%% can call Python again to any depth. %%% -%%% When Python code calls `erlang.call()`, the NIF returns a `{suspended, ...}` -%%% tuple instead of blocking. The context process handles the callback inline -%%% using a recursive receive pattern, enabling arbitrarily deep callback nesting. -%%% -%%% This approach is inspired by PyO3's suspension mechanism and avoids the -%%% deadlock issues that occur with separate callback handler processes. +%%% Owns: the context resource, the request in flight, its timeout and +%%% the process-local envs (`py:call(Ctx, ...)'). +%%% Talks to: `py_nif' (context NIFs), `py_isolated', `py_callback', +%%% `py_context_sup'. +%%% Never: runs Python on a scheduler thread; the context thread does. %%% %%% @end -module(py_context). diff --git a/src/py_context_router.erl b/src/py_context_router.erl index 06e6e27..eb609e5 100644 --- a/src/py_context_router.erl +++ b/src/py_context_router.erl @@ -53,6 +53,10 @@ %%% ok = py_context_router:unbind_context(). %%% %%% +%%% +%%% Owns: the pool tables and the scheduler to context assignment. +%%% Talks to: `py_context' (creation, calls), `py_context_sup'. +%%% Never: executes Python; it picks a context and forwards. %%% @end -module(py_context_router). diff --git a/src/py_event_loop.erl b/src/py_event_loop.erl index 0121cff..c074e7f 100644 --- a/src/py_event_loop.erl +++ b/src/py_event_loop.erl @@ -19,6 +19,13 @@ %% and registers callback functions for Python to call. %% %% @private +%% +%% Owns: the lifecycle of main-interpreter loops and the `erlang.*' loop +%% callbacks Python needs. +%% Talks to: `py_event_worker' (one per loop), `py_event_loop_pool', the loop +%% NIFs. +%% Never: dispatches to owngil loops; those are reached through +%% `py_context:loop_ref/1'. -module(py_event_loop). -behaviour(gen_server). diff --git a/src/py_event_worker.erl b/src/py_event_worker.erl index da8b98e..20963d0 100644 --- a/src/py_event_worker.erl +++ b/src/py_event_worker.erl @@ -5,6 +5,13 @@ %% - Receives `{select, FdRes, Ref, ready_input|ready_output}' directly from enif_select %% - Handles `{timeout, TimerRef}' messages for timer dispatch %% - Manages timers via erlang:send_after to self() +%% +%% Owns: the readiness and timer messages of one loop, and its `task_ready' +%% coalescing. +%% Talks to: the `py_event_loop.c' NIFs (`process_ready_tasks', timers), +%% `py_event_worker_registry'. +%% Never: runs the loop itself in owngil mode (the context thread does) and +%% never blocks on Python. -module(py_event_worker). -behaviour(gen_server). diff --git a/src/py_isolated.erl b/src/py_isolated.erl index 4de2b7a..566c4f8 100644 --- a/src/py_isolated.erl +++ b/src/py_isolated.erl @@ -51,6 +51,12 @@ %%% Use `sys:get_state/1' to see the state and `sys:trace/2' for events. %%% %%% @private +%%% +%%% Owns: the child OS process, its Unix socket and the request in flight. +%%% Talks to: `py_context' (public API, the same messages as the embedded +%%% loop), `py_shm' (region handles crossing the socket), `py_callback' +%%% (registered functions the child calls). +%%% Never: runs Python in the VM, touches NIF resources other than `os_kill'. -module(py_isolated). -behaviour(gen_statem). diff --git a/src/py_shm.erl b/src/py_shm.erl index d14e86f..d9157e6 100644 --- a/src/py_shm.erl +++ b/src/py_shm.erl @@ -37,6 +37,14 @@ %%% a region used as a ring, with the write position and the closed flag in %%% a header page and flow control through the `_py_buffer_wait' and %%% `_py_buffer_consumed' callbacks the Python side calls. +%%% +%%% Owns: the region table (ETS), the backing files, and the ring state of +%%% shared buffers. +%%% Talks to: iommap (through `apply/3', optional dependency), `py_buffer' +%%% (shared variant), `py_callback' (registers `_py_buffer_wait', +%%% `_py_buffer_consumed', `_py_buffer_state'). +%%% Never: maps memory into Python itself; that is `_erlang_impl/_shm.py' in +%%% each interpreter. -module(py_shm). -behaviour(gen_server). diff --git a/src/py_thread_handler.erl b/src/py_thread_handler.erl index 3fe00a4..4455061 100644 --- a/src/py_thread_handler.erl +++ b/src/py_thread_handler.erl @@ -35,6 +35,13 @@ %%% 6. Python thread receives response and continues %%% %%% @private +%%% +%%% Owns: the coordinator process, one handler process and one pipe per Python +%%% thread. +%%% Talks to: `py_callback' (function lookup), the `py_thread_worker.c' side +%%% of the pipe. +%%% Never: touches a context: Python threads that call Erlang are not on a +%%% context thread. -module(py_thread_handler). -behaviour(gen_server). From 6aafcf6a02158b00e661fc31e2912718a8811d4b Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sat, 29 Aug 2026 17:20:24 +0200 Subject: [PATCH 08/15] Add the protocols and state machines pages (#77) Every message, frame and status code now has one page saying who sends it, what answers it and which files must change together, and every long-lived process and thread has its states, triggers and timers written down next to the invariants a change must keep. --- docs/architecture.md | 6 +- docs/protocols.md | 247 +++++++++++++++++++++++++++++++++++++++++ docs/state-machines.md | 191 +++++++++++++++++++++++++++++++ rebar.config | 4 + 4 files changed, 445 insertions(+), 3 deletions(-) create mode 100644 docs/protocols.md create mode 100644 docs/state-machines.md diff --git a/docs/architecture.md b/docs/architecture.md index 841a2a7..37588ae 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -127,9 +127,9 @@ In isolated mode there is one path: a status-3 frame on the socket, answered by a process the `py_isolated` state machine spawns; nested calls into the same context are dispatched immediately because they come from that process. -The frame format shared by the pipe and the socket, and the ETF conventions, -will get their own page (protocols); until then `c_src/py_convert.c` (type -mapping) and `priv/_erlang_impl/_etf.py` are the reference. +Every message and frame, with the rules for changing them, is in +[protocols](protocols.md); the states each process and thread moves through +are in [state machines](state-machines.md). ## asyncio diff --git a/docs/protocols.md b/docs/protocols.md new file mode 100644 index 0000000..3308a40 --- /dev/null +++ b/docs/protocols.md @@ -0,0 +1,247 @@ +# Protocols + +Every message and frame that crosses a boundary in erlang_python: between +an Erlang process and the context process, between the context thread in C +and Erlang, and between the VM and an isolated child. Read this when you +change a message shape, add a request kind, or debug a hang with +`sys:trace/2` or `strace`. The code is the reference; this page tells you +where each piece is and what must stay consistent. + +## Boundaries + +``` + caller process --(1) Erlang messages--> py_context / py_isolated process + | + embedded modes | isolated mode + (2) NIF request queue | (4) Unix socket frames + (3) callback pipe / thread pipe | + v + context thread in C child OS process +``` + +1. Erlang messages: `src/py_context.erl` (API side and embedded loop), + `src/py_isolated.erl` (isolated loop). +2. Request queue: `ctx_request_t` in `c_src/py_nif.h`, enqueued by + `nif_context_call_async`, answered with `{py_result, Ref, Result}`. +3. Callbacks: `erlang_call_impl` in `c_src/py_callback.c`, + `c_src/py_thread_worker.c`, `src/py_thread_handler.erl`. +4. Socket: `src/py_isolated.erl`, `priv/_erlang_impl/_isolated.py`, + `priv/_erlang_impl/_etf.py`. + +## 1. Caller to context process + +The public API (`py:call/3`, `py_context:call/4`, ...) is a plain message +plus a monitor. Both context loops accept the same messages, so a caller +never knows the mode. + +```erlang +%% py_context:submit/5 and friends +MRef = erlang:monitor(process, Ctx), +Ctx ! {call, self(), MRef, Module, Func, Args, Kwargs}, +%% reply +{MRef, Result} %% {ok, Term} | {error, Reason} +``` + +| Message | Reply | Notes | +|---|---|---| +| `{call, From, MRef, Module, Func, Args, Kwargs}` | `{ok, R}` / `{error, E}` | 8-tuple variant adds `EnvRef` for process-local envs | +| `{eval, From, MRef, Code, Locals}` | same | 6-tuple variant adds `EnvRef` | +| `{exec, From, MRef, Code}` | `ok` / `{error, E}` | 5-tuple variant adds `EnvRef` | +| `{submit, From, MRef, TaskRef, Module, Func, Args, Kwargs}` | `ok`, then `{async_result, TaskRef, Result}` to `From` | asyncio coroutine on the context loop | +| `{start_loop, From, MRef, Owner}` | `ok` / `{error, already_running}` | runs `run_forever` on the context thread; `Owner` is monitored and gets `{py_loop_exit, Ctx, Result}` when the loop ends | +| `{stop_loop, From, MRef, GraceMs}` | `ok` / `{error, no_loop}` | cooperative stop, interrupt after `GraceMs`, then kill | +| `{loop_ref, From, MRef}` | `{ok, LoopRef}` / `{error, no_loop}` | isolated: `{error, not_supported_in_isolated}` | +| `{interrupt, From, MRef}` | `ok` / `not_running` | | +| `{kill, From, MRef}` | `ok` | isolated only: SIGKILL, answered once the new child is up | +| `{pass_fd, From, MRef, Fd}` | `ok` / `{error, E}` | isolated only: SCM_RIGHTS | +| `{child_info, From, MRef}` | `{ok, Map}` | isolated only | +| `{stop, From, MRef}` | `ok` | | + +Timeouts are the caller's business: `await_reply/3` waits `Timeout`, then +sends `{interrupt_request, MRef}` so the request executing now is +interrupted, waits a short grace for the late reply, and returns +`{error, timeout}`. Loop control messages use `await_ctrl_reply/3`, which +must not interrupt (it would stop the loop it manages) and instead sends +`{cancel_ctrl, MRef}` so an isolated context drops the pending entry. + +Rules: + +- One request at a time per context. The embedded loop blocks in the + request; `py_isolated` postpones with `gen_statem` `postpone` while in + `{busy, Id}`. +- A request from a process that is running a callback for this context is + nested and served at once (both loops track those pids). +- Replies always go to `From`, tagged with `MRef`; a late reply after a + timeout is flushed by `demonitor(MRef, [flush])`. + +## 2. Request queue (embedded modes) + +`nif_context_call_async(Ctx, Kind, Data, RequestId)` allocates a +`ctx_request_t`, copies the terms into `request_env`, appends it under +`queue_mutex` and returns. The context thread dequeues, runs the request, +and sends: + +```erlang +{py_result, RequestId, Result} +``` + +to `caller_pid` through `msg_env`. Cancelled requests (context destroyed +while queued) are answered with `{py_result, Id, {error, cancelled}}`. +Request kinds are `ctx_request_type_t` in `c_src/py_nif.h`; the execute +functions read them from the request mirror on `py_context_t` +(`request_type`, `request_term`, ...). + +A result that needs a callback served before completion comes back as +`{suspended, CallbackId, StateRef, {Name, Args}}` in place of the final +result (see 3.1); the process resumes with `resume_callback/2` and waits +for the next `py_result`. + +## 3. Python calling Erlang + +`erlang.call(Name, Args...)` picks a path in `erlang_call_impl` +(`c_src/py_callback.c`); the comment above that function is authoritative. +Precedence: suspension when the calling thread is a context thread with +suspension enabled, then the context's blocking pipe when a handler is set, +then the thread worker for every other Python thread. + +### 3.1 Suspension + +The Python call raises `SuspensionRequired` and sets thread-local pending +state; the execute function sees the flag before the exception type and +returns the `{suspended, ...}` result above. `py_context` runs the fun +(`execute/2` in `py_callback`), serving nested requests meanwhile +(`wait_for_callback/2`), and calls `resume_callback(StateRef, Response)` +with the response body of 3.4. + +### 3.2 Context callback pipe + +Set up with `context_set_callback_handler(Ref, Pid)`. The context thread +sends: + +```erlang +{erlang_callback, CallbackId, FuncName, Args} +``` + +to the handler process and blocks (GIL released) reading the pipe with a +30 s timeout. The handler runs the fun and answers with +`context_write_callback_response(Ref, Body)`, which writes +`<>` on the pipe. There is no id on this pipe: +one context thread, one outstanding call. + +### 3.3 Thread worker pipe + +Any other Python thread (a `threading.Thread`, an executor worker) sends: + +```erlang +{thread_callback, WorkerId, CallbackId, FuncName, Args} +``` + +to the `py_thread_handler` coordinator, which gives each `WorkerId` a +handler process (`{thread_worker_spawn, WorkerId, WriteFd}`) and replies +with `thread_worker_write_response(Fd, CallbackId, Body)`. The frame on the +response pipe is: + +``` +<> +``` + +The reader discards frames whose id is not the one it waits for; a short +read or a timeout poisons the worker so the pipe is never read out of +phase. `erlang.async_call` uses the same frame on a per-interpreter +non-blocking pipe (`async_callback_pipe`, `{async_callback, CallbackId, +FuncName, Args, WriteFd}`), parsed incrementally by the reader tick. + +### 3.4 Response body + +Every path answers with the same body: + +``` +<<2, ETF/binary>> %% ok, term_to_binary(Result) +<<1, Message/binary>> %% error, UTF-8 text, raised as RuntimeError +``` + +Built in `handle_blocking_callback/3` (`py_context`), `py_thread_handler` +and the isolated loop; parsed by `parse_callback_response` in C and +`_isolated.py` in the child. Keep the three writers and two parsers in +step. + +## 4. Isolated socket + +The child (`priv/py_isolated_child.py`) connects to a Unix socket in the +private directory (`py_isolated:sock_dir/0`) and both sides exchange frames: + +``` +<> +Body = <> +``` + +| Status | Direction | Meaning | Payload | +|---|---|---|---| +| 0 | Erlang to child | request | `{call, M, F, Args, Kwargs}`, `{eval, Code, Locals}`, `{exec, Code}`, `{submit, TaskRef, ...}`, `start_loop`, `{pass_fd, ...}`, `ping`, `shutdown`, `{init, Opts}` | +| 1 | either | error reply | `{Class, Message, Traceback}` or an atom | +| 2 | either | ok reply | the result term | +| 3 | child to Erlang | request from Python | `{call, Name, Args}`, `{send, Pid, Msg}`, `{whereis, Name}` | +| 4 | child to Erlang | event, `Id` = 0 | `{ready, Info}`, `{startup_error, Problems}`, `{memory_limit, Rss}`, `{log, Level, Msg}`, `{async_result, TaskRef, R}`, `{loop_exit, R}` | +| 5 | Erlang to child | control, `Id` = 0 | `{interrupt, Target}`, `{cancel, Id}`, `stop_loop`, `{shm_close, Id}` | + +Ids: Erlang numbers its requests from 1 (`next_id`); the child numbers its +status-3 requests independently; replies carry the id of the request they +answer. Control and events use id 0 and never get a reply. + +ETF: `_etf.py` encodes what `py_convert.c` would produce for the same +Python value, with two differences worth knowing: `Pid`, `Ref` and `Port` +are opaque and keep their raw bytes, and the child can create atoms +(`binary_to_term/1` is not called with `safe`). Shared handles +(`{'$py_shm', Id, Path, Size}`, `{'$py_buffer', Id, Path, Ring}`) are plain +tuples on the wire and become `SharedMemory` / `SharedBuffer` on arrival +(`_shm.from_term`); NIF resources cannot cross and the API answers +`{error, not_supported_in_isolated}`. + +Handshake (`py_isolated:handshake/1`): the child sends `{ready, Info}`, +Erlang sends `{init, Opts}` (status 0, id 1) and the preload `exec`, +serving status-3 requests that arrive meanwhile with a bounded callback +runner. Anything else during the handshake fails the start. + +Interrupt: `{interrupt, Target}` is read by the child's reader thread, which +sends SIGUSR1 to the main thread when `Target` is the request executing +now (top of `_exec_stack`); the signal handler raises `_Interrupted` +(a `KeyboardInterrupt` subclass) only while a request runs. `{cancel, Id}` +marks a queued request so it is answered `{error, cancelled}` instead of +run. Erlang arms a SIGKILL backstop bound to the request id when it sends +an interrupt; a reply for that id cancels it. + +Flow control: the socket buffers are 1 MB each way; a frame larger than +that is written in pieces by both sides. Bulk data goes through shared +memory, not the socket (see [isolated](isolated.md)). + +## 5. Shared buffer ring + +`py_buffer:new(#{shared => true})` allocates a `py_shm` region used as a +ring. The first page is a header written by Erlang after each `write/2` +(write position, closed flag, ring size) and read by Python; the +`py_shm` server keeps the consumed position from the callbacks below, not +from the header. Notifications are callbacks, so they work in every mode: + +| Callback | Called by | Returns | +|---|---|---| +| `_py_buffer_wait(Id, ReadPos)` | reader, before blocking | `{WPos, Closed}` once `WPos > ReadPos` or the buffer is closed | +| `_py_buffer_consumed(Id, N)` | reader, after a read | `ok`; a writer blocked on space wakes up | +| `_py_buffer_state(Id)` | reader, on (re)map | `{WPos, Closed}` without waiting | + +All three answer `{error, closed}` for an unknown id. The header is only +read after a callback returned, so ordering follows the round trip and the +Python side needs no fence. + +## Changing a protocol + +- Add a request kind: the message in `py_context` (API and embedded loop), + the `ctx_request_type_t` and execute function in C, the `request/6` + clause in `py_isolated`, and `_on_request` in `_isolated.py`. +- Add a control: `send_frame(Child, 0, ?STATUS_CONTROL, Term)` in + `py_isolated` and `_on_control` in `_isolated.py`; controls must be safe + to handle on the reader thread. +- Add a child event: `self.event(...)` in `_isolated.py` and an + `{ok, {0, ?STATUS_EVENT, ...}}` clause in `drain_socket` or `handshake`. +- Change the response body: three writers and two parsers listed in 3.4. +- Never change the frame header: the child, the callback pipe reader and + the async pipe parser share it. diff --git a/docs/state-machines.md b/docs/state-machines.md new file mode 100644 index 0000000..871bab1 --- /dev/null +++ b/docs/state-machines.md @@ -0,0 +1,191 @@ +# State machines + +The states each long-lived thing in erlang_python moves through, what moves +it, and what is allowed in each state. Read this before changing a loop, +a shutdown path or a restart policy; the invariants at the end are the ones +a change must keep. Messages and frames are in [protocols](protocols.md). + +## The Python runtime (C) + +`g_runtime_state` in `c_src/py_nif.h`, moved with compare-and-swap so only +one thread wins each transition. + +``` +UNINIT --init--> INITING --ok--> RUNNING --finalize--> SHUTTING_DOWN --> STOPPED + | ^ + +--------------------- failure ------------------------+ +``` + +- `runtime_is_running()` gates every NIF that touches Python; a NIF that + finds `SHUTTING_DOWN` or `STOPPED` returns `not_running` or an error. +- `STOPPED` may re-enter `INITING`: the runtime can be finalized and + initialized again in one VM (the suites do this). + +## An embedded context (`py_context` process + context thread) + +Two cooperating machines: the Erlang process and the pthread in C. + +Context thread (`worker_context_thread_main`, `owngil_context_thread_main` +in `c_src/py_nif.c`): + +``` +starting --namespaces created--> waiting --dequeue--> executing --reply--> waiting + | | + +-- init_error -----> exited +-- shutdown_requested ----> exited +``` + +- `waiting` blocks on `queue_not_empty` under `queue_mutex`. +- `executing` is bracketed by `py_context_exec_enter` / `exec_leave` + (interrupt bookkeeping) around the GIL; the request mirror on + `py_context_t` is valid only here. +- `exited` sets `worker_running = false`; `nif_context_destroy` joins with a + timeout and, if the join fails, marks the context `leaked` and pins the + resource instead of freeing it. + +Erlang process (`loop/1` in `py_context`): + +``` +idle --{call|eval|exec|submit}--> in_request --{py_result}--> idle + | | + | +--{suspended, ...}--> in_callback --resume--> in_request + | + +--{start_loop}--> loop_running --{py_result, LoopReq}--> idle + | + +--{stop_loop, GraceMs}--> stopping --grace--> interrupt --deadline--> idle +``` + +- `in_request` is a blocking receive for the reply; nested callbacks + arrive as `{erlang_callback, ...}` (pipe) or `{suspended, ...}` and are + served inline, so the process never deadlocks with its own thread. +- In `loop_running` the loop request `LoopReq` occupies the thread: `call`, + `eval`, `exec` and `call_method` answer `{error, loop_running}`; `stop` + first stops the loop. The owner is monitored and its `DOWN` stops the loop. +- `stopping` arms `loop_stop_deadline` (cooperative) then + `loop_interrupt_deadline`; the loop exit is the `py_result` for `LoopReq`, + and the owner gets `{py_loop_exit, Ctx, Result}`. + +## A request (`ctx_request_t`) + +``` +created (refcount 1) --enqueue--> queued (2) --dequeue--> running --done--> completed + | | + +-- cancelled (destroy, timeout) -------+ + v + freed when refcount hits 0 +``` + +The queue and the caller each hold one reference; whoever releases last +frees. `cancelled` is checked by the thread before running, so a request +cancelled while queued is answered `{error, cancelled}` without touching +Python. + +## An isolated context (`py_isolated`, `gen_statem`) + +States are the `state()` type in `src/py_isolated.erl`; `sys:get_state/1` +shows the current one and `sys:trace/2` prints transitions. + +``` + start_child + handshake + | + v + +--------------> idle <----------------------------------+ + | | | + | main request | start_loop | + | v | + | {busy, Id} ---reply Id---> idle | + | | | + | | looping --stop_loop-----> stopping_loop + | | | (grace: interrupt, then kill) + | | +--loop_exit event--------+ + | | + +-- child exit / kill / socket error --> {restarting, Reason} --new child--> idle + | + +-- budget exhausted --> stop +``` + +Per state: + +| State | Main requests (`call`, `eval`, `exec`, `start_loop`) | Other requests | Timers armed | +|---|---|---|---| +| `idle` | dispatched, go to `{busy, Id}` (`start_loop` goes to `looping`) | dispatched | none | +| `{busy, Id}` | postponed, unless from a process running a callback for this context (nested, dispatched) | dispatched | `{timeout, kill}` bound to `Id` once an interrupt was sent | +| `looping` | `{error, loop_running}` | dispatched (`submit`, `pass_fd`, ...) | none | +| `stopping_loop` | postponed | dispatched | `state_timeout` for the interrupt, then `{timeout, kill}` bound to `loop` | +| `{restarting, R}` | postponed | postponed | `state_timeout` waiting for the port's `exit_status` | + +Transitions and their triggers: + +- `{busy, Id}` to `idle`: a status-1/2 frame with `Id`. Frames for other + ids (nested requests) do not change state. +- Anything to `{restarting, Reason}`: `{Port, {exit_status, S}}`, a socket + `abort`, a `{memory_limit, Rss}` event, `kill/1`, or a request the state + machine cannot deliver. In-flight requests, submitted tasks and a running + loop fail with `Reason` (`fail_pending/2`); postponed requests are kept + and served by the next child. +- `{restarting, _}` to `idle`: the port reported the exit and a new child + passed the handshake. `restart_allowed/1` counts restarts in + `restart_period`; over `max_restarts` (or with `restart => false`) the + process stops with `{child_exited, Reason}`. +- `looping` to `stopping_loop`: `stop_loop/2` or the owner's `DOWN`. + `stopping_loop` to `idle`: the `{loop_exit, R}` event. The interrupt + `state_timeout` and the kill backstop escalate if the loop does not exit. + +Interrupt timing: `interrupt/1` in `{busy, Id}` sends `{interrupt, Id}` and +arms `{{timeout, kill}, KillAfter, Id}`. The reply for `Id` cancels the +timer; if it fires, the child gets SIGKILL and the machine goes to +`{restarting, killed}`. An interrupt in any other state answers +`not_running`. + +## The child (`_isolated.py`) + +The main thread runs one request at a time but can nest: + +``` +idle --request--> executing [stack: Id1] + | + +-- erlang.call --> waiting for reply, serving nested requests [Id1, Id2] ... + | + +-- SIGUSR1 while running --> _Interrupted raised in the request on top +``` + +- `_exec_stack` holds the ids being executed, innermost last. An + `{interrupt, Target}` control is honoured only if `Target` is the top of + the stack; otherwise it is stale and dropped. The signal handler raises + only while `running` is true, so an interrupt between requests cannot + leak into the next one. +- The reader thread never runs Python code: it parses frames, resolves + waiters, and pushes requests and interrupts to the main thread's inbox. +- `broken` (EOF or a hard error on the socket) is terminal: the reader + calls `os._exit`, since nothing useful can happen in the process any more + and a main thread stuck in a C call must not keep it alive. Erlang sees + the port's `exit_status` and runs the restart policy above. +- With a loop: `start_loop` runs `run_forever` on the main thread; requests + that need the main thread are refused with `loop_running`, `submit` goes + through `call_soon_threadsafe`, and `stop_loop` calls `loop.stop()` from + the reader thread. + +## A shared region (`py_shm`) + +``` +new --> open --close/1 or owner DOWN--> closed (file unlinked, handle closed) +``` + +Mappings in Python outlive `closed` until the wrapper is closed or +collected; a later access raises `ValueError`, never a fault. A shared +buffer adds `closed = true` in its header at `py_buffer:close/1`, and +readers waiting in `_py_buffer_wait` are answered with the closed flag. + +## Invariants + +- A context executes one top-level request at a time, in every mode. + Embedded: one thread, one dequeue. Isolated: `{busy, Id}` plus `postpone`. +- Nested requests only come from a process serving a callback of the same + context. Anything else waits. +- A restart never loses a queued request, only the ones in flight, and + callers of those get an error naming the cause. +- Interrupts target the request executing now; a stale interrupt is + dropped on both sides (kill timer bound to the id in Erlang, stack check + in the child, `interrupt_pending` cleared in `exec_leave` for embedded + contexts). +- Shutdown never frees memory a thread may still use: embedded contexts + leak on a failed join, the child is reaped through the port. diff --git a/rebar.config b/rebar.config index d3ee4e1..6e23e77 100644 --- a/rebar.config +++ b/rebar.config @@ -79,6 +79,8 @@ <<"docs/architecture.md">>, <<"docs/code-map.md">>, <<"docs/glossary.md">>, + <<"docs/protocols.md">>, + <<"docs/state-machines.md">>, <<"docs/preload.md">>, <<"docs/owngil_internals.md">>, <<"docs/event_loop_architecture.md">> @@ -117,6 +119,8 @@ <<"docs/architecture.md">>, <<"docs/code-map.md">>, <<"docs/glossary.md">>, + <<"docs/protocols.md">>, + <<"docs/state-machines.md">>, <<"docs/preload.md">>, <<"docs/owngil_internals.md">>, <<"docs/event_loop_architecture.md">> From ca7f8a70276f2d52dedb14b8724898690d4e2a38 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sat, 29 Aug 2026 17:51:11 +0200 Subject: [PATCH 09/15] Wait on callback pipes with poll, not select (#78) select() is undefined for a descriptor above FD_SETSIZE, so a VM with more than 1024 open files could not bring up a thread worker and every thread callback after that failed with "Failed to spawn thread handler". The ready-wait also releases the GIL, and the coordinator logs a failed ready signal instead of leaving Python to time out. --- CHANGELOG.md | 5 ++++ c_src/py_nif.h | 24 ++++++------------ c_src/py_thread_worker.c | 8 ++++-- src/py_thread_handler.erl | 16 ++++++++---- test/py_reentrant_SUITE.erl | 34 +++++++++++++++++++++++++ test/py_test_high_fds.py | 50 +++++++++++++++++++++++++++++++++++++ 6 files changed, 114 insertions(+), 23 deletions(-) create mode 100644 test/py_test_high_fds.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 437a85c..8983353 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,11 @@ - `pthread_timedjoin_np` was called without `_GNU_SOURCE`, an implicit declaration on Linux that newer compilers reject. +- Callback pipes waited with `select()`, which is undefined for a file + descriptor above 1024: in a VM with many open files a thread callback + could time out with "Failed to spawn thread handler". The waits use + `poll()`, the handler ready-wait no longer holds the GIL, and + `py_thread_handler` logs a failed ready signal. ## 4.1.0 (2026-08-15) diff --git a/c_src/py_nif.h b/c_src/py_nif.h index 3b1fcf5..6832c40 100644 --- a/c_src/py_nif.h +++ b/c_src/py_nif.h @@ -64,7 +64,7 @@ #define NEED_DLOPEN_GLOBAL 1 #endif -#include +#include /** @} */ /* ============================================================================ @@ -1598,13 +1598,10 @@ static ssize_t read_with_timeout(int fd, void *buf, size_t count, int timeout_ms errno = ETIMEDOUT; return (ssize_t)got; } - struct timeval tv; - tv.tv_sec = remain_ms / 1000; - tv.tv_usec = (remain_ms % 1000) * 1000; - fd_set fds; - FD_ZERO(&fds); - FD_SET(fd, &fds); - int s = select(fd + 1, &fds, NULL, NULL, &tv); + /* poll, not select: select() is undefined for fd >= FD_SETSIZE + * (1024) and a VM with many open files gets pipe fds above it. */ + struct pollfd pfd = { .fd = fd, .events = POLLIN, .revents = 0 }; + int s = poll(&pfd, 1, (int)remain_ms); if (s < 0) { if (errno == EINTR) continue; return -1; @@ -1690,7 +1687,7 @@ typedef enum { * @brief Write exactly @p count bytes to a (typically non-blocking) fd * with a deadline. * - * Loops on partial writes / EINTR / EAGAIN. On EAGAIN, uses select() for + * Loops on partial writes / EINTR / EAGAIN. On EAGAIN, uses poll() for * write-readiness with the remaining deadline. Used by the thread-worker * write path to avoid pinning a dirty I/O scheduler thread on a stalled * Python reader. @@ -1741,13 +1738,8 @@ static write_result_t write_all_with_deadline(int fd, const void *buf, (deadline.tv_sec - now.tv_sec) * 1000L + (deadline.tv_nsec - now.tv_nsec) / 1000000L; if (remain_ms <= 0) return WRITE_TIMEOUT; - struct timeval tv; - tv.tv_sec = remain_ms / 1000; - tv.tv_usec = (remain_ms % 1000) * 1000; - fd_set fds; - FD_ZERO(&fds); - FD_SET(fd, &fds); - int s = select(fd + 1, NULL, &fds, NULL, &tv); + struct pollfd pfd = { .fd = fd, .events = POLLOUT, .revents = 0 }; + int s = poll(&pfd, 1, (int)remain_ms); if (s < 0) { if (errno == EINTR) continue; return WRITE_ERROR; diff --git a/c_src/py_thread_worker.c b/c_src/py_thread_worker.c index d8a3e37..31fde37 100644 --- a/c_src/py_thread_worker.c +++ b/c_src/py_thread_worker.c @@ -488,8 +488,12 @@ static int thread_worker_spawn_handler(thread_worker_t *tw) { * condition (Defect 5): both the byte count must match AND the * value must be zero. Short reads are not silently accepted. */ uint32_t response_len = 0; - ssize_t n = read_with_timeout(tw->response_pipe[0], &response_len, - sizeof(response_len), 10000); + ssize_t n; + /* The coordinator answers without Python; do not hold the GIL for it. */ + Py_BEGIN_ALLOW_THREADS + n = read_with_timeout(tw->response_pipe[0], &response_len, + sizeof(response_len), 10000); + Py_END_ALLOW_THREADS if (n != (ssize_t)sizeof(response_len) || response_len != 0) { return -1; } diff --git a/src/py_thread_handler.erl b/src/py_thread_handler.erl index 4455061..756e5e0 100644 --- a/src/py_thread_handler.erl +++ b/src/py_thread_handler.erl @@ -115,11 +115,17 @@ handle_info({thread_worker_spawn, WorkerId, WriteFd}, #state{handlers = Handlers HandlerPid = spawn_link(fun() -> handler_loop(WorkerId, WriteFd) end), %% Signal readiness to Python (write 0 length to indicate success) - py_nif:thread_worker_signal_ready(WriteFd), - - %% Store handler mapping - NewHandlers = Handlers#{WorkerId => {HandlerPid, WriteFd}}, - {noreply, State#state{handlers = NewHandlers}}; + case py_nif:thread_worker_signal_ready(WriteFd) of + ok -> + NewHandlers = Handlers#{WorkerId => {HandlerPid, WriteFd}}, + {noreply, State#state{handlers = NewHandlers}}; + {error, Reason} -> + %% The Python side times out and reports it; say why here + logger:error("py_thread_handler: ready signal for worker ~p (fd ~p) failed: ~p", + [WorkerId, WriteFd, Reason]), + HandlerPid ! shutdown, + {noreply, State} + end; %% Handle callback request from Python thread handle_info({thread_callback, WorkerId, CallbackId, FuncName, Args}, diff --git a/test/py_reentrant_SUITE.erl b/test/py_reentrant_SUITE.erl index 4eb6f18..77f447a 100644 --- a/test/py_reentrant_SUITE.erl +++ b/test/py_reentrant_SUITE.erl @@ -22,6 +22,7 @@ test_callback_with_complex_types/1, test_multiple_sequential_callbacks/1, test_call_from_non_worker_thread/1, + test_thread_callback_fd_above_fd_setsize/1, test_callback_with_try_except/1, test_async_call/1, test_callback_name_registry/1, @@ -38,6 +39,7 @@ all() -> test_callback_with_complex_types, test_multiple_sequential_callbacks, test_call_from_non_worker_thread, + test_thread_callback_fd_above_fd_setsize, test_callback_with_try_except, test_async_call, test_callback_name_registry, @@ -133,6 +135,12 @@ test_etf_decode_safe(_Config) -> %% Negative: many DISTINCT brand-new atoms wrapped in marker-shaped binaries %% must all come back verbatim, never decoded into atoms. + %% Warm up first so modules loaded on first use (base64, the callback + %% path) do not count as atoms minted by the round trips below. + Warm = etf_marker(novel_atom_etf("zzqx_etf_safe_warmup")), + py:register_function(etf_probe_novel, fun(_) -> Warm end), + {ok, Warm} = py:eval(<<"__import__('erlang').call('etf_probe_novel', [])">>), + assert_atom_absent("zzqx_etf_safe_warmup"), Before = erlang:system_info(atom_count), N = 50, lists:foreach( @@ -349,6 +357,32 @@ test_call_from_non_worker_thread(_Config) -> py:unregister_function(simple_add), ok. +%% @doc Thread callbacks must work when the response pipe lands on an fd +%% above FD_SETSIZE: select() is undefined there and used to make the +%% handler ready-wait time out after 10 s ("Failed to spawn thread handler"). +test_thread_callback_fd_above_fd_setsize(_Config) -> + py:register_function(high_fd_add, fun([A, B]) -> A + B end), + TestDir = filename:join(code:lib_dir(erlang_python), "test"), + ok = py:exec(iolist_to_binary(io_lib:format( + "import sys; sys.path.insert(0, '~s')", [TestDir]))), + try + case py:call(py_test_high_fds, prepare, [1200]) of + {ok, Last} when Last >= 1200 -> + ct:log("highest fd opened: ~p", [Last]), + N = 8, + {ok, Results} = py:call(py_test_high_fds, call_from_threads, [N]), + Expected = [I + 1 || I <- lists:seq(0, N - 1)], + Expected = Results; + {ok, -1} -> + {skip, "fd hard limit too low to open 1200 files"}; + Other -> + ct:fail({prepare_failed, Other}) + end + after + _ = py:call(py_test_high_fds, cleanup, []), + py:unregister_function(high_fd_add) + end. + %% @doc Test that erlang.call() works even when wrapped in try/except blocks. %% This simulates ASGI/WSGI middleware that catches all exceptions. %% The flag-based detection should work even when the SuspensionRequired diff --git a/test/py_test_high_fds.py b/test/py_test_high_fds.py new file mode 100644 index 0000000..ef386fe --- /dev/null +++ b/test/py_test_high_fds.py @@ -0,0 +1,50 @@ +"""Thread callbacks with pipe fds above FD_SETSIZE. + +Opens enough files to push every fd the runtime creates afterwards above +1024, then has several new threads call Erlang at once so fresh thread +workers (and their pipes) are created in that range. select() cannot +watch such fds; poll() can. +""" +import concurrent.futures +import os +import resource + +_kept = [] + + +def prepare(target=1200): + """Raise the fd soft limit and fill descriptors up to `target`. + + Returns the highest fd opened, or -1 if the hard limit is too low. + """ + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + want = target + 256 + if hard != resource.RLIM_INFINITY and hard < want: + return -1 + if soft < want: + resource.setrlimit(resource.RLIMIT_NOFILE, (want, hard)) + last = -1 + while last < target: + fd = os.open(os.devnull, os.O_RDONLY) + _kept.append(fd) + last = fd + return last + + +def call_from_threads(n): + import erlang + with concurrent.futures.ThreadPoolExecutor(max_workers=n) as ex: + futures = [ex.submit(erlang.call, 'high_fd_add', i, 1) for i in range(n)] + results = [] + for f in futures: + try: + results.append(f.result()) + except Exception as exc: + results.append('error: %s' % exc) + return results + + +def cleanup(): + while _kept: + os.close(_kept.pop()) + return 'ok' From 8e42d42ca58b493155d4d040386d80a086f4ac04 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sat, 29 Aug 2026 18:21:20 +0200 Subject: [PATCH 10/15] Add the contributing guide (#79) Build, test, the FreeBSD VM, and one recipe per kind of change (a NIF, an erlang.* function, a context option, a suite, a guide) listing every file that change touches, so a contributor can finish a change without reading the whole tree first. --- README.md | 4 + docs/contributing.md | 210 +++++++++++++++++++++++++++++++++++++++++++ rebar.config | 2 + 3 files changed, 216 insertions(+) create mode 100644 docs/contributing.md diff --git a/README.md b/README.md index 6045ac8..ca06e00 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,10 @@ Key features: rebar3 compile ``` +To work on erlang_python itself (tests, CI matrix, FreeBSD VM, how to add +a NIF, an `erlang.*` function, an option, a suite or a guide), see +[docs/contributing.md](docs/contributing.md). + ## Quick Start ### Erlang diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 0000000..825c7e7 --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,210 @@ +# Contributing + +How to build, test and extend erlang_python. Read this before your first +change; each recipe lists every file a change of that kind touches, so +nothing is left half done. Design background is in +[architecture](architecture.md), file ownership in [code map](code-map.md), +message shapes in [protocols](protocols.md). + +## Build + +You need OTP 27 or later, Python 3.12 or later with headers, CMake and a C +compiler. + +```sh +rebar3 compile # runs do_cmake.sh / do_build.sh, builds priv/py_nif.so +PYTHON_CONFIG=python3.13-config rebar3 compile # pick an interpreter +``` + +Notes: + +- `c_src/py_nif.c` includes the other `.c` files; there is no per-file + compile. A change in any `c_src` file rebuilds the one NIF. +- The interpreter used at build time is the one embedded at run time. + `python3-config` on `PATH` is the default. +- `rm -rf _build` when switching Python versions or build flags; CMake + caches the interpreter. + +## Test + +```sh +rebar3 ct --readable=compact # everything +rebar3 ct --suite test/py_isolated_SUITE # one suite +rebar3 ct --suite test/py_context_SUITE --case test_call # one case +rebar3 dialyzer && rebar3 xref # required before a PR +make lint-docs # snippets in README and docs/ +``` + +Notes: + +- `test/test.config` turns memory limits on for the whole run; it is + applied automatically. +- The `test` profile pulls `iommap` for the shared memory suites; users + add it to their own deps. +- Free-threaded Python: build and run with `PYTHON_GIL=0` and a `3.13t` + interpreter; cases that need the GIL skip themselves. +- ASan: configure CMake by hand, then let rebar3 pick the objects up: + +```sh +rm -rf _build && mkdir -p _build/cmake && cd _build/cmake +cmake ../../c_src -DENABLE_ASAN=ON -DENABLE_UBSAN=ON && cmake --build . && cd ../.. +rebar3 compile +LD_PRELOAD=$(gcc -print-file-name=libasan.so) ASAN_OPTIONS=detect_leaks=0 rebar3 ct +``` + +CI runs the matrix in `.github/workflows/ci.yml`: OTP 27 to 29 with Python +3.12 to 3.14 on Ubuntu and macOS, FreeBSD 14 through `vmactions/freebsd-vm`, +free-threaded 3.13t, and ASan. A PR is merged when all of them are green. + +### FreeBSD by hand + +CI covers FreeBSD, but the isolated mode (procctl, no cgroups, no +`/dev/shm`) is easier to debug in a local VM. On an Apple Silicon Mac: + +```sh +# 1. image (arm64 on Apple Silicon, amd64 elsewhere) +curl -O https://download.freebsd.org/releases/VM-IMAGES/14.1-RELEASE/aarch64/Latest/FreeBSD-14.1-RELEASE-arm64-aarch64.qcow2.xz +xz -d FreeBSD-14.1-RELEASE-arm64-aarch64.qcow2.xz +qemu-img resize FreeBSD-14.1-RELEASE-arm64-aarch64.qcow2 +20G + +# 2. boot headless, ssh on host port 2222 +qemu-system-aarch64 -M virt -accel hvf -cpu host -m 4096 -smp 4 \ + -bios /opt/homebrew/share/qemu/edk2-aarch64-code.fd \ + -drive file=FreeBSD-14.1-RELEASE-arm64-aarch64.qcow2,if=virtio,format=qcow2 \ + -netdev user,id=n0,hostfwd=tcp::2222-:22 -device virtio-net-pci,netdev=n0 \ + -nographic -serial mon:stdio +``` + +On the console, log in as `root` (no password), set one, and enable ssh: + +```sh +passwd +sysrc sshd_enable=YES +sed -i '' 's/^#PermitRootLogin no/PermitRootLogin yes/' /etc/ssh/sshd_config +service sshd start +pkg install -y erlang-runtime28 python313 py313-numpy cmake gmake git +fetch -o /root/rebar3 https://github.com/erlang/rebar3/releases/download/3.25.0/rebar3 +chmod +x /root/rebar3 +``` + +Then from the host, ship the tree and run: + +```sh +git archive --format=tgz -o /tmp/ep.tgz HEAD +scp -P 2222 /tmp/ep.tgz root@127.0.0.1:/root/ +ssh -p 2222 root@127.0.0.1 'rm -rf ep && mkdir ep && tar -C ep -xzf ep.tgz && cd ep \ + && export PATH=/usr/local/lib/erlang28/bin:$PATH PYTHON_CONFIG=python3.13-config \ + && /root/rebar3 compile && /root/rebar3 ct --readable=compact' +``` + +`erlang-runtime28` installs outside the default `PATH`; `pkg info -l +erlang-runtime28 | grep bin/erl` shows where. Use `-accel kvm` and the +amd64 image on Linux hosts. + +## Recipes + +Every recipe ends with the same three steps: a test case, `rebar3 +dialyzer && rebar3 xref`, and an entry in `CHANGELOG.md` under the +unreleased version. + +### Add a NIF + +1. Implement `static ERL_NIF_TERM nif_x(ErlNifEnv*, int, const ERL_NIF_TERM[])` + in the `c_src` file that owns the area (see `c_src/README.md`). +2. Add `{"x", Arity, nif_x, Flags}` to `nif_funcs[]` at the end of + `c_src/py_nif.c`. `Flags` is `ERL_NIF_DIRTY_JOB_CPU_BOUND` or + `ERL_NIF_DIRTY_JOB_IO_BOUND` when the NIF can block or run Python, `0` + otherwise. +3. Add the stub, its `-spec` and a `@doc` to `src/py_nif.erl`, and the + export. +4. Follow the rules in `c_src/README.md`: only the context thread touches + a context's Python objects; release the GIL around every blocking + wait; respect the lock order on `py_context_t`. +5. Test it from a suite through the Erlang API that uses it, not through + `py_nif` directly, unless it is a test helper. + +### Add an `erlang.*` function for Python + +The `erlang` module has three implementations that must agree: + +1. Embedded modes, C: add `erlang_x_impl` and its entry in the method + table of `c_src/py_callback.c` (`PyMethodDef erlang_methods`). Names + starting with `_` are internal helpers for the Python package. +2. Embedded modes, Python: if the function is written in Python, add it to + `priv/_erlang_impl/__init__.py` and to `__all__`; the C module copies + it onto `erlang` at import (see the bootstrap code near the end of + `py_callback.c`). +3. Isolated mode: add it to `install_erlang_module` in + `priv/_erlang_impl/_isolated.py`. Anything the child cannot support + goes through `_not_supported(name)` so the user gets a clear + `RuntimeError`, never a silent difference. +4. Document it in the guide of its area and in `README.md` (API + reference section), and add the row to `test/coverage_audit.md`. +5. Test it in a cross-mode suite so it runs in `worker` and `isolated` + groups. + +### Add a context option + +Options arrive as the map given to `py_context:new/1`. Pool contexts are +started by `py_context_sup:start_context/2` with the mode only, so an +option that must apply pool-wide also needs `py_context_router` and the +supervisor to carry it. + +1. Embedded modes: read it where the context is set up in + `src/py_context.erl` (`init/4` and the helpers it calls; `memory_limit`, + `preload`, `owner` and `start_timeout` are the existing examples). + If the C side needs it, pass it to `nif_context_create` and extend the + options parsing there. +2. Isolated mode: read it in `src/py_isolated.erl` (`start_child/1` + assembles the child command line and environment; `rlimits`, `cgroup`, + `python`, `restart`, `max_restarts`, `kill_after` are the examples) and, + if the child must know, add it to the `{init, Opts}` request handled by + `_init` in `priv/_erlang_impl/_isolated.py`. +3. If the option makes no sense in one mode, reject it there with an + error that names the option rather than ignoring it. +4. Document it in the options table of the relevant guide + (`docs/isolated.md`, `docs/workers.md`, `docs/memory.md`) and test both + the effect and the rejection. + +### Add a test suite + +1. Name it `test/py__SUITE.erl`; Python helpers go in + `test/py_test_.py` and are imported after + `sys.path.insert(0, TestDir)` (see `py_reentrant_SUITE` for the + pattern). +2. Start and stop the application in `init_per_suite` / + `end_per_suite`; the default pool is `py:start_contexts/0`. +3. Behaviour every mode must share runs in groups: `groups/0` with + `worker` and `isolated`, contexts created with + `py_context:new(#{mode => Mode})` from `init_per_group` + (`py_isolated_SUITE` shows the layout). +4. Skip, do not fail, when a platform or interpreter cannot run a case: + `{skip, Reason}` with the reason a human can act on. +5. Add the suite to the table in `docs/code-map.md` and the cases that + cover a documented API to `test/coverage_audit.md`. +6. Cases that measure time or memory print their numbers with `ct:pal` + and assert only on invariants, never on absolute timings. + +### Add or change a guide + +1. Create `docs/.md` in the task-oriented form: one paragraph on + what it is and when you need it, then steps with code, then short + notes. Second person, Erlang snippets, no hype. +2. Every Erlang snippet must call real exports at the right arity and + every Python snippet must parse: `make lint-docs` checks both. + `` above a fence exempts it; say why in the prose. +3. Register the page in both `extras` lists of `rebar.config` (the flat + list and the grouped one) so `rebar3 ex_doc` builds it, and link it + from the guide or README section that leads to it. +4. If the page documents a new API, add its rows to + `test/coverage_audit.md`. + +## Before opening a pull request + +- `rebar3 ct`, `rebar3 dialyzer`, `rebar3 xref`, `make lint-docs` all + clean locally. +- `CHANGELOG.md` updated under the unreleased version: `Added`, `Changed`, + `Removed` or `Fixed`. Removing a public function is a major version. +- The PR text says what the change intends and which path it takes; the + diff already lists the files. +- One squashed commit per PR. diff --git a/rebar.config b/rebar.config index 6e23e77..7be76e5 100644 --- a/rebar.config +++ b/rebar.config @@ -81,6 +81,7 @@ <<"docs/glossary.md">>, <<"docs/protocols.md">>, <<"docs/state-machines.md">>, + <<"docs/contributing.md">>, <<"docs/preload.md">>, <<"docs/owngil_internals.md">>, <<"docs/event_loop_architecture.md">> @@ -121,6 +122,7 @@ <<"docs/glossary.md">>, <<"docs/protocols.md">>, <<"docs/state-machines.md">>, + <<"docs/contributing.md">>, <<"docs/preload.md">>, <<"docs/owngil_internals.md">>, <<"docs/event_loop_architecture.md">> From 1fb871461d0ca76209dfba70d3df561f8d32922a Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sat, 29 Aug 2026 18:41:19 +0200 Subject: [PATCH 11/15] Add the decision records (#81) Eight records, one per design decision that shapes the code today, each with the situation, what was decided, what it costs and where the code is, so a change that reverses one is made knowingly. --- docs/architecture.md | 3 +- docs/contributing.md | 2 + docs/decisions/0001-one-thread-per-context.md | 36 ++++++++++++++++ .../decisions/0002-callback-delivery-paths.md | 43 +++++++++++++++++++ .../decisions/0003-callback-results-as-etf.md | 32 ++++++++++++++ .../0004-isolated-mode-child-process.md | 40 +++++++++++++++++ docs/decisions/0005-py-isolated-gen-statem.md | 39 +++++++++++++++++ .../0006-shared-memory-over-iommap.md | 42 ++++++++++++++++++ .../0007-remove-legacy-execution-paths.md | 31 +++++++++++++ docs/decisions/0008-pipe-io-rules.md | 31 +++++++++++++ docs/decisions/overview.md | 18 ++++++++ rebar.config | 20 +++++++++ 12 files changed, 336 insertions(+), 1 deletion(-) create mode 100644 docs/decisions/0001-one-thread-per-context.md create mode 100644 docs/decisions/0002-callback-delivery-paths.md create mode 100644 docs/decisions/0003-callback-results-as-etf.md create mode 100644 docs/decisions/0004-isolated-mode-child-process.md create mode 100644 docs/decisions/0005-py-isolated-gen-statem.md create mode 100644 docs/decisions/0006-shared-memory-over-iommap.md create mode 100644 docs/decisions/0007-remove-legacy-execution-paths.md create mode 100644 docs/decisions/0008-pipe-io-rules.md create mode 100644 docs/decisions/overview.md diff --git a/docs/architecture.md b/docs/architecture.md index 37588ae..7c30bf3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -129,7 +129,8 @@ same context are dispatched immediately because they come from that process. Every message and frame, with the rules for changing them, is in [protocols](protocols.md); the states each process and thread moves through -are in [state machines](state-machines.md). +are in [state machines](state-machines.md); the reasons behind the design +are in the [decision records](decisions/overview.md). ## asyncio diff --git a/docs/contributing.md b/docs/contributing.md index 825c7e7..52c1226 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -205,6 +205,8 @@ supervisor to carry it. clean locally. - `CHANGELOG.md` updated under the unreleased version: `Added`, `Changed`, `Removed` or `Fixed`. Removing a public function is a major version. +- A change that reverses or extends a decision in `docs/decisions/` gets a + new record there; the old one is not edited. - The PR text says what the change intends and which path it takes; the diff already lists the files. - One squashed commit per PR. diff --git a/docs/decisions/0001-one-thread-per-context.md b/docs/decisions/0001-one-thread-per-context.md new file mode 100644 index 0000000..33fe2bc --- /dev/null +++ b/docs/decisions/0001-one-thread-per-context.md @@ -0,0 +1,36 @@ +# 0001: One pthread per context, NIFs only enqueue + +Since 3.0.0. Code: `worker_context_thread_main`, `owngil_context_thread_main`, +the request queue on `py_context_t` (`c_src/py_nif.c`, `c_src/py_nif.h`). + +## Situation + +Before 3.0.0 Python ran on dirty schedulers: a NIF took the GIL, swapped +in the context's thread state and executed the call. That pinned a dirty +scheduler for the whole call with a 30 s cap, gave numpy, torch and +tensorflow a different OS thread on every call (they keep per-thread +state and some refuse to work across threads), and needed a single-slot +request buffer that raced with concurrent callers. + +## Decision + +Every context owns one pthread that runs all of its Python. NIFs called +from Erlang processes allocate a `ctx_request_t`, append it to the +context's queue and return at once; the thread dequeues, executes with +the GIL, and sends `{py_result, Ref, Result}` back. The same thread +serves worker mode (main interpreter, shared GIL) and owngil mode (its +own interpreter and GIL). + +## Consequences + +- Stable thread affinity: a context's Python always runs on the same OS + thread, and no dirty scheduler is held during a call. +- Erlang processes never touch a context's Python objects; the lock and + ownership contract on `py_context_t` follows from this. +- Shutdown must join the thread. A thread stuck in a C call cannot be + joined, so the context is leaked on purpose rather than freed under a + running thread. +- Interrupts have to reach the thread from outside: `interrupt_mutex`, + `exec_thread_id` and `PyThreadState_SetAsyncExc`. +- One request at a time per context is a property of the design, not a + limitation to work around; parallelism comes from more contexts. diff --git a/docs/decisions/0002-callback-delivery-paths.md b/docs/decisions/0002-callback-delivery-paths.md new file mode 100644 index 0000000..db44aac --- /dev/null +++ b/docs/decisions/0002-callback-delivery-paths.md @@ -0,0 +1,43 @@ +# 0002: Three callback delivery paths, chosen by the calling thread + +Since 3.0.0. Code: `erlang_call_impl` in `c_src/py_callback.c`, +`c_src/py_thread_worker.c`, `src/py_thread_handler.erl`, the callback +handling in `src/py_context.erl`. + +## Situation + +`erlang.call` must work from the context thread, from a Python thread the +user started (executors, `threading.Thread`), and while the context +thread is already waiting for Erlang (nested calls). A single blocking +handler process deadlocks as soon as the callback calls Python again on +the same context. + +## Decision + +The path is chosen by who is calling: + +1. Suspension, on a context thread with suspension enabled: the call + raises `SuspensionRequired`, the context returns + `{suspended, ...}` to its process, which runs the function, serves + nested requests meanwhile, and resumes. Inspired by PyO3's model. +2. The context callback pipe, on a context thread with a handler + registered: the thread blocks on a pipe while the handler process + runs the function. +3. The thread worker, for every other Python thread: a per-thread worker + with its own pipe and handler process, coordinated by + `py_thread_handler`. + +All three answer with the same response body (`<<2, ETF>>` or +`<<1, Message>>`). + +## Consequences + +- Nested callbacks of any depth work on the suspension path because the + Erlang process is never blocked in a NIF while it waits. +- Three writers and two parsers of the response body must stay in step; + the protocols page lists them. +- A Python thread that calls Erlang is not a context thread and cannot + reach the calling context's Python objects; re-entrant calls into the + same owngil context from a spawned thread are not supported. +- Adding a mode means adding a path (isolated mode has a fourth: the + socket frame), not extending one of these. diff --git a/docs/decisions/0003-callback-results-as-etf.md b/docs/decisions/0003-callback-results-as-etf.md new file mode 100644 index 0000000..9e812bc --- /dev/null +++ b/docs/decisions/0003-callback-results-as-etf.md @@ -0,0 +1,32 @@ +# 0003: Callback results cross as external term format + +Since 4.0.0. Code: `handle_blocking_callback/3` and friends in +`src/py_context.erl`, `parse_callback_response` in `c_src/py_callback.c`, +`priv/_erlang_impl/_etf.py` for the child. + +## Situation + +Results of an Erlang callback used to reach Python as the Python repr of +the term, parsed with `ast.literal_eval`. Binaries with backslashes, +quotes or newlines produced unparseable literals and were handed over as +raw text, `[]` arrived as `''`, floats lost precision, and pids and +references went through a base64 marker that had to be decoded with an +unsafe `binary_to_term`. + +## Decision + +Results are encoded with `term_to_binary` and decoded by the same +`term_to_py` converter that call arguments use, with +`ERL_NIF_BIN2TERM_SAFE`. Pids and references cross as native objects. +The one visible change is accepted as breaking: an Erlang string +(`"abc"`) reaches Python as `[97, 98, 99]`, exactly as it does for +arguments; return a binary for a `str`. + +## Consequences + +- One type mapping in both directions, documented once + (`docs/type-conversion.md`, `c_src/py_convert.c`). +- The `__etf__:` marker is data, never re-interpreted; + `test_etf_decode_safe` guards that no atoms are minted. +- The child needs a Python ETF codec (`_etf.py`) that produces the same + terms as the C converter; keeping them in step is a maintenance duty. diff --git a/docs/decisions/0004-isolated-mode-child-process.md b/docs/decisions/0004-isolated-mode-child-process.md new file mode 100644 index 0000000..7d281bb --- /dev/null +++ b/docs/decisions/0004-isolated-mode-child-process.md @@ -0,0 +1,40 @@ +# 0004: Isolation is a child OS process over a Unix socket + +Since 5.0.0. Code: `src/py_isolated.erl`, `priv/py_isolated_child.py`, +`priv/_erlang_impl/_isolated.py`. + +## Situation + +Embedded modes cannot stop a Python call stuck in C, cannot bound its +memory or CPU, and a segfault in an extension kills the node. Running +untrusted or unknown Python code needs those guarantees, with the public +API unchanged so a pool can mix modes. + +## Decision + +An isolated context is one child process per context, started with +`open_port` so the VM reaps it, talking to its `py_context` process over +a Unix socket with the frame format of the callback pipe +(`<>`, body `<>`). Interrupts are a +signal to the child's main thread with a `SIGKILL` backstop; limits are +rlimits, cgroups v2 on Linux and an RSS watchdog on macOS; a crash +restarts the child within a budget. The child uses the standard asyncio +loop; there is no C code of its own in the VM beyond `os_kill`. + +Not chosen: a seccomp or Capsicum sandbox (a later hardening step, the +process boundary is the first one), a NIF-side sub-process pool, and a +new wire protocol (the existing frame and ETF conventions are enough). + +## Consequences + +- Everything crossing the socket is a term. NIF resources (channels, + native buffers, object references) do not cross; the API says so with + `{error, not_supported_in_isolated}`. +- The child can create atoms (`binary_to_term` is not called with + `safe` because handles and control terms need atoms); untrusted code + must not mint unbounded distinct atoms. +- Each call copies arguments and results through the socket; bulk data + needs shared memory (0006). +- Python state is lost on restart; the context stays usable. +- Platform code lives in the child launcher: parent-death signal + (`prctl` on Linux, `procctl` on FreeBSD), memory limits per OS. diff --git a/docs/decisions/0005-py-isolated-gen-statem.md b/docs/decisions/0005-py-isolated-gen-statem.md new file mode 100644 index 0000000..f8d838d --- /dev/null +++ b/docs/decisions/0005-py-isolated-gen-statem.md @@ -0,0 +1,39 @@ +# 0005: The isolated context process is a gen_statem + +Since 5.0.0. Code: `src/py_isolated.erl`, entered from `init/4` in `py_context` +with `gen_statem:enter_loop/5`. + +## Situation + +The embedded context process is a hand-written receive loop in +`py_context`. The isolated process has more to track: a request in +flight, requests to hold while the child restarts, a running loop with a +grace period, an interrupt with a kill backstop bound to one request id, +and callback processes whose nested requests must pass while others +wait. A first version as a receive loop mirrored `py_context` but every +wait needed its own selective receive and its own timer bookkeeping. + +## Decision + +`py_isolated` is a `gen_statem` (`handle_event_function`, state enter +calls) with states `idle`, `{busy, Id}`, `looping`, `stopping_loop` and +`{restarting, Reason}`. Requests that must wait are `postpone`d and +replayed by the behaviour in arrival order; timers are `state_timeout` +and named generic timeouts (`{timeout, kill}`) cancelled by the state +change that makes them moot. It is spawned with `proc_lib` so it keeps +the process identity and message protocol of `py_context`. + +Not chosen: sharing the receive loop with `py_context` (the two have +different failure models: a child can die and restart, a thread cannot), +or a `gen_server` with a state field (postpone and state timeouts would +have to be reimplemented). + +## Consequences + +- `sys:get_state/1` shows what a context is doing and `sys:trace/2` + prints every event; the state machines page is a transcription of the + callback module. +- Callers do not see the behaviour: messages and replies are those of + `py_context`. +- The two context processes are different code. A change to the message + protocol touches both. diff --git a/docs/decisions/0006-shared-memory-over-iommap.md b/docs/decisions/0006-shared-memory-over-iommap.md new file mode 100644 index 0000000..73f780a --- /dev/null +++ b/docs/decisions/0006-shared-memory-over-iommap.md @@ -0,0 +1,42 @@ +# 0006: Bulk data through iommap regions, handles as plain tuples + +Since 5.0.0. Code: `src/py_shm.erl`, `src/py_buffer.erl` (shared variant), +`priv/_erlang_impl/_shm.py`, the tagged-tuple case in `c_src/py_convert.c`. + +## Situation + +An isolated call copies its arguments and result through the socket: +1.3 ms per MB, 300 ms for 64 MB. Request bodies, arrays and model inputs +need a path that does not copy, and it must work the same in a pool that +mixes embedded and isolated contexts. + +## Decision + +A region is a file mapped `MAP_SHARED` by the VM through iommap +(`region_binary/3` gives a refcounted binary with no copy) and by any +interpreter through `mmap`. Its handle is the plain term +`{'$py_shm', Id, Path, Size}`, so it travels inside any argument or +result with no special encoding and becomes a `SharedMemory` on arrival +in every mode (the C converter and the child both call `_shm.from_term`). +A shared `py_buffer` is a region used as a ring, with flow control +through registered callbacks (`_py_buffer_wait`, `_py_buffer_consumed`), +the mechanism channels already use, so no new control frames exist. +iommap is optional: `py_shm:new/1` returns `{error, iommap_not_available}` +without it. + +Not chosen: passing the region's fd (iommap exposes none; a path in a +0700 directory is enough), a NIF of our own for mapping, and channels +over shared memory (small terms stay on the socket). + +## Consequences + +- Python-produced data is zero-copy both ways; Erlang-produced data + costs one `pwrite` because a binary cannot be written in place. +- Sharing memory weakens isolation for that region only; read-only + handles keep a callee from writing. Sealing and syscall filtering are + separate work. +- In embedded contexts a shared buffer costs a callback round trip per + blocking read where the native buffer costs a pointer; the guide says + when to use which. +- The region file must never be truncated (`SIGBUS`); sizes are fixed at + creation and verified with `fstat` before mapping. diff --git a/docs/decisions/0007-remove-legacy-execution-paths.md b/docs/decisions/0007-remove-legacy-execution-paths.md new file mode 100644 index 0000000..5c7372c --- /dev/null +++ b/docs/decisions/0007-remove-legacy-execution-paths.md @@ -0,0 +1,31 @@ +# 0007: One execution path per mode; the legacy API is removed + +Since 5.0.0. Code: the removal in `c_src/py_nif.c`, `c_src/py_exec.c`, +`c_src/py_callback.c`, `src/py_nif.erl` (PR #75). + +## Situation + +After 0001 every context had a thread, but the code still carried the +paths from before: an executor thread with a worker pool, blocking NIF +variants that ran Python on dirty schedulers, suspended-state resources +for a resume protocol nothing used, and `py_nif` stubs for all of it. +Roughly 4 500 lines were reachable only from suites or from nothing, and +every reader had to work out which of two paths was live. + +## Decision + +Remove them. A context created today has exactly one path per mode: +the queue and context thread for worker and owngil, the socket for +isolated. NIFs that needed a thread now answer +`{error, context_has_no_thread}` instead of falling back to a scheduler. +Because public functions disappeared, the release that carries this is a +major version (5.0.0), not a minor one. + +## Consequences + +- `docs/code-map.md` can say "live" for everything in `src/` and + `c_src/` except the test helpers, and mean it. +- There is no fallback when a context has no thread; that state is a + bug, and it is reported as one. +- Anyone on the removed functions upgrades through the changelog's + Removed section. diff --git a/docs/decisions/0008-pipe-io-rules.md b/docs/decisions/0008-pipe-io-rules.md new file mode 100644 index 0000000..da9fac3 --- /dev/null +++ b/docs/decisions/0008-pipe-io-rules.md @@ -0,0 +1,31 @@ +# 0008: Pipe I/O is non-blocking, deadlined and waited with poll + +Since 3.1.0 (deadlines), 5.0.0 (poll). Code: `read_with_timeout` and +`write_all_with_deadline` in `c_src/py_nif.h`, the pipe setup in +`c_src/py_thread_worker.c` and `c_src/py_callback.c`. + +## Situation + +Callback responses are written by Erlang processes on dirty I/O +schedulers into pipes read by Python threads. A blocking write to a +stalled reader pinned a dirty scheduler for good; a short read or write +left the framed protocol out of phase with no way back. Later, a CI run +with more than 1024 open files showed that `select()` cannot watch such a +descriptor at all, and the first thread worker created past that point +took every later thread callback down with it. + +## Decision + +Write ends are `O_NONBLOCK`; every write is `write_all_with_deadline` +and every read `read_with_timeout`, both waiting with `poll()`. A +partial frame is never recovered in band: the thread worker is poisoned +and replaced, and the context pipe is closed only when its thread has +been joined. The coordinator reports a failed ready signal instead of +leaving Python to time out. + +## Consequences + +- No scheduler thread waits on Python without a bound. +- A desynchronised pipe costs one worker, not a hang. +- `select()` and `` do not appear in the NIF; a review + that sees them come back should ask why. diff --git a/docs/decisions/overview.md b/docs/decisions/overview.md new file mode 100644 index 0000000..0d1f7eb --- /dev/null +++ b/docs/decisions/overview.md @@ -0,0 +1,18 @@ +# Decision records + +Why the code is the way it is, one decision per file, in the order they +were taken. Read the record before changing what it decided; if the +reasons no longer hold, write a new record that supersedes it rather than +editing the old one. Each record has the same four parts: the situation, +what was decided, what it costs, and where the code is. + +| # | Decision | Since | +|---|---|---| +| [0001](0001-one-thread-per-context.md) | One pthread per context, NIFs only enqueue | 3.0.0 | +| [0002](0002-callback-delivery-paths.md) | Three callback delivery paths, chosen by the calling thread | 3.0.0 | +| [0003](0003-callback-results-as-etf.md) | Callback results cross as external term format | 4.0.0 | +| [0004](0004-isolated-mode-child-process.md) | Isolation is a child OS process over a Unix socket | 5.0.0 | +| [0005](0005-py-isolated-gen-statem.md) | The isolated context process is a gen_statem | 5.0.0 | +| [0006](0006-shared-memory-over-iommap.md) | Bulk data through iommap regions, handles as plain tuples | 5.0.0 | +| [0007](0007-remove-legacy-execution-paths.md) | One execution path per mode; the legacy API is removed | 5.0.0 | +| [0008](0008-pipe-io-rules.md) | Pipe I/O is non-blocking, deadlined and waited with poll | 3.1.0, 5.0.0 | diff --git a/rebar.config b/rebar.config index 7be76e5..ddae020 100644 --- a/rebar.config +++ b/rebar.config @@ -82,6 +82,15 @@ <<"docs/protocols.md">>, <<"docs/state-machines.md">>, <<"docs/contributing.md">>, + <<"docs/decisions/overview.md">>, + <<"docs/decisions/0001-one-thread-per-context.md">>, + <<"docs/decisions/0002-callback-delivery-paths.md">>, + <<"docs/decisions/0003-callback-results-as-etf.md">>, + <<"docs/decisions/0004-isolated-mode-child-process.md">>, + <<"docs/decisions/0005-py-isolated-gen-statem.md">>, + <<"docs/decisions/0006-shared-memory-over-iommap.md">>, + <<"docs/decisions/0007-remove-legacy-execution-paths.md">>, + <<"docs/decisions/0008-pipe-io-rules.md">>, <<"docs/preload.md">>, <<"docs/owngil_internals.md">>, <<"docs/event_loop_architecture.md">> @@ -126,6 +135,17 @@ <<"docs/preload.md">>, <<"docs/owngil_internals.md">>, <<"docs/event_loop_architecture.md">> + ]}, + {<<"Decisions">>, [ + <<"docs/decisions/overview.md">>, + <<"docs/decisions/0001-one-thread-per-context.md">>, + <<"docs/decisions/0002-callback-delivery-paths.md">>, + <<"docs/decisions/0003-callback-results-as-etf.md">>, + <<"docs/decisions/0004-isolated-mode-child-process.md">>, + <<"docs/decisions/0005-py-isolated-gen-statem.md">>, + <<"docs/decisions/0006-shared-memory-over-iommap.md">>, + <<"docs/decisions/0007-remove-legacy-execution-paths.md">>, + <<"docs/decisions/0008-pipe-io-rules.md">> ]} ]} ]}. From e44a0aa1592681c0792ea7aa16563754fb915467 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sat, 29 Aug 2026 20:09:26 +0200 Subject: [PATCH 12/15] Collapse the dispatch paths and split the large modules (#82) One dispatcher for every context request in C, one NIF table entry list per area, one process body module per mode on the Erlang side, and a py facade that delegates streaming, venvs and shared dicts to their own modules. The thread functions and flags are named for what they are rather than for the worker mode they came from. No public API changes. --- CHANGELOG.md | 16 + c_src/README.md | 10 +- c_src/py_buffer.c | 7 + c_src/py_callback.c | 6 + c_src/py_channel.c | 18 + c_src/py_event_loop.c | 77 +- c_src/py_logging.c | 8 + c_src/py_nif.c | 2160 +++-------------- c_src/py_nif.h | 57 +- c_src/py_shared_dict.c | 10 + c_src/py_thread_worker.c | 8 + docs/architecture.md | 8 +- docs/code-map.md | 8 +- docs/contributing.md | 5 +- docs/decisions/0001-one-thread-per-context.md | 2 +- docs/glossary.md | 4 +- docs/owngil_internals.md | 34 +- docs/state-machines.md | 6 +- src/py.erl | 514 +--- src/py_context.erl | 894 +------ src/py_context_embedded.erl | 847 +++++++ src/py_reactor_context.erl | 2 +- src/py_shared_dict.erl | 110 + src/py_stream.erl | 255 ++ src/py_util.erl | 49 +- src/py_venv.erl | 350 +++ 26 files changed, 2153 insertions(+), 3312 deletions(-) create mode 100644 src/py_context_embedded.erl create mode 100644 src/py_shared_dict.erl create mode 100644 src/py_stream.erl create mode 100644 src/py_venv.erl diff --git a/CHANGELOG.md b/CHANGELOG.md index 8983353..37bdd1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,22 @@ storms, loop churn, 60 s mixed workload with resource counters checked. - Guide: `docs/isolated.md`, with what each of the three modes guarantees. +### Changed + +- The NIF side of every context request goes through one dispatcher + (`ctx_dispatch`, `ctx_dispatch_async` in `c_src/py_nif.c`) instead of a + per-request copy of the enqueue-and-wait loop; the execute functions are + `ctx_execute_*` and the thread functions `ctx_thread_main_*`, since both + serve worker and owngil contexts. Creating a process-local env and + applying imports or paths run on the context thread in `worker` mode too; + the scheduler-side copies of those paths are gone. +- The NIF function table is assembled from one `PY_*_NIFS` macro per area, + defined at the end of the file that owns the NIFs. +- `py_context` keeps the API and the reply protocol; the process body for + embedded modes moved to `py_context_embedded`. `py` delegates streaming, + virtual environments and shared dicts to `py_stream`, `py_venv` and + `py_shared_dict`. The public API is unchanged. + ### Removed - The legacy worker API (`py_nif:worker_new/0,1`, `worker_call`, `worker_eval`, diff --git a/c_src/README.md b/c_src/README.md index 8d1dc06..d3c02f8 100644 --- a/c_src/README.md +++ b/c_src/README.md @@ -14,7 +14,7 @@ where things are. | File | What it owns | Notes | |---|---|---| | `py_nif.h` | All shared types: `py_context_t` and its request queue, request types, callback and suspension state, runtime state machine, atoms, globals, declarations | 2.4k lines. The struct comments carry the locking rules; read `py_context_t` before touching threads | -| `py_nif.c` | Runtime init and finalize, resource types, context create/destroy, the request queue, `worker_context_thread_main` and `owngil_context_thread_main`, `owngil_execute_*` (used by both thread kinds), the `nif_context_*` NIFs, process-local envs, `py_ref`, the NIF function table at the end | Sections are banner-separated; `grep -n '^ \* ===\|^/\* ==='` lists them | +| `py_nif.c` | Runtime init and finalize, resource types, context create/destroy, the request queue, `ctx_thread_main_worker` and `ctx_thread_main_owngil`, `ctx_execute_*` (one set for both thread kinds), `ctx_dispatch` / `ctx_dispatch_async` (the only way a NIF reaches a context thread), the `nif_context_*` NIFs, process-local envs, `py_ref`, the NIF function table at the end, assembled from the `PY_*_NIFS` macros of the other files | Sections are banner-separated; `grep -n '^ \* ===\|^/\* ==='` lists them | | `py_convert.c` | `py_to_term`, `term_to_py`, depth limits, tagged tuples (`{bytes, B}`, `{'$py_shm', ...}`), error tuples `{error, {Type, Msg}}` | The type mapping tables in the comments are the reference for `_etf.py` | | `py_exec.c` | Execution mode detection (free-threaded or GIL build) | | | `py_callback.c` | The `erlang` Python module: `call`, `send`, `whereis`, `schedule*`, `Atom`/`Pid`/`Ref` types, callback delivery paths (suspension, blocking pipe, async pipe), channel and shared-dict methods, callback name registry | `erlang_call_impl` documents the path precedence | @@ -29,8 +29,8 @@ where things are. ## Where the live paths are - `py:call/3` in worker or owngil mode: `nif_context_call_async` (`py_nif.c`) - enqueues; `worker_context_thread_main` or `owngil_context_thread_main` - dequeues and calls `owngil_execute_request`; the reply goes out as + enqueues; `ctx_thread_main_worker` or `ctx_thread_main_owngil` + dequeues and calls `ctx_execute_request`; the reply goes out as `{py_result, Ref, Result}`. - `erlang.call` from Python: `erlang_call_impl` (`py_callback.c`). - Interrupt: `nif_context_interrupt` (`py_nif.c`), `interrupt_mutex` rules on @@ -57,7 +57,9 @@ where things are. 1. Implement `static ERL_NIF_TERM nif_x(ErlNifEnv*, int, const ERL_NIF_TERM[])` next to related code. -2. Add `{"x", Arity, nif_x, Flags}` to `nif_funcs[]` at the end of `py_nif.c`. +2. Add `{"x", Arity, nif_x, Flags}` to the `PY_*_NIFS` macro at the end of + that file (or to the `py_nif.c` block of `nif_funcs[]` for NIFs that + live there). 3. Add the stub and its `-spec` and doc to `src/py_nif.erl`. 4. Cover it in a suite; `rebar3 dialyzer` and `rebar3 xref` must stay clean. diff --git a/c_src/py_buffer.c b/c_src/py_buffer.c index f21f25e..1ee1927 100644 --- a/c_src/py_buffer.c +++ b/c_src/py_buffer.c @@ -1103,3 +1103,10 @@ static ERL_NIF_TERM nif_py_buffer_close(ErlNifEnv *env, int argc, return ATOM_OK; } + +/* NIF table entries of this file; py_nif.c concatenates them into nif_funcs[]. + * Flags: ERL_NIF_DIRTY_JOB_* for anything that can block or run Python. */ +#define PY_BUFFER_NIFS \ + {"py_buffer_create", 1, nif_py_buffer_create, 0}, \ + {"py_buffer_write", 2, nif_py_buffer_write, 0}, \ + {"py_buffer_close", 1, nif_py_buffer_close, 0} diff --git a/c_src/py_callback.c b/c_src/py_callback.c index b560673..772d817 100644 --- a/c_src/py_callback.c +++ b/c_src/py_callback.c @@ -4053,3 +4053,9 @@ static ERL_NIF_TERM nif_unregister_callback_name(ErlNifEnv *env, int argc, const return ATOM_OK; } + +/* NIF table entries of this file; py_nif.c concatenates them into nif_funcs[]. + * Flags: ERL_NIF_DIRTY_JOB_* for anything that can block or run Python. */ +#define PY_CALLBACK_NIFS \ + {"register_callback_name", 1, nif_register_callback_name, 0}, \ + {"unregister_callback_name", 1, nif_unregister_callback_name, 0} diff --git a/c_src/py_channel.c b/c_src/py_channel.c index ecfba9f..c78ba38 100644 --- a/c_src/py_channel.c +++ b/c_src/py_channel.c @@ -1033,3 +1033,21 @@ ERL_NIF_TERM nif_byte_channel_wait_bytes(ErlNifEnv *env, int argc, const ERL_NIF /* Return ok - Python will await Future */ return ATOM_OK; } + +/* NIF table entries of this file; py_nif.c concatenates them into nif_funcs[]. + * Flags: ERL_NIF_DIRTY_JOB_* for anything that can block or run Python. */ +#define PY_CHANNEL_NIFS \ + {"channel_create", 0, nif_channel_create, 0}, \ + {"channel_create", 1, nif_channel_create, 0}, \ + {"channel_send", 2, nif_channel_send, 0}, \ + {"channel_receive", 2, nif_channel_receive, 0}, \ + {"channel_try_receive", 1, nif_channel_try_receive, 0}, \ + {"channel_reply", 3, nif_channel_reply, 0}, \ + {"channel_close", 1, nif_channel_close, 0}, \ + {"channel_info", 1, nif_channel_info, 0}, \ + {"channel_wait", 3, nif_channel_wait, 0}, \ + {"channel_cancel_wait", 2, nif_channel_cancel_wait, 0}, \ + {"channel_register_sync_waiter", 1, nif_channel_register_sync_waiter, 0}, \ + {"byte_channel_send_bytes", 2, nif_byte_channel_send_bytes, 0}, \ + {"byte_channel_try_receive_bytes", 1, nif_byte_channel_try_receive_bytes, 0}, \ + {"byte_channel_wait_bytes", 3, nif_byte_channel_wait_bytes, 0} diff --git a/c_src/py_event_loop.c b/c_src/py_event_loop.c index 91b37ca..f88d46f 100644 --- a/c_src/py_event_loop.c +++ b/c_src/py_event_loop.c @@ -5735,7 +5735,7 @@ ERL_NIF_TERM nif_reactor_on_read_ready(ErlNifEnv *env, int argc, #ifdef HAVE_SUBINTERPRETERS /* OWN_GIL mode: dispatch to dedicated thread */ if (ctx->uses_own_gil) { - return dispatch_reactor_read_to_owngil(env, ctx, fd, buffer); + return dispatch_reactor_read(env, ctx, fd, buffer); } #endif @@ -5831,7 +5831,7 @@ ERL_NIF_TERM nif_reactor_on_write_ready(ErlNifEnv *env, int argc, #ifdef HAVE_SUBINTERPRETERS /* OWN_GIL mode: dispatch to dedicated thread */ if (ctx->uses_own_gil) { - return dispatch_reactor_write_to_owngil(env, ctx, fd); + return dispatch_reactor_write(env, ctx, fd); } #endif @@ -5917,7 +5917,7 @@ ERL_NIF_TERM nif_reactor_init_connection(ErlNifEnv *env, int argc, #ifdef HAVE_SUBINTERPRETERS /* OWN_GIL mode: dispatch to dedicated thread */ if (ctx->uses_own_gil) { - return dispatch_reactor_init_to_owngil(env, ctx, fd, argv[2]); + return dispatch_reactor_init(env, ctx, fd, argv[2]); } #endif @@ -8528,3 +8528,74 @@ int init_subinterpreter_event_loop(ErlNifEnv *env) { } return 0; } + +/* NIF table entries of this file; py_nif.c concatenates them into nif_funcs[]. + * Flags: ERL_NIF_DIRTY_JOB_* for anything that can block or run Python. */ +#define PY_EVENT_LOOP_NIFS \ + {"set_event_loop_priv_dir", 1, nif_set_event_loop_priv_dir, 0}, \ + {"event_loop_new", 0, nif_event_loop_new, 0}, \ + {"event_loop_destroy", 1, nif_event_loop_destroy, 0}, \ + {"event_loop_set_router", 2, nif_event_loop_set_router, 0}, \ + {"event_loop_set_worker", 2, nif_event_loop_set_worker, 0}, \ + {"event_loop_set_id", 2, nif_event_loop_set_id, 0}, \ + {"event_loop_wakeup", 1, nif_event_loop_wakeup, 0}, \ + {"event_loop_run_async", 7, nif_event_loop_run_async, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"submit_task", 7, nif_submit_task, 0}, \ + {"submit_task_with_env", 8, nif_submit_task_with_env, 0}, \ + {"process_ready_tasks", 1, nif_process_ready_tasks, ERL_NIF_DIRTY_JOB_CPU_BOUND}, \ + {"event_loop_set_py_loop", 2, nif_event_loop_set_py_loop, 0}, \ + {"event_loop_exec", 2, nif_event_loop_exec, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"event_loop_eval", 2, nif_event_loop_eval, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"add_reader", 3, nif_add_reader, 0}, \ + {"remove_reader", 2, nif_remove_reader, 0}, \ + {"add_writer", 3, nif_add_writer, 0}, \ + {"remove_writer", 2, nif_remove_writer, 0}, \ + {"call_later", 3, nif_call_later, 0}, \ + {"cancel_timer", 2, nif_cancel_timer, 0}, \ + {"poll_events", 2, nif_poll_events, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"get_pending", 1, nif_get_pending, 0}, \ + {"dispatch_callback", 3, nif_dispatch_callback, 0}, \ + {"dispatch_timer", 2, nif_dispatch_timer, 0}, \ + {"get_fd_callback_id", 2, nif_get_fd_callback_id, 0}, \ + {"reselect_reader", 2, nif_reselect_reader, 0}, \ + {"reselect_writer", 2, nif_reselect_writer, 0}, \ + {"reselect_reader_fd", 1, nif_reselect_reader_fd, 0}, \ + {"reselect_writer_fd", 1, nif_reselect_writer_fd, 0}, \ + {"handle_fd_event", 2, nif_handle_fd_event, 0}, \ + {"handle_fd_event_and_reselect", 2, nif_handle_fd_event_and_reselect, 0}, \ + {"fd_arm", 2, nif_fd_arm, 0}, \ + {"stop_reader", 1, nif_stop_reader, 0}, \ + {"start_reader", 1, nif_start_reader, 0}, \ + {"stop_writer", 1, nif_stop_writer, 0}, \ + {"start_writer", 1, nif_start_writer, 0}, \ + {"close_fd", 1, nif_close_fd, 0}, \ + {"create_test_pipe", 0, nif_create_test_pipe, 0}, \ + {"close_test_fd", 1, nif_close_test_fd, 0}, \ + {"dup_fd", 1, nif_dup_fd, 0}, \ + {"write_test_fd", 2, nif_write_test_fd, 0}, \ + {"read_test_fd", 2, nif_read_test_fd, 0}, \ + {"create_test_tcp_listener", 1, nif_create_test_tcp_listener, 0}, \ + {"accept_test_tcp", 1, nif_accept_test_tcp, 0}, \ + {"connect_test_tcp", 2, nif_connect_test_tcp, 0}, \ + {"create_test_udp_socket", 1, nif_create_test_udp_socket, 0}, \ + {"recvfrom_test_udp", 2, nif_recvfrom_test_udp, 0}, \ + {"sendto_test_udp", 4, nif_sendto_test_udp, 0}, \ + {"set_udp_broadcast", 2, nif_set_udp_broadcast, 0}, \ + {"set_python_event_loop", 1, nif_set_python_event_loop, 0}, \ + {"set_isolation_mode", 1, nif_set_isolation_mode, 0}, \ + {"set_shared_worker", 1, nif_set_shared_worker, 0}, \ + {"context_get_event_loop", 1, nif_context_get_event_loop, 0}, \ + {"reactor_register_fd", 3, nif_reactor_register_fd, 0}, \ + {"reactor_reselect_read", 1, nif_reactor_reselect_read, 0}, \ + {"reactor_select_write", 1, nif_reactor_select_write, 0}, \ + {"get_fd_from_resource", 1, nif_get_fd_from_resource, 0}, \ + {"reactor_on_read_ready", 2, nif_reactor_on_read_ready, ERL_NIF_DIRTY_JOB_CPU_BOUND}, \ + {"reactor_on_write_ready", 2, nif_reactor_on_write_ready, ERL_NIF_DIRTY_JOB_CPU_BOUND}, \ + {"reactor_init_connection", 3, nif_reactor_init_connection, ERL_NIF_DIRTY_JOB_CPU_BOUND}, \ + {"reactor_close_fd", 2, nif_reactor_close_fd, 0}, \ + {"fd_read", 2, nif_fd_read, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"fd_write", 2, nif_fd_write, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"fd_select_read", 1, nif_fd_select_read, 0}, \ + {"fd_select_write", 1, nif_fd_select_write, 0}, \ + {"fd_close", 1, nif_fd_close, 0}, \ + {"socketpair", 0, nif_socketpair, 0} diff --git a/c_src/py_logging.c b/c_src/py_logging.c index 14b1f7e..1ca5077 100644 --- a/c_src/py_logging.c +++ b/c_src/py_logging.c @@ -453,3 +453,11 @@ static ERL_NIF_TERM nif_clear_trace_receiver(ErlNifEnv *env, int argc, const ERL return ATOM_OK; } + +/* NIF table entries of this file; py_nif.c concatenates them into nif_funcs[]. + * Flags: ERL_NIF_DIRTY_JOB_* for anything that can block or run Python. */ +#define PY_LOGGING_NIFS \ + {"set_log_receiver", 2, nif_set_log_receiver, 0}, \ + {"clear_log_receiver", 0, nif_clear_log_receiver, 0}, \ + {"set_trace_receiver", 1, nif_set_trace_receiver, 0}, \ + {"clear_trace_receiver", 0, nif_clear_trace_receiver, 0} diff --git a/c_src/py_nif.c b/c_src/py_nif.c index 14c3b62..69fc110 100644 --- a/c_src/py_nif.c +++ b/c_src/py_nif.c @@ -670,218 +670,6 @@ static inline_continuation_t *create_inline_continuation( return cont; } -/** - * @brief NIF: Execute inline continuation - * - * This is the continuation function called by enif_schedule_nif(). - * It executes the Python function and handles the result: - * - InlineScheduleMarker: chain via another enif_schedule_nif - * - ScheduleMarker: return {schedule, ...} to Erlang - * - Suspension: return {suspended, ...} to Erlang - * - Normal result: return {ok, Result} - */ -static ERL_NIF_TERM nif_inline_continuation(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - (void)argc; - - inline_continuation_t *cont; - if (!enif_get_resource(env, argv[0], INLINE_CONTINUATION_RESOURCE_TYPE, (void **)&cont)) { - return make_error(env, "invalid_continuation"); - } - - if (!runtime_is_running()) { - return make_error(env, "python_not_running"); - } - - /* Check depth limit */ - if (cont->depth >= MAX_INLINE_CONTINUATION_DEPTH) { - return make_error(env, "inline_continuation_depth_exceeded"); - } - - py_context_t *ctx = cont->ctx; - if (ctx == NULL || ctx->destroyed) { - return make_error(env, "context_destroyed"); - } - - /* Acquire thread state */ - py_context_guard_t guard = py_context_acquire(ctx); - if (!guard.acquired) { - return make_error(env, "acquire_failed"); - } - - /* Set thread-local context for callback support */ - py_context_t *prev_context = tl_current_context; - tl_current_context = ctx; - - /* Enable suspension for callback support */ - bool prev_allow_suspension = tl_allow_suspension; - tl_allow_suspension = true; - - /* Set callback env for consume_time_slice */ - ErlNifEnv *prev_callback_env = tl_callback_env; - tl_callback_env = env; - - ERL_NIF_TERM result; - - /* Import module and get function */ - PyObject *func = NULL; - PyObject *module = NULL; - - /* Priority for __main__ lookups: - * 1. Captured globals/locals from the marker (caller's frame) - * 2. local_env globals (process-local environment) - * 3. ctx->globals/locals (context defaults) - */ - py_env_resource_t *local_env = (py_env_resource_t *)cont->local_env; - - if (strcmp(cont->module_name, "__main__") == 0) { - /* Try captured globals first (from caller's frame) */ - if (cont->globals != NULL) { - func = PyDict_GetItemString(cont->globals, cont->func_name); - } - /* Try captured locals */ - if (func == NULL && cont->locals != NULL) { - func = PyDict_GetItemString(cont->locals, cont->func_name); - } - /* Fallback to local_env globals */ - if (func == NULL && local_env != NULL) { - func = PyDict_GetItemString(local_env->globals, cont->func_name); - } - /* Fallback to context globals/locals */ - if (func == NULL) { - func = PyDict_GetItemString(ctx->globals, cont->func_name); - } - if (func == NULL) { - func = PyDict_GetItemString(ctx->locals, cont->func_name); - } - if (func != NULL) { - Py_INCREF(func); - } else { - PyErr_Format(PyExc_NameError, "name '%s' is not defined", cont->func_name); - } - } else { - module = PyImport_ImportModule(cont->module_name); - if (module != NULL) { - func = PyObject_GetAttrString(module, cont->func_name); - Py_DECREF(module); - } - } - - if (func == NULL) { - result = make_py_error(env); - goto cleanup; - } - - /* Build args tuple */ - PyObject *args = cont->args; - if (args == NULL) { - args = PyTuple_New(0); - if (args == NULL) { - Py_DECREF(func); - result = make_py_error(env); - goto cleanup; - } - } else { - Py_INCREF(args); - } - - /* Get kwargs */ - PyObject *kwargs = cont->kwargs; - - /* Call the function */ - PyObject *py_result = PyObject_Call(func, args, kwargs); - Py_DECREF(func); - Py_DECREF(args); - - if (py_result == NULL) { - /* Check for pending callback */ - if (tl_pending_callback) { - PyErr_Clear(); - - /* Create suspended context state for callback handling */ - ErlNifBinary module_bin, func_bin; - enif_alloc_binary(cont->module_len, &module_bin); - memcpy(module_bin.data, cont->module_name, cont->module_len); - enif_alloc_binary(cont->func_len, &func_bin); - memcpy(func_bin.data, cont->func_name, cont->func_len); - - /* Convert args to Erlang term for replay */ - ERL_NIF_TERM args_term = enif_make_list(env, 0); - if (cont->args != NULL) { - args_term = py_to_term(env, cont->args); - } - - ERL_NIF_TERM kwargs_term = enif_make_new_map(env); - if (cont->kwargs != NULL) { - kwargs_term = py_to_term(env, cont->kwargs); - } - - suspended_context_state_t *suspended = create_suspended_context_state_for_call( - env, ctx, &module_bin, &func_bin, args_term, kwargs_term); - - enif_release_binary(&module_bin); - enif_release_binary(&func_bin); - - if (suspended == NULL) { - tl_pending_callback = false; - Py_CLEAR(tl_pending_args); - result = make_error(env, "create_suspended_state_failed"); - } else { - result = build_suspended_context_result(env, suspended); - } - } else { - result = make_py_error(env); - } - } else if (is_inline_schedule_marker(py_result)) { - /* Chain via another enif_schedule_nif */ - inline_continuation_t *next_cont = create_inline_continuation( - ctx, cont->local_env, py_result, cont->depth + 1); - Py_DECREF(py_result); - - if (next_cont == NULL) { - result = make_error(env, "create_continuation_failed"); - } else { - ERL_NIF_TERM cont_ref = enif_make_resource(env, next_cont); - enif_release_resource(next_cont); - - /* Restore thread-local state before scheduling */ - tl_allow_suspension = prev_allow_suspension; - tl_current_context = prev_context; - tl_callback_env = prev_callback_env; - clear_pending_callback_tls(); - - py_context_release(&guard); - - return enif_schedule_nif(env, "inline_continuation", - ERL_NIF_DIRTY_JOB_IO_BOUND, nif_inline_continuation, 1, &cont_ref); - } - } else if (is_schedule_marker(py_result)) { - /* Switch to schedule_py path */ - ScheduleMarkerObject *marker = (ScheduleMarkerObject *)py_result; - ERL_NIF_TERM callback_name = py_to_term(env, marker->callback_name); - ERL_NIF_TERM callback_args = py_to_term(env, marker->args); - Py_DECREF(py_result); - result = enif_make_tuple3(env, ATOM_SCHEDULE, callback_name, callback_args); - } else { - /* Normal result */ - ERL_NIF_TERM term_result = py_to_term(env, py_result); - Py_DECREF(py_result); - result = enif_make_tuple2(env, ATOM_OK, term_result); - } - -cleanup: - /* Restore thread-local state */ - tl_allow_suspension = prev_allow_suspension; - tl_current_context = prev_context; - tl_callback_env = prev_callback_env; - - /* Clear pending callback TLS */ - clear_pending_callback_tls(); - - /* Release thread state */ - py_context_release(&guard); - - return result; -} /* ============================================================================ * Initialization @@ -1571,7 +1359,7 @@ static void ctx_queue_cancel_all(py_context_t *ctx) { /** * @brief Execute a call request in the OWN_GIL thread */ -static void owngil_execute_call(py_context_t *ctx) { +static void ctx_execute_call(py_context_t *ctx) { /* Decode request from shared_env */ ERL_NIF_TERM module_term, func_term, args_term, kwargs_term; const ERL_NIF_TERM *tuple_terms; @@ -1713,7 +1501,7 @@ static void owngil_execute_call(py_context_t *ctx) { /** * @brief Execute an eval request in the OWN_GIL thread */ -static void owngil_execute_eval(py_context_t *ctx) { +static void ctx_execute_eval(py_context_t *ctx) { /* Decode request: {Code, Locals} */ const ERL_NIF_TERM *tuple_terms; int tuple_arity; @@ -1782,7 +1570,7 @@ static void owngil_execute_eval(py_context_t *ctx) { /** * @brief Execute an exec request in the OWN_GIL thread */ -static void owngil_execute_exec(py_context_t *ctx) { +static void ctx_execute_exec(py_context_t *ctx) { ErlNifBinary code_bin; if (!enif_inspect_binary(ctx->shared_env, ctx->request_term, &code_bin)) { ctx->response_term = enif_make_tuple2(ctx->shared_env, @@ -1829,7 +1617,7 @@ static void owngil_execute_exec(py_context_t *ctx) { /** * @brief Execute a reactor on_read_ready request in OWN_GIL thread */ -static void owngil_execute_reactor_read(py_context_t *ctx) { +static void ctx_execute_reactor_read(py_context_t *ctx) { /* Extract fd from request term (it's just an integer) */ int fd; if (!enif_get_int(ctx->shared_env, ctx->request_term, &fd)) { @@ -1860,7 +1648,7 @@ static void owngil_execute_reactor_read(py_context_t *ctx) { /** * @brief Execute a reactor on_write_ready request in OWN_GIL thread */ -static void owngil_execute_reactor_write(py_context_t *ctx) { +static void ctx_execute_reactor_write(py_context_t *ctx) { /* Extract fd from request term */ int fd; if (!enif_get_int(ctx->shared_env, ctx->request_term, &fd)) { @@ -1879,7 +1667,7 @@ static void owngil_execute_reactor_write(py_context_t *ctx) { /** * @brief Execute a reactor init_connection request in OWN_GIL thread */ -static void owngil_execute_reactor_init(py_context_t *ctx) { +static void ctx_execute_reactor_init(py_context_t *ctx) { /* Extract {Fd, ClientInfo} from request term */ const ERL_NIF_TERM *tuple; int arity; @@ -1910,7 +1698,7 @@ static void owngil_execute_reactor_init(py_context_t *ctx) { * * Uses penv->globals/locals instead of ctx->globals/locals */ -static void owngil_execute_exec_with_env(py_context_t *ctx) { +static void ctx_execute_exec_with_env(py_context_t *ctx) { py_env_resource_t *penv = (py_env_resource_t *)ctx->local_env_ptr; ctx->local_env_ptr = NULL; /* Clear after use */ @@ -1987,7 +1775,7 @@ static void owngil_execute_exec_with_env(py_context_t *ctx) { * * Uses penv->globals/locals instead of ctx->globals/locals */ -static void owngil_execute_eval_with_env(py_context_t *ctx) { +static void ctx_execute_eval_with_env(py_context_t *ctx) { py_env_resource_t *penv = (py_env_resource_t *)ctx->local_env_ptr; ctx->local_env_ptr = NULL; /* Clear after use */ @@ -2250,7 +2038,7 @@ static void owngil_execute_eval_with_env(py_context_t *ctx) { * * Uses penv->globals for function lookup in __main__ module */ -static void owngil_execute_call_with_env(py_context_t *ctx) { +static void ctx_execute_call_with_env(py_context_t *ctx) { py_env_resource_t *penv = (py_env_resource_t *)ctx->local_env_ptr; ctx->local_env_ptr = NULL; /* Clear after use */ @@ -2427,7 +2215,7 @@ static void owngil_execute_call_with_env(py_context_t *ctx) { * Creates globals/locals dicts in the correct interpreter context. * The py_env_resource_t is passed via local_env_ptr. */ -static void owngil_execute_create_local_env(py_context_t *ctx) { +static void ctx_execute_create_local_env(py_context_t *ctx) { py_env_resource_t *res = (py_env_resource_t *)ctx->local_env_ptr; ctx->local_env_ptr = NULL; /* Clear after use */ @@ -2498,7 +2286,7 @@ static void owngil_execute_create_local_env(py_context_t *ctx) { * Note: OWN_GIL contexts have their own dedicated interpreter, * so sys.modules is per-context in this mode. */ -static void owngil_execute_apply_imports(py_context_t *ctx) { +static void ctx_execute_apply_imports(py_context_t *ctx) { /* Process each import from request_term */ ERL_NIF_TERM head, tail = ctx->request_term; int arity; @@ -2547,7 +2335,7 @@ static void owngil_execute_apply_imports(py_context_t *ctx) { * * Paths are inserted at the beginning of sys.path. */ -static void owngil_execute_apply_paths(py_context_t *ctx) { +static void ctx_execute_apply_paths(py_context_t *ctx) { /* Get sys.path */ PyObject *sys_module = PyImport_ImportModule("sys"); if (sys_module == NULL) { @@ -2616,43 +2404,43 @@ static void owngil_execute_apply_paths(py_context_t *ctx) { /** * @brief Execute a request based on its type */ -static void owngil_execute_request(py_context_t *ctx) { +static void ctx_execute_request(py_context_t *ctx) { switch (ctx->request_type) { case CTX_REQ_CALL: - owngil_execute_call(ctx); + ctx_execute_call(ctx); break; case CTX_REQ_EVAL: - owngil_execute_eval(ctx); + ctx_execute_eval(ctx); break; case CTX_REQ_EXEC: - owngil_execute_exec(ctx); + ctx_execute_exec(ctx); break; case CTX_REQ_REACTOR_ON_READ_READY: - owngil_execute_reactor_read(ctx); + ctx_execute_reactor_read(ctx); break; case CTX_REQ_REACTOR_ON_WRITE_READY: - owngil_execute_reactor_write(ctx); + ctx_execute_reactor_write(ctx); break; case CTX_REQ_REACTOR_INIT_CONNECTION: - owngil_execute_reactor_init(ctx); + ctx_execute_reactor_init(ctx); break; case CTX_REQ_EXEC_WITH_ENV: - owngil_execute_exec_with_env(ctx); + ctx_execute_exec_with_env(ctx); break; case CTX_REQ_EVAL_WITH_ENV: - owngil_execute_eval_with_env(ctx); + ctx_execute_eval_with_env(ctx); break; case CTX_REQ_CALL_WITH_ENV: - owngil_execute_call_with_env(ctx); + ctx_execute_call_with_env(ctx); break; case CTX_REQ_CREATE_LOCAL_ENV: - owngil_execute_create_local_env(ctx); + ctx_execute_create_local_env(ctx); break; case CTX_REQ_APPLY_IMPORTS: - owngil_execute_apply_imports(ctx); + ctx_execute_apply_imports(ctx); break; case CTX_REQ_APPLY_PATHS: - owngil_execute_apply_paths(ctx); + ctx_execute_apply_paths(ctx); break; default: ctx->response_term = enif_make_tuple2(ctx->shared_env, @@ -2681,7 +2469,7 @@ static void owngil_execute_request(py_context_t *ctx) { * with other Python threads. The benefit is stable thread affinity and * compatibility with all Python extensions. */ -static void *worker_context_thread_main(void *arg) { +static void *ctx_thread_main_worker(void *arg) { py_context_t *ctx = (py_context_t *)arg; /* Create namespace dictionaries on the worker thread under GIL */ @@ -2696,7 +2484,7 @@ static void *worker_context_thread_main(void *arg) { if (ctx->globals == NULL || ctx->locals == NULL || ctx->module_cache == NULL) { PyGILState_Release(gstate); atomic_store(&ctx->init_error, true); - atomic_store(&ctx->worker_running, false); + atomic_store(&ctx->thread_running, false); return NULL; } @@ -2717,7 +2505,7 @@ static void *worker_context_thread_main(void *arg) { PyGILState_Release(gstate); /* Signal that we're ready */ - atomic_store(&ctx->worker_running, true); + atomic_store(&ctx->thread_running, true); /* Main request loop - uses queue instead of single-slot */ while (!atomic_load(&ctx->shutdown_requested)) { @@ -2786,7 +2574,7 @@ static void *worker_context_thread_main(void *arg) { * invariant on py_context::interrupt_mutex). */ py_context_exec_enter(ctx); gstate = PyGILState_Ensure(); - owngil_execute_request(ctx); /* Reuse execute functions */ + ctx_execute_request(ctx); /* Reuse execute functions */ PyGILState_Release(gstate); py_context_exec_leave(ctx); @@ -2842,7 +2630,7 @@ static void *worker_context_thread_main(void *arg) { ctx->module_cache = NULL; PyGILState_Release(gstate); - atomic_store(&ctx->worker_running, false); + atomic_store(&ctx->thread_running, false); return NULL; } @@ -2853,10 +2641,10 @@ static void *worker_context_thread_main(void *arg) { * @return 0 on success, -1 on failure */ static int worker_context_init(py_context_t *ctx) { - ctx->uses_worker_thread = true; + ctx->has_thread = true; /* Initialize worker thread state */ - atomic_store(&ctx->worker_running, false); + atomic_store(&ctx->thread_running, false); atomic_store(&ctx->shutdown_requested, false); atomic_store(&ctx->leaked, false); @@ -2898,7 +2686,7 @@ static int worker_context_init(py_context_t *ctx) { ctx->module_cache = NULL; /* Start the worker thread */ - if (pthread_create(&ctx->worker_thread, NULL, worker_context_thread_main, ctx) != 0) { + if (pthread_create(&ctx->thread, NULL, ctx_thread_main_worker, ctx) != 0) { enif_free_env(ctx->msg_env); ctx->msg_env = NULL; pthread_cond_destroy(&ctx->queue_not_empty); @@ -2908,16 +2696,16 @@ static int worker_context_init(py_context_t *ctx) { /* Wait for thread to initialize or fail */ int wait_count = 0; - while (!atomic_load(&ctx->worker_running) && + while (!atomic_load(&ctx->thread_running) && !atomic_load(&ctx->init_error) && wait_count < 2000) { usleep(1000); /* 1ms */ wait_count++; } - if (atomic_load(&ctx->init_error) || !atomic_load(&ctx->worker_running)) { + if (atomic_load(&ctx->init_error) || !atomic_load(&ctx->thread_running)) { /* Thread failed to start */ - pthread_join(ctx->worker_thread, NULL); + pthread_join(ctx->thread, NULL); if (ctx->msg_env != NULL) { enif_free_env(ctx->msg_env); ctx->msg_env = NULL; @@ -2939,10 +2727,10 @@ static int worker_context_init(py_context_t *ctx) { * * @param ctx Context to shutdown */ -#define WORKER_SHUTDOWN_TIMEOUT_SECS 30 +#define CTX_THREAD_JOIN_TIMEOUT_SECS 30 -static void worker_context_shutdown(py_context_t *ctx) { - if (!ctx->uses_worker_thread) { +static void ctx_thread_shutdown_worker(py_context_t *ctx) { + if (!ctx->has_thread) { return; } @@ -2969,19 +2757,19 @@ static void worker_context_shutdown(py_context_t *ctx) { #if defined(__linux__) struct timespec deadline; clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += WORKER_SHUTDOWN_TIMEOUT_SECS; - int rc = pthread_timedjoin_np(ctx->worker_thread, NULL, &deadline); + deadline.tv_sec += CTX_THREAD_JOIN_TIMEOUT_SECS; + int rc = pthread_timedjoin_np(ctx->thread, NULL, &deadline); join_succeeded = (rc == 0); #else - /* macOS/other: poll worker_running flag with timeout */ + /* macOS/other: poll thread_running flag with timeout */ int wait_ms = 0; - while (atomic_load(&ctx->worker_running) && - wait_ms < WORKER_SHUTDOWN_TIMEOUT_SECS * 1000) { + while (atomic_load(&ctx->thread_running) && + wait_ms < CTX_THREAD_JOIN_TIMEOUT_SECS * 1000) { usleep(100000); /* 100ms */ wait_ms += 100; } - if (!atomic_load(&ctx->worker_running)) { - pthread_join(ctx->worker_thread, NULL); + if (!atomic_load(&ctx->thread_running)) { + pthread_join(ctx->thread, NULL); join_succeeded = true; } #endif @@ -2999,7 +2787,7 @@ static void worker_context_shutdown(py_context_t *ctx) { * !ctx->leaked for the same reason). Future cleanup happens * at VM exit. */ fprintf(stderr, "Worker thread shutdown timeout after %d seconds, leaking context\n", - WORKER_SHUTDOWN_TIMEOUT_SECS); + CTX_THREAD_JOIN_TIMEOUT_SECS); atomic_store(&ctx->leaked, true); enif_keep_resource(ctx); return; @@ -3014,7 +2802,7 @@ static void worker_context_shutdown(py_context_t *ctx) { pthread_cond_destroy(&ctx->queue_not_empty); pthread_mutex_destroy(&ctx->queue_mutex); - ctx->uses_worker_thread = false; + ctx->has_thread = false; } /** @@ -3029,94 +2817,95 @@ static void worker_context_shutdown(py_context_t *ctx) { * @param request_data Request data term * @return Result term copied back to caller's env */ -#define WORKER_DISPATCH_TIMEOUT_SECS 30 +#define CTX_DISPATCH_TIMEOUT_SECS 30 /** - * @brief Dispatch a request to the worker thread with optional local environment + * @brief Allocate a request for @p ctx, or return NULL with *err set * - * @param env NIF environment - * @param ctx Context to dispatch to - * @param req_type Request type - * @param request_data Request data term - * @param local_env Optional local environment (NULL for default) - * @return Result term + * Every dispatch starts here: the context must have a running thread + * and must not be destroyed. The caller fills the request fields and + * hands it to ctx_dispatch_wait() or ctx_dispatch_async(). */ -static ERL_NIF_TERM dispatch_to_worker_thread_impl( - ErlNifEnv *env, - py_context_t *ctx, - ctx_request_type_t req_type, - ERL_NIF_TERM request_data, - void *local_env -) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); +static ctx_request_t *ctx_request_begin(ErlNifEnv *env, py_context_t *ctx, + ctx_request_type_t req_type, + ERL_NIF_TERM *err) { + if (!atomic_load(&ctx->thread_running)) { + *err = make_error(env, "thread_not_running"); + return NULL; } - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); + *err = make_error(env, "context_destroyed"); + return NULL; } - - /* Create request struct */ ctx_request_t *req = ctx_request_create(); if (req == NULL) { - return make_error(env, "alloc_failed"); + *err = make_error(env, "alloc_failed"); + return NULL; } - - /* Populate request */ req->type = req_type; - req->request_data = enif_make_copy(req->request_env, request_data); - req->local_env_ptr = local_env; + return req; +} - /* Add extra reference for queue (caller holds 1, queue holds 1) */ +/** + * @brief Enqueue a prepared request and block until the context thread + * answers it (or the dispatch timeout passes) + * + * Takes over the caller's reference on @p req. Used by the blocking NIFs + * and by the reactor callbacks; the async NIFs use ctx_dispatch_async(). + */ +static ERL_NIF_TERM ctx_dispatch_wait(ErlNifEnv *env, py_context_t *ctx, + ctx_request_t *req) { + /* Queue holds one reference, the caller keeps one */ ctx_request_addref(req); ctx_queue_enqueue(ctx, req); - /* Wait for completion with timeout */ struct timespec deadline; clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += WORKER_DISPATCH_TIMEOUT_SECS; + deadline.tv_sec += CTX_DISPATCH_TIMEOUT_SECS; - ERL_NIF_TERM result; pthread_mutex_lock(&req->mutex); - while (!atomic_load(&req->completed)) { int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); if (rc == ETIMEDOUT) { - /* Timeout - mark as cancelled and return error */ + /* The thread may still be inside a long Python call: fail this + * request only, the thread will skip it as cancelled. */ atomic_store(&req->cancelled, true); pthread_mutex_unlock(&req->mutex); + fprintf(stderr, "context dispatch timeout after %d seconds (request type %d)\n", + CTX_DISPATCH_TIMEOUT_SECS, (int)req->type); ctx_request_release(req); return make_error(env, "worker_timeout"); } } - pthread_mutex_unlock(&req->mutex); - /* Copy result to caller's environment */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - /* Release caller's reference */ + ERL_NIF_TERM result = (req->result_env != NULL) + ? enif_make_copy(env, req->result) + : make_error(env, "no_result"); ctx_request_release(req); - return result; } /** - * @brief Convenience wrapper for dispatch without local environment + * @brief Blocking dispatch of a request whose data is one term + * + * @param local_env Process-local env resource for *_WITH_ENV requests, + * NULL otherwise. */ -static ERL_NIF_TERM dispatch_to_worker_thread( - ErlNifEnv *env, - py_context_t *ctx, - ctx_request_type_t req_type, - ERL_NIF_TERM request_data -) { - return dispatch_to_worker_thread_impl(env, ctx, req_type, request_data, NULL); +static ERL_NIF_TERM ctx_dispatch(ErlNifEnv *env, py_context_t *ctx, + ctx_request_type_t req_type, + ERL_NIF_TERM request_data, void *local_env) { + ERL_NIF_TERM err; + ctx_request_t *req = ctx_request_begin(env, ctx, req_type, &err); + if (req == NULL) { + return err; + } + req->request_data = enif_make_copy(req->request_env, request_data); + req->local_env_ptr = local_env; + return ctx_dispatch_wait(env, ctx, req); } + /** * @brief Async dispatch to worker thread (non-blocking) * @@ -3143,10 +2932,10 @@ static inline bool ctx_uses_async_thread(const py_context_t *ctx) { return true; } #endif - return ctx->uses_worker_thread; + return ctx->has_thread; } -static ERL_NIF_TERM dispatch_to_worker_thread_async( +static ERL_NIF_TERM ctx_dispatch_async( ErlNifEnv *env, py_context_t *ctx, ctx_request_type_t req_type, @@ -3155,22 +2944,11 @@ static ERL_NIF_TERM dispatch_to_worker_thread_async( ERL_NIF_TERM request_id, void *local_env ) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); + ERL_NIF_TERM err; + ctx_request_t *req = ctx_request_begin(env, ctx, req_type, &err); if (req == NULL) { - return make_error(env, "alloc_failed"); + return err; } - - /* Populate request */ - req->type = req_type; req->request_data = enif_make_copy(req->request_env, request_data); req->local_env_ptr = local_env; @@ -3198,7 +2976,7 @@ static ERL_NIF_TERM dispatch_to_worker_thread_async( * The queue-based pattern replaces the old single-slot pattern which had race * conditions when multiple callers dispatched concurrently. */ -static void *owngil_context_thread_main(void *arg) { +static void *ctx_thread_main_owngil(void *arg) { py_context_t *ctx = (py_context_t *)arg; /* Attach to Python runtime to create the subinterpreter. @@ -3222,7 +3000,7 @@ static void *owngil_context_thread_main(void *arg) { status.err_msg ? status.err_msg : "unknown error"); PyGILState_Release(gstate); atomic_store(&ctx->init_error, true); - atomic_store(&ctx->worker_running, false); + atomic_store(&ctx->thread_running, false); return NULL; } @@ -3238,7 +3016,7 @@ static void *owngil_context_thread_main(void *arg) { PyErr_Print(); Py_EndInterpreter(ctx->own_gil_tstate); atomic_store(&ctx->init_error, true); - atomic_store(&ctx->worker_running, false); + atomic_store(&ctx->thread_running, false); return NULL; } @@ -3251,7 +3029,7 @@ static void *owngil_context_thread_main(void *arg) { PyErr_Print(); Py_EndInterpreter(ctx->own_gil_tstate); atomic_store(&ctx->init_error, true); - atomic_store(&ctx->worker_running, false); + atomic_store(&ctx->thread_running, false); return NULL; } ctx->event_loop = get_current_interpreter_event_loop(); @@ -3271,7 +3049,7 @@ static void *owngil_context_thread_main(void *arg) { Py_XDECREF(ctx->module_cache); Py_EndInterpreter(ctx->own_gil_tstate); atomic_store(&ctx->init_error, true); - atomic_store(&ctx->worker_running, false); + atomic_store(&ctx->thread_running, false); return NULL; } @@ -3293,7 +3071,7 @@ static void *owngil_context_thread_main(void *arg) { PyEval_SaveThread(); /* Signal that we're ready */ - atomic_store(&ctx->worker_running, true); + atomic_store(&ctx->thread_running, true); /* Main request loop - uses queue instead of single-slot */ while (!atomic_load(&ctx->shutdown_requested)) { @@ -3360,7 +3138,7 @@ static void *owngil_context_thread_main(void *arg) { * invariant on py_context::interrupt_mutex). */ py_context_exec_enter(ctx); PyEval_RestoreThread(ctx->own_gil_tstate); - owngil_execute_request(ctx); + ctx_execute_request(ctx); PyEval_SaveThread(); py_context_exec_leave(ctx); @@ -3443,7 +3221,7 @@ static void *owngil_context_thread_main(void *arg) { * After Py_NewInterpreterFromConfig switched us to the OWN_GIL interpreter, * the original gstate is no longer valid. Py_EndInterpreter handles cleanup. */ - atomic_store(&ctx->worker_running, false); + atomic_store(&ctx->thread_running, false); return NULL; } @@ -3451,755 +3229,83 @@ static void *owngil_context_thread_main(void *arg) { * Timeout for OWN_GIL dispatch in seconds. * If worker thread doesn't respond within this time, assume it's dead. */ -#define OWNGIL_DISPATCH_TIMEOUT_SECS 30 + /** - * @brief Dispatch a request to the worker thread and wait for response - * - * Uses the queue-based pattern: creates a request, enqueues it, waits for - * completion, and copies the result back to the caller's environment. + * @brief Run the reactor on_read_ready handler on the context thread * - * This replaces the old single-slot pattern which had race conditions when - * multiple callers dispatched concurrently. - * - * @param env Caller's NIF environment - * @param ctx Context with worker thread - * @param req_type Request type (CTX_REQ_CALL, CTX_REQ_EVAL, CTX_REQ_EXEC, etc.) - * @param request_data Request data term - * @return Result term copied back to caller's env + * @param buffer_ptr Reactor buffer resource; ownership moves to the request. */ -static ERL_NIF_TERM dispatch_to_owngil_thread( - ErlNifEnv *env, - py_context_t *ctx, - ctx_request_type_t req_type, - ERL_NIF_TERM request_data -) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); +ERL_NIF_TERM dispatch_reactor_read(ErlNifEnv *env, py_context_t *ctx, + int fd, void *buffer_ptr) { + ERL_NIF_TERM err; + ctx_request_t *req = ctx_request_begin(env, ctx, CTX_REQ_REACTOR_ON_READ_READY, &err); + if (req == NULL) { + return err; } + req->request_data = enif_make_int(req->request_env, fd); + req->reactor_buffer_ptr = buffer_ptr; + req->reactor_fd = fd; + return ctx_dispatch_wait(env, ctx, req); +} - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); +/** @brief Run the reactor on_write_ready handler on the context thread */ +ERL_NIF_TERM dispatch_reactor_write(ErlNifEnv *env, py_context_t *ctx, int fd) { + ERL_NIF_TERM err; + ctx_request_t *req = ctx_request_begin(env, ctx, CTX_REQ_REACTOR_ON_WRITE_READY, &err); + if (req == NULL) { + return err; } + req->request_data = enif_make_int(req->request_env, fd); + req->reactor_fd = fd; + return ctx_dispatch_wait(env, ctx, req); +} - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); +/** @brief Run the reactor init_connection handler on the context thread */ +ERL_NIF_TERM dispatch_reactor_init(ErlNifEnv *env, py_context_t *ctx, + int fd, ERL_NIF_TERM client_info) { + ERL_NIF_TERM err; + ctx_request_t *req = ctx_request_begin(env, ctx, CTX_REQ_REACTOR_INIT_CONNECTION, &err); if (req == NULL) { - return make_error(env, "alloc_failed"); + return err; } + req->request_data = enif_make_tuple2(req->request_env, + enif_make_int(req->request_env, fd), + enif_make_copy(req->request_env, client_info)); + req->reactor_fd = fd; + return ctx_dispatch_wait(env, ctx, req); +} - /* Populate request */ - req->type = req_type; - req->request_data = enif_make_copy(req->request_env, request_data); - - /* Add ref for queue (now refcount = 2: caller + queue) */ - ctx_request_addref(req); - - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - /* Worker thread is unresponsive - mark request as cancelled */ - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - /* Don't mark worker as dead - it might still be processing - * a long-running Python operation. Just fail this request. */ - fprintf(stderr, "OWN_GIL dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - ctx_request_release(req); /* Release caller's ref */ - return make_error(env, "worker_timeout"); - } - } - pthread_mutex_unlock(&req->mutex); - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - /* Release caller's ref */ - ctx_request_release(req); - return result; -} +#endif /* HAVE_SUBINTERPRETERS */ /** - * @brief Dispatch reactor on_read_ready to OWN_GIL thread + * @brief Initialize OWN_GIL fields in a context and start the worker thread * - * Uses queue-based dispatch with per-request synchronization. + * @param ctx Context to initialize + * @return 0 on success, -1 on failure */ -ERL_NIF_TERM dispatch_reactor_read_to_owngil(ErlNifEnv *env, py_context_t *ctx, - int fd, void *buffer_ptr) { - if (!atomic_load(&ctx->worker_running)) { - enif_release_resource(buffer_ptr); - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - enif_release_resource(buffer_ptr); - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); - if (req == NULL) { - enif_release_resource(buffer_ptr); - return make_error(env, "alloc_failed"); - } - - /* Populate request */ - req->type = CTX_REQ_REACTOR_ON_READ_READY; - req->request_data = enif_make_int(req->request_env, fd); - req->reactor_buffer_ptr = buffer_ptr; /* Transfer ownership */ - req->reactor_fd = fd; +#ifdef HAVE_SUBINTERPRETERS +static int owngil_context_init(py_context_t *ctx) { + ctx->uses_own_gil = true; + ctx->own_gil_tstate = NULL; + ctx->own_gil_interp = NULL; + ctx->event_loop = NULL; - /* Add ref for queue (now refcount = 2: caller + queue) */ - ctx_request_addref(req); + /* Initialize worker thread state */ + atomic_store(&ctx->thread_running, false); + atomic_store(&ctx->init_error, false); + atomic_store(&ctx->shutdown_requested, false); + atomic_store(&ctx->leaked, false); - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - /* Request timeout - mark as cancelled but don't release buffer - * (worker will handle it when it gets to this request) */ - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - - fprintf(stderr, "OWN_GIL reactor dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - - ctx_request_release(req); /* Release caller's ref */ - return make_error(env, "worker_timeout"); - } - } - - pthread_mutex_unlock(&req->mutex); - - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - /* Release caller's ref */ - ctx_request_release(req); - - return result; -} - -/** - * @brief Dispatch reactor on_write_ready to OWN_GIL thread - * - * Uses queue-based dispatch with per-request synchronization. - */ -ERL_NIF_TERM dispatch_reactor_write_to_owngil(ErlNifEnv *env, py_context_t *ctx, - int fd) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); - if (req == NULL) { - return make_error(env, "alloc_failed"); - } - - /* Populate request */ - req->type = CTX_REQ_REACTOR_ON_WRITE_READY; - req->request_data = enif_make_int(req->request_env, fd); - req->reactor_fd = fd; - - /* Add ref for queue (now refcount = 2: caller + queue) */ - ctx_request_addref(req); - - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - - fprintf(stderr, "OWN_GIL reactor write dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - - ctx_request_release(req); - return make_error(env, "worker_timeout"); - } - } - - pthread_mutex_unlock(&req->mutex); - - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - ctx_request_release(req); - - return result; -} - -/** - * @brief Dispatch reactor init_connection to OWN_GIL thread - * - * Uses queue-based dispatch with per-request synchronization. - */ -ERL_NIF_TERM dispatch_reactor_init_to_owngil(ErlNifEnv *env, py_context_t *ctx, - int fd, ERL_NIF_TERM client_info) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); - if (req == NULL) { - return make_error(env, "alloc_failed"); - } - - /* Populate request */ - req->type = CTX_REQ_REACTOR_INIT_CONNECTION; - ERL_NIF_TERM fd_term = enif_make_int(req->request_env, fd); - ERL_NIF_TERM info_copy = enif_make_copy(req->request_env, client_info); - req->request_data = enif_make_tuple2(req->request_env, fd_term, info_copy); - req->reactor_fd = fd; - - /* Add ref for queue (now refcount = 2: caller + queue) */ - ctx_request_addref(req); - - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - - fprintf(stderr, "OWN_GIL reactor init dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - - ctx_request_release(req); - return make_error(env, "worker_timeout"); - } - } - - pthread_mutex_unlock(&req->mutex); - - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - ctx_request_release(req); - - return result; -} - -/** - * @brief Dispatch exec_with_env to OWN_GIL thread - * - * Uses queue-based dispatch with per-request synchronization. - */ -static ERL_NIF_TERM dispatch_exec_with_env_to_owngil( - ErlNifEnv *env, py_context_t *ctx, - ERL_NIF_TERM code, py_env_resource_t *penv -) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); - if (req == NULL) { - return make_error(env, "alloc_failed"); - } - - /* Populate request */ - req->type = CTX_REQ_EXEC_WITH_ENV; - req->request_data = enif_make_copy(req->request_env, code); - req->local_env_ptr = penv; - - /* Add ref for queue */ - ctx_request_addref(req); - - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - - fprintf(stderr, "OWN_GIL exec_with_env dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - - ctx_request_release(req); - return make_error(env, "worker_timeout"); - } - } - - pthread_mutex_unlock(&req->mutex); - - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - ctx_request_release(req); - - return result; -} - -/** - * @brief Dispatch eval_with_env to OWN_GIL thread - * - * Uses queue-based dispatch with per-request synchronization. - */ -static ERL_NIF_TERM dispatch_eval_with_env_to_owngil( - ErlNifEnv *env, py_context_t *ctx, - ERL_NIF_TERM code, ERL_NIF_TERM locals, - py_env_resource_t *penv -) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); - if (req == NULL) { - return make_error(env, "alloc_failed"); - } - - /* Populate request: {Code, Locals} */ - req->type = CTX_REQ_EVAL_WITH_ENV; - ERL_NIF_TERM code_copy = enif_make_copy(req->request_env, code); - ERL_NIF_TERM locals_copy = enif_make_copy(req->request_env, locals); - req->request_data = enif_make_tuple2(req->request_env, code_copy, locals_copy); - req->local_env_ptr = penv; - - /* Add ref for queue */ - ctx_request_addref(req); - - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - - fprintf(stderr, "OWN_GIL eval_with_env dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - - ctx_request_release(req); - return make_error(env, "worker_timeout"); - } - } - - pthread_mutex_unlock(&req->mutex); - - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - ctx_request_release(req); - - return result; -} - -/** - * @brief Dispatch call_with_env to OWN_GIL thread - * - * Uses queue-based dispatch with per-request synchronization. - */ -static ERL_NIF_TERM dispatch_call_with_env_to_owngil( - ErlNifEnv *env, py_context_t *ctx, - ERL_NIF_TERM module, ERL_NIF_TERM func, - ERL_NIF_TERM args, ERL_NIF_TERM kwargs, - py_env_resource_t *penv -) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); - if (req == NULL) { - return make_error(env, "alloc_failed"); - } - - /* Populate request: {Module, Func, Args, Kwargs} */ - req->type = CTX_REQ_CALL_WITH_ENV; - ERL_NIF_TERM module_copy = enif_make_copy(req->request_env, module); - ERL_NIF_TERM func_copy = enif_make_copy(req->request_env, func); - ERL_NIF_TERM args_copy = enif_make_copy(req->request_env, args); - ERL_NIF_TERM kwargs_copy = enif_make_copy(req->request_env, kwargs); - req->request_data = enif_make_tuple4(req->request_env, - module_copy, func_copy, args_copy, kwargs_copy); - req->local_env_ptr = penv; - - /* Add ref for queue */ - ctx_request_addref(req); - - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - - fprintf(stderr, "OWN_GIL call_with_env dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - - ctx_request_release(req); - return make_error(env, "worker_timeout"); - } - } - - pthread_mutex_unlock(&req->mutex); - - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - ctx_request_release(req); - - return result; -} - -/** - * @brief Dispatch create_local_env to OWN_GIL thread - * - * Uses queue-based dispatch with per-request synchronization. - */ -static ERL_NIF_TERM dispatch_create_local_env_to_owngil( - ErlNifEnv *env, py_context_t *ctx, - py_env_resource_t *res -) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); - if (req == NULL) { - return make_error(env, "alloc_failed"); - } - - /* Populate request */ - req->type = CTX_REQ_CREATE_LOCAL_ENV; - req->local_env_ptr = res; - - /* Add ref for queue */ - ctx_request_addref(req); - - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - - fprintf(stderr, "OWN_GIL create_local_env dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - - ctx_request_release(req); - return make_error(env, "worker_timeout"); - } - } - - pthread_mutex_unlock(&req->mutex); - - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - ctx_request_release(req); - - return result; -} - -/** - * @brief Dispatch apply_imports to OWN_GIL worker thread - * - * Uses queue-based dispatch with per-request synchronization. - */ -static ERL_NIF_TERM dispatch_apply_imports_to_owngil( - ErlNifEnv *env, py_context_t *ctx, ERL_NIF_TERM imports_term -) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); - if (req == NULL) { - return make_error(env, "alloc_failed"); - } - - /* Populate request */ - req->type = CTX_REQ_APPLY_IMPORTS; - req->request_data = enif_make_copy(req->request_env, imports_term); - - /* Add ref for queue */ - ctx_request_addref(req); - - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - - fprintf(stderr, "OWN_GIL apply_imports dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - - ctx_request_release(req); - return make_error(env, "worker_timeout"); - } - } - - pthread_mutex_unlock(&req->mutex); - - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - ctx_request_release(req); - - return result; -} - -/** - * @brief Dispatch apply_paths request to OWN_GIL worker thread - * - * Uses queue-based dispatch with per-request synchronization. - */ -static ERL_NIF_TERM dispatch_apply_paths_to_owngil( - ErlNifEnv *env, py_context_t *ctx, ERL_NIF_TERM paths_term -) { - if (!atomic_load(&ctx->worker_running)) { - return make_error(env, "thread_not_running"); - } - - if (atomic_load(&ctx->destroyed)) { - return make_error(env, "context_destroyed"); - } - - /* Create request struct */ - ctx_request_t *req = ctx_request_create(); - if (req == NULL) { - return make_error(env, "alloc_failed"); - } - - /* Populate request */ - req->type = CTX_REQ_APPLY_PATHS; - req->request_data = enif_make_copy(req->request_env, paths_term); - - /* Add ref for queue */ - ctx_request_addref(req); - - /* Enqueue the request */ - ctx_queue_enqueue(ctx, req); - - /* Wait for completion with timeout */ - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS; - - ERL_NIF_TERM result; - pthread_mutex_lock(&req->mutex); - - while (!atomic_load(&req->completed)) { - int rc = pthread_cond_timedwait(&req->cond, &req->mutex, &deadline); - if (rc == ETIMEDOUT) { - atomic_store(&req->cancelled, true); - pthread_mutex_unlock(&req->mutex); - - fprintf(stderr, "OWN_GIL apply_paths dispatch timeout after %d seconds\n", - OWNGIL_DISPATCH_TIMEOUT_SECS); - - ctx_request_release(req); - return make_error(env, "worker_timeout"); - } - } - - pthread_mutex_unlock(&req->mutex); - - /* Copy result to caller's env */ - if (req->result_env != NULL) { - result = enif_make_copy(env, req->result); - } else { - result = make_error(env, "no_result"); - } - - ctx_request_release(req); - - return result; -} - -#endif /* HAVE_SUBINTERPRETERS */ - -/** - * @brief Initialize OWN_GIL fields in a context and start the worker thread - * - * @param ctx Context to initialize - * @return 0 on success, -1 on failure - */ -#ifdef HAVE_SUBINTERPRETERS -static int owngil_context_init(py_context_t *ctx) { - ctx->uses_own_gil = true; - ctx->own_gil_tstate = NULL; - ctx->own_gil_interp = NULL; - ctx->event_loop = NULL; - - /* Initialize worker thread state */ - atomic_store(&ctx->worker_running, false); - atomic_store(&ctx->init_error, false); - atomic_store(&ctx->shutdown_requested, false); - atomic_store(&ctx->leaked, false); - - /* Initialize request queue */ - ctx->queue_head = NULL; - ctx->queue_tail = NULL; + /* Initialize request queue */ + ctx->queue_head = NULL; + ctx->queue_tail = NULL; /* Initialize legacy compatibility fields */ ctx->shared_env = NULL; @@ -4230,7 +3336,7 @@ static int owngil_context_init(py_context_t *ctx) { } /* Start the worker thread */ - if (pthread_create(&ctx->worker_thread, NULL, owngil_context_thread_main, ctx) != 0) { + if (pthread_create(&ctx->thread, NULL, ctx_thread_main_owngil, ctx) != 0) { enif_free_env(ctx->msg_env); ctx->msg_env = NULL; pthread_cond_destroy(&ctx->queue_not_empty); @@ -4240,16 +3346,16 @@ static int owngil_context_init(py_context_t *ctx) { /* Wait for thread to initialize or fail */ int wait_count = 0; - while (!atomic_load(&ctx->worker_running) && + while (!atomic_load(&ctx->thread_running) && !atomic_load(&ctx->init_error) && wait_count < 2000) { usleep(1000); /* 1ms */ wait_count++; } - if (atomic_load(&ctx->init_error) || !atomic_load(&ctx->worker_running)) { + if (atomic_load(&ctx->init_error) || !atomic_load(&ctx->thread_running)) { /* Thread failed to start */ - pthread_join(ctx->worker_thread, NULL); + pthread_join(ctx->thread, NULL); if (ctx->msg_env != NULL) { enif_free_env(ctx->msg_env); ctx->msg_env = NULL; @@ -4273,13 +3379,13 @@ static int owngil_context_init(py_context_t *ctx) { */ #define OWNGIL_SHUTDOWN_TIMEOUT_SECS 30 -static void owngil_context_shutdown(py_context_t *ctx) { +static void ctx_thread_shutdown_owngil(py_context_t *ctx) { if (!ctx->uses_own_gil) { return; } /* Signal shutdown and wake any worker parked on the condvar. - * See worker_context_shutdown for why we broadcast instead of + * See ctx_thread_shutdown_worker for why we broadcast instead of * enqueuing a CTX_REQ_SHUTDOWN sentinel. */ atomic_store(&ctx->shutdown_requested, true); ctx_queue_cancel_all(ctx); @@ -4294,18 +3400,18 @@ static void owngil_context_shutdown(py_context_t *ctx) { struct timespec deadline; clock_gettime(CLOCK_REALTIME, &deadline); deadline.tv_sec += OWNGIL_SHUTDOWN_TIMEOUT_SECS; - int rc = pthread_timedjoin_np(ctx->worker_thread, NULL, &deadline); + int rc = pthread_timedjoin_np(ctx->thread, NULL, &deadline); join_succeeded = (rc == 0); #else - /* macOS/other: poll worker_running flag with timeout */ + /* macOS/other: poll thread_running flag with timeout */ int wait_ms = 0; - while (atomic_load(&ctx->worker_running) && + while (atomic_load(&ctx->thread_running) && wait_ms < OWNGIL_SHUTDOWN_TIMEOUT_SECS * 1000) { usleep(100000); /* 100ms */ wait_ms += 100; } - if (!atomic_load(&ctx->worker_running)) { - pthread_join(ctx->worker_thread, NULL); + if (!atomic_load(&ctx->thread_running)) { + pthread_join(ctx->thread, NULL); join_succeeded = true; } #endif @@ -4313,7 +3419,7 @@ static void owngil_context_shutdown(py_context_t *ctx) { if (!join_succeeded) { /* Worker thread is unresponsive - leak the context. Pin the * resource so the BEAM doesn't free its memory under the - * stuck pthread (UAF). See worker_context_shutdown for the + * stuck pthread (UAF). See ctx_thread_shutdown_worker for the * full rationale. */ fprintf(stderr, "OWN_GIL shutdown timeout after %d seconds, leaking context\n", OWNGIL_SHUTDOWN_TIMEOUT_SECS); @@ -4401,7 +3507,7 @@ static ERL_NIF_TERM nif_context_create(ErlNifEnv *env, int argc, const ERL_NIF_T ctx->globals = NULL; ctx->locals = NULL; ctx->module_cache = NULL; - ctx->uses_worker_thread = false; + ctx->has_thread = false; /* Interrupt support */ ctx->interrupt_mutex_init = (pthread_mutex_init(&ctx->interrupt_mutex, NULL) == 0); @@ -4640,7 +3746,7 @@ static ERL_NIF_TERM nif_context_destroy(ErlNifEnv *env, int argc, const ERL_NIF_ #ifdef HAVE_SUBINTERPRETERS /* OWN_GIL mode: shutdown the dedicated thread */ if (ctx->uses_own_gil) { - owngil_context_shutdown(ctx); + ctx_thread_shutdown_owngil(ctx); /* Close callback pipes only on a clean shutdown. If the * worker timed out (ctx->leaked == true) it may still write * to / read from these fds; closing them here would let the @@ -4662,8 +3768,8 @@ static ERL_NIF_TERM nif_context_destroy(ErlNifEnv *env, int argc, const ERL_NIF_ #endif /* Worker mode: shutdown the dedicated worker thread */ - if (ctx->uses_worker_thread) { - worker_context_shutdown(ctx); + if (ctx->has_thread) { + ctx_thread_shutdown_worker(ctx); /* Close callback pipes (see OWN_GIL branch for why this is * gated on !ctx->leaked). */ if (!atomic_load(&ctx->leaked)) { @@ -4736,36 +3842,15 @@ static ERL_NIF_TERM nif_context_call(ErlNifEnv *env, int argc, const ERL_NIF_TER return make_error(env, "invalid_context"); } -#ifdef HAVE_SUBINTERPRETERS - /* OWN_GIL mode: dispatch to dedicated thread */ - if (ctx->uses_own_gil) { - /* Build request tuple: {Module, Func, Args, Kwargs} */ - ERL_NIF_TERM kwargs = (argc > 4 && enif_is_map(env, argv[4])) - ? argv[4] : enif_make_new_map(env); - ERL_NIF_TERM request = enif_make_tuple4(env, - argv[1], /* Module */ - argv[2], /* Func */ - argv[3], /* Args */ - kwargs); - return dispatch_to_owngil_thread(env, ctx, CTX_REQ_CALL, request); - } -#endif - - /* Worker thread mode: dispatch to dedicated thread */ - if (ctx->uses_worker_thread) { - /* Build request tuple: {Module, Func, Args, Kwargs} */ - ERL_NIF_TERM kwargs = (argc > 4 && enif_is_map(env, argv[4])) - ? argv[4] : enif_make_new_map(env); - ERL_NIF_TERM request = enif_make_tuple4(env, - argv[1], /* Module */ - argv[2], /* Func */ - argv[3], /* Args */ - kwargs); - return dispatch_to_worker_thread(env, ctx, CTX_REQ_CALL, request); + if (!ctx_uses_async_thread(ctx)) { + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); } - - /* Every context created by nif_context_create has a thread */ - return make_error(env, "context_has_no_thread"); + /* Request tuple: {Module, Func, Args, Kwargs} */ + ERL_NIF_TERM kwargs = (argc > 4 && enif_is_map(env, argv[4])) + ? argv[4] : enif_make_new_map(env); + ERL_NIF_TERM request = enif_make_tuple4(env, argv[1], argv[2], argv[3], kwargs); + return ctx_dispatch(env, ctx, CTX_REQ_CALL, request, NULL); } /** @@ -4810,12 +3895,12 @@ static ERL_NIF_TERM nif_context_call_async(ErlNifEnv *env, int argc, const ERL_N argv[4], /* Func */ argv[5], /* Args */ kwargs); - return dispatch_to_worker_thread_async(env, ctx, CTX_REQ_CALL, + return ctx_dispatch_async(env, ctx, CTX_REQ_CALL, request, caller_pid, request_id, NULL); } /* Not using worker thread - fall back to blocking call */ - return make_error(env, "async_requires_worker_thread"); + return make_error(env, "context_has_no_thread"); } /** @@ -4856,12 +3941,12 @@ static ERL_NIF_TERM nif_context_eval_async(ErlNifEnv *env, int argc, const ERL_N ERL_NIF_TERM locals = (argc > 4 && enif_is_map(env, argv[4])) ? argv[4] : enif_make_new_map(env); ERL_NIF_TERM request = enif_make_tuple2(env, argv[3], locals); - return dispatch_to_worker_thread_async(env, ctx, CTX_REQ_EVAL, + return ctx_dispatch_async(env, ctx, CTX_REQ_EVAL, request, caller_pid, request_id, NULL); } /* Not using worker thread - fall back to blocking call */ - return make_error(env, "async_requires_worker_thread"); + return make_error(env, "context_has_no_thread"); } /** @@ -4898,12 +3983,12 @@ static ERL_NIF_TERM nif_context_exec_async(ErlNifEnv *env, int argc, const ERL_N /* Dedicated thread (worker or OWN_GIL): dispatch async */ if (ctx_uses_async_thread(ctx)) { - return dispatch_to_worker_thread_async(env, ctx, CTX_REQ_EXEC, + return ctx_dispatch_async(env, ctx, CTX_REQ_EXEC, argv[3], caller_pid, request_id, NULL); } /* Not using worker thread - fall back to blocking call */ - return make_error(env, "async_requires_worker_thread"); + return make_error(env, "context_has_no_thread"); } /** @@ -4941,7 +4026,7 @@ static ERL_NIF_TERM nif_context_call_with_env_async(ErlNifEnv *env, int argc, } if (!ctx_uses_async_thread(ctx)) { - return make_error(env, "async_requires_worker_thread"); + return make_error(env, "context_has_no_thread"); } ERL_NIF_TERM kwargs = enif_is_map(env, argv[6]) @@ -4951,7 +4036,7 @@ static ERL_NIF_TERM nif_context_call_with_env_async(ErlNifEnv *env, int argc, argv[4], /* Func */ argv[5], /* Args */ kwargs); - return dispatch_to_worker_thread_async(env, ctx, CTX_REQ_CALL_WITH_ENV, + return ctx_dispatch_async(env, ctx, CTX_REQ_CALL_WITH_ENV, request, caller_pid, request_id, penv); } @@ -4986,13 +4071,13 @@ static ERL_NIF_TERM nif_context_eval_with_env_async(ErlNifEnv *env, int argc, } if (!ctx_uses_async_thread(ctx)) { - return make_error(env, "async_requires_worker_thread"); + return make_error(env, "context_has_no_thread"); } ERL_NIF_TERM locals = enif_is_map(env, argv[4]) ? argv[4] : enif_make_new_map(env); ERL_NIF_TERM request = enif_make_tuple2(env, argv[3], locals); - return dispatch_to_worker_thread_async(env, ctx, CTX_REQ_EVAL_WITH_ENV, + return ctx_dispatch_async(env, ctx, CTX_REQ_EVAL_WITH_ENV, request, caller_pid, request_id, penv); } @@ -5027,10 +4112,10 @@ static ERL_NIF_TERM nif_context_exec_with_env_async(ErlNifEnv *env, int argc, } if (!ctx_uses_async_thread(ctx)) { - return make_error(env, "async_requires_worker_thread"); + return make_error(env, "context_has_no_thread"); } - return dispatch_to_worker_thread_async(env, ctx, CTX_REQ_EXEC_WITH_ENV, + return ctx_dispatch_async(env, ctx, CTX_REQ_EXEC_WITH_ENV, argv[3], caller_pid, request_id, penv); } @@ -5055,28 +4140,15 @@ static ERL_NIF_TERM nif_context_eval(ErlNifEnv *env, int argc, const ERL_NIF_TER return make_error(env, "invalid_context"); } -#ifdef HAVE_SUBINTERPRETERS - /* OWN_GIL mode: dispatch to dedicated thread */ - if (ctx->uses_own_gil) { - /* Build request tuple: {Code, Locals} */ - ERL_NIF_TERM locals = (argc > 2 && enif_is_map(env, argv[2])) - ? argv[2] : enif_make_new_map(env); - ERL_NIF_TERM request = enif_make_tuple2(env, argv[1], locals); - return dispatch_to_owngil_thread(env, ctx, CTX_REQ_EVAL, request); - } -#endif - - /* Worker thread mode: dispatch to dedicated thread */ - if (ctx->uses_worker_thread) { - /* Build request tuple: {Code, Locals} */ - ERL_NIF_TERM locals = (argc > 2 && enif_is_map(env, argv[2])) - ? argv[2] : enif_make_new_map(env); - ERL_NIF_TERM request = enif_make_tuple2(env, argv[1], locals); - return dispatch_to_worker_thread(env, ctx, CTX_REQ_EVAL, request); - } - - /* Every context created by nif_context_create has a thread */ - return make_error(env, "context_has_no_thread"); + if (!ctx_uses_async_thread(ctx)) { + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); + } + /* Request tuple: {Code, Locals} */ + ERL_NIF_TERM locals = (argc > 2 && enif_is_map(env, argv[2])) + ? argv[2] : enif_make_new_map(env); + ERL_NIF_TERM request = enif_make_tuple2(env, argv[1], locals); + return ctx_dispatch(env, ctx, CTX_REQ_EVAL, request, NULL); } /** @@ -5098,20 +4170,11 @@ static ERL_NIF_TERM nif_context_exec(ErlNifEnv *env, int argc, const ERL_NIF_TER return make_error(env, "invalid_context"); } -#ifdef HAVE_SUBINTERPRETERS - /* OWN_GIL mode: dispatch to dedicated thread */ - if (ctx->uses_own_gil) { - return dispatch_to_owngil_thread(env, ctx, CTX_REQ_EXEC, argv[1]); - } -#endif - - /* Worker thread mode: dispatch to dedicated thread */ - if (ctx->uses_worker_thread) { - return dispatch_to_worker_thread(env, ctx, CTX_REQ_EXEC, argv[1]); + if (!ctx_uses_async_thread(ctx)) { + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); } - - /* Every context created by nif_context_create has a thread */ - return make_error(env, "context_has_no_thread"); + return ctx_dispatch(env, ctx, CTX_REQ_EXEC, argv[1], NULL); } /* ============================================================================ @@ -5152,79 +4215,32 @@ static ERL_NIF_TERM nif_create_local_env(ErlNifEnv *env, int argc, const ERL_NIF res->locals = NULL; res->interp_id = 0; -#ifdef HAVE_SUBINTERPRETERS - /* OWN_GIL mode: dispatch to the dedicated thread to create dicts */ - if (ctx->uses_own_gil) { - ERL_NIF_TERM dispatch_result = dispatch_create_local_env_to_owngil(env, ctx, res); - - /* Check if dispatch succeeded */ - ERL_NIF_TERM error_atom = enif_make_atom(env, "error"); - const ERL_NIF_TERM *tuple_elems; - int arity; - if (enif_get_tuple(env, dispatch_result, &arity, &tuple_elems) && - arity == 2 && enif_is_identical(tuple_elems[0], error_atom)) { - /* Dispatch failed - release resource and return error */ - enif_release_resource(res); - return dispatch_result; - } - - /* Success - return the resource */ - ERL_NIF_TERM ref = enif_make_resource(env, res); - enif_release_resource(res); /* Ref now owns it */ - return enif_make_tuple2(env, ATOM_OK, ref); - } -#endif - - /* Acquire context to switch to correct interpreter */ - py_context_guard_t guard = py_context_acquire(ctx); - if (!guard.acquired) { + if (!ctx_uses_async_thread(ctx)) { enif_release_resource(res); - return make_error(env, "acquire_failed"); + return make_error(env, "context_has_no_thread"); } - /* Copy globals from context to inherit preloaded code */ - res->globals = PyDict_Copy(ctx->globals); - if (res->globals == NULL) { - py_context_release(&guard); + /* The dicts are created on the context thread so they belong to the + * right interpreter and allocator. */ + ERL_NIF_TERM err; + ctx_request_t *req = ctx_request_begin(env, ctx, CTX_REQ_CREATE_LOCAL_ENV, &err); + if (req == NULL) { enif_release_resource(res); - return make_error(env, "globals_copy_failed"); - } - - /* Ensure __builtins__ is present (may not be in subinterpreter mode) */ - if (PyDict_GetItemString(res->globals, "__builtins__") == NULL) { - PyObject *builtins = PyEval_GetBuiltins(); - if (builtins != NULL) { - PyDict_SetItemString(res->globals, "__builtins__", builtins); - } - } - - /* Ensure __name__ = '__main__' is set */ - if (PyDict_GetItemString(res->globals, "__name__") == NULL) { - PyObject *main_name = PyUnicode_FromString("__main__"); - if (main_name != NULL) { - PyDict_SetItemString(res->globals, "__name__", main_name); - Py_DECREF(main_name); - } + return err; } + req->local_env_ptr = res; + ERL_NIF_TERM dispatch_result = ctx_dispatch_wait(env, ctx, req); - /* Ensure erlang module is available */ - if (PyDict_GetItemString(res->globals, "erlang") == NULL) { - PyObject *erlang = PyImport_ImportModule("erlang"); - if (erlang != NULL) { - PyDict_SetItemString(res->globals, "erlang", erlang); - Py_DECREF(erlang); - } + const ERL_NIF_TERM *tuple_elems; + int arity; + if (enif_get_tuple(env, dispatch_result, &arity, &tuple_elems) && + arity == 2 && enif_is_identical(tuple_elems[0], enif_make_atom(env, "error"))) { + enif_release_resource(res); + return dispatch_result; } - /* Use the same dict for locals (module-level execution) */ - res->locals = res->globals; - Py_INCREF(res->locals); - - py_context_release(&guard); - ERL_NIF_TERM ref = enif_make_resource(env, res); enif_release_resource(res); /* Ref now owns it */ - return enif_make_tuple2(env, ATOM_OK, ref); } @@ -5257,60 +4273,11 @@ static ERL_NIF_TERM nif_interp_apply_imports(ErlNifEnv *env, int argc, const ERL return make_error(env, "context_destroyed"); } -#ifdef HAVE_SUBINTERPRETERS - /* OWN_GIL mode: dispatch to the dedicated thread */ - if (ctx->uses_own_gil) { - return dispatch_apply_imports_to_owngil(env, ctx, argv[1]); - } -#endif - - py_context_guard_t guard = py_context_acquire(ctx); - if (!guard.acquired) { - return make_error(env, "acquire_failed"); - } - - /* Process each import - imports go into interpreter's sys.modules */ - ERL_NIF_TERM head, tail = argv[1]; - int arity; - const ERL_NIF_TERM *tuple; - - while (enif_get_list_cell(env, tail, &head, &tail)) { - if (!enif_get_tuple(env, head, &arity, &tuple) || arity != 2) { - continue; - } - - ErlNifBinary module_bin; - if (!enif_inspect_binary(env, tuple[0], &module_bin)) { - continue; - } - - /* Convert to C string */ - char *module_name = enif_alloc(module_bin.size + 1); - if (module_name == NULL) continue; - memcpy(module_name, module_bin.data, module_bin.size); - module_name[module_bin.size] = '\0'; - - /* Skip __main__ */ - if (strcmp(module_name, "__main__") == 0) { - enif_free(module_name); - continue; - } - - /* Import the module - this caches in interpreter's sys.modules - * which is shared by all contexts using this interpreter */ - PyObject *mod = PyImport_ImportModule(module_name); - if (mod != NULL) { - Py_DECREF(mod); /* sys.modules holds the reference */ - } else { - /* Clear error - import failure is not fatal */ - PyErr_Clear(); - } - - enif_free(module_name); + if (!ctx_uses_async_thread(ctx)) { + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); } - - py_context_release(&guard); - return ATOM_OK; + return ctx_dispatch(env, ctx, CTX_REQ_APPLY_IMPORTS, argv[1], NULL); } /** @@ -5337,78 +4304,11 @@ static ERL_NIF_TERM nif_interp_apply_paths(ErlNifEnv *env, int argc, const ERL_N return make_error(env, "context_destroyed"); } -#ifdef HAVE_SUBINTERPRETERS - /* OWN_GIL mode: dispatch to the dedicated thread */ - if (ctx->uses_own_gil) { - return dispatch_apply_paths_to_owngil(env, ctx, argv[1]); - } -#endif - - py_context_guard_t guard = py_context_acquire(ctx); - if (!guard.acquired) { - return make_error(env, "acquire_failed"); - } - - /* Get sys.path */ - PyObject *sys_module = PyImport_ImportModule("sys"); - if (sys_module == NULL) { - py_context_release(&guard); - return make_error(env, "sys_import_failed"); - } - - PyObject *sys_path = PyObject_GetAttrString(sys_module, "path"); - Py_DECREF(sys_module); - if (sys_path == NULL || !PyList_Check(sys_path)) { - Py_XDECREF(sys_path); - py_context_release(&guard); - return make_error(env, "sys_path_not_list"); - } - - /* Process each path - insert at beginning in reverse order */ - /* First, collect all paths */ - ERL_NIF_TERM head, tail = argv[1]; - int path_count = 0; - ERL_NIF_TERM paths_list = argv[1]; - - /* Count paths */ - while (enif_get_list_cell(env, tail, &head, &tail)) { - path_count++; - } - - /* Insert in reverse order so first path ends up first */ - tail = paths_list; - for (int i = 0; i < path_count; i++) { - /* Skip to the i-th element from the end */ - ERL_NIF_TERM current = paths_list; - for (int j = 0; j < path_count - 1 - i; j++) { - enif_get_list_cell(env, current, &head, ¤t); - } - enif_get_list_cell(env, current, &head, ¤t); - - ErlNifBinary path_bin; - if (!enif_inspect_binary(env, head, &path_bin)) { - continue; - } - - /* Convert to Python string */ - PyObject *path_str = PyUnicode_FromStringAndSize((char *)path_bin.data, path_bin.size); - if (path_str == NULL) { - PyErr_Clear(); - continue; - } - - /* Check if already in sys.path */ - int already_present = PySequence_Contains(sys_path, path_str); - if (already_present <= 0) { - /* Insert at position 0 */ - PyList_Insert(sys_path, 0, path_str); - } - Py_DECREF(path_str); + if (!ctx_uses_async_thread(ctx)) { + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); } - - Py_DECREF(sys_path); - py_context_release(&guard); - return ATOM_OK; + return ctx_dispatch(env, ctx, CTX_REQ_APPLY_PATHS, argv[1], NULL); } /** @@ -5428,79 +4328,29 @@ static ERL_NIF_TERM nif_context_exec_with_env(ErlNifEnv *env, int argc, const ER py_context_t *ctx; py_env_resource_t *penv; - if (!runtime_is_running()) { - return make_error(env, "python_not_running"); - } - - if (!enif_get_resource(env, argv[0], PY_CONTEXT_RESOURCE_TYPE, (void **)&ctx)) { - return make_error(env, "invalid_context"); - } - - ErlNifBinary code_bin; - if (!enif_inspect_binary(env, argv[1], &code_bin)) { - return make_error(env, "invalid_code"); - } - - /* Get process-local environment */ - if (!enif_get_resource(env, argv[2], PY_ENV_RESOURCE_TYPE, (void **)&penv)) { - return make_error(env, "invalid_env"); - } - -#ifdef HAVE_SUBINTERPRETERS - /* OWN_GIL mode: dispatch to the dedicated thread */ - if (ctx->uses_own_gil) { - return dispatch_exec_with_env_to_owngil(env, ctx, argv[1], penv); - } -#endif - - /* Worker thread mode: dispatch to dedicated thread with local env */ - if (ctx->uses_worker_thread) { - /* For exec, we just pass the code binary */ - return dispatch_to_worker_thread_impl(env, ctx, CTX_REQ_EXEC_WITH_ENV, argv[1], penv); - } - - char *code = binary_to_string(&code_bin); - if (code == NULL) { - return make_error(env, "alloc_failed"); - } - - ERL_NIF_TERM result; - - /* Acquire thread state */ - py_context_guard_t guard = py_context_acquire(ctx); - if (!guard.acquired) { - enif_free(code); - return make_error(env, "acquire_failed"); - } - - /* Set thread-local context and env for callback/reentrant support */ - py_context_t *prev_context = tl_current_context; - tl_current_context = ctx; - py_env_resource_t *prev_local_env = tl_current_local_env; - tl_current_local_env = penv; - - /* Always use process-local environment */ - PyObject *exec_globals = penv->globals; - PyObject *exec_locals = penv->globals; - - /* Execute statements */ - PyObject *py_result = PyRun_String(code, Py_file_input, exec_globals, exec_locals); + if (!runtime_is_running()) { + return make_error(env, "python_not_running"); + } - if (py_result == NULL) { - result = make_py_error(env); - } else { - Py_DECREF(py_result); - result = ATOM_OK; + if (!enif_get_resource(env, argv[0], PY_CONTEXT_RESOURCE_TYPE, (void **)&ctx)) { + return make_error(env, "invalid_context"); } - /* Restore thread-local state */ - tl_current_context = prev_context; - tl_current_local_env = prev_local_env; + ErlNifBinary code_bin; + if (!enif_inspect_binary(env, argv[1], &code_bin)) { + return make_error(env, "invalid_code"); + } - enif_free(code); - py_context_release(&guard); + /* Get process-local environment */ + if (!enif_get_resource(env, argv[2], PY_ENV_RESOURCE_TYPE, (void **)&penv)) { + return make_error(env, "invalid_env"); + } - return result; + if (!ctx_uses_async_thread(ctx)) { + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); + } + return ctx_dispatch(env, ctx, CTX_REQ_EXEC_WITH_ENV, argv[1], penv); } /** @@ -5534,135 +4384,14 @@ static ERL_NIF_TERM nif_context_eval_with_env(ErlNifEnv *env, int argc, const ER return make_error(env, "invalid_env"); } -#ifdef HAVE_SUBINTERPRETERS - /* OWN_GIL mode: dispatch to the dedicated thread */ - if (ctx->uses_own_gil) { - return dispatch_eval_with_env_to_owngil(env, ctx, argv[1], argv[2], penv); - } -#endif - - /* Worker thread mode: dispatch to dedicated thread with local env */ - if (ctx->uses_worker_thread) { - /* Build request tuple: {Code, Locals} */ - ERL_NIF_TERM locals = (argc > 2 && enif_is_map(env, argv[2])) - ? argv[2] : enif_make_new_map(env); - ERL_NIF_TERM request = enif_make_tuple2(env, argv[1], locals); - return dispatch_to_worker_thread_impl(env, ctx, CTX_REQ_EVAL_WITH_ENV, request, penv); - } - - char *code = binary_to_string(&code_bin); - if (code == NULL) { - return make_error(env, "alloc_failed"); - } - - ERL_NIF_TERM result; - - /* Acquire thread state */ - py_context_guard_t guard = py_context_acquire(ctx); - if (!guard.acquired) { - enif_free(code); - return make_error(env, "acquire_failed"); - } - - /* Set thread-local context and env for callback/reentrant support */ - py_context_t *prev_context = tl_current_context; - tl_current_context = ctx; - py_env_resource_t *prev_local_env = tl_current_local_env; - tl_current_local_env = penv; - - /* Enable suspension for callback support */ - bool prev_allow_suspension = tl_allow_suspension; - tl_allow_suspension = true; - - /* Always use process-local environment */ - PyObject *eval_globals = penv->globals; - - /* Build locals dict from Erlang map (if provided) */ - PyObject *eval_locals = PyDict_Copy(eval_globals); - if (enif_is_map(env, argv[2])) { - ErlNifMapIterator iter; - ERL_NIF_TERM key, value; - - enif_map_iterator_create(env, argv[2], &iter, ERL_NIF_MAP_ITERATOR_FIRST); - while (enif_map_iterator_get_pair(env, &iter, &key, &value)) { - PyObject *py_key = term_to_py(env, key); - PyObject *py_value = term_to_py(env, value); - if (py_key != NULL && py_value != NULL) { - PyDict_SetItem(eval_locals, py_key, py_value); - } - Py_XDECREF(py_key); - Py_XDECREF(py_value); - enif_map_iterator_next(env, &iter); - } - enif_map_iterator_destroy(env, &iter); - } - - /* Evaluate expression */ - PyObject *py_result = PyRun_String(code, Py_eval_input, eval_globals, eval_locals); - Py_DECREF(eval_locals); - - if (py_result == NULL) { - /* Check for pending callback (flag-based detection) */ - if (tl_pending_callback) { - PyErr_Clear(); - /* Create suspended state for callback handling */ - suspended_context_state_t *suspended = create_suspended_context_state_for_eval( - env, ctx, &code_bin, argv[2]); - if (suspended == NULL) { - tl_pending_callback = false; - Py_CLEAR(tl_pending_args); - result = make_error(env, "create_suspended_state_failed"); - } else { - result = build_suspended_context_result(env, suspended); - } - } else { - result = make_py_error(env); - } - } else if (is_inline_schedule_marker(py_result)) { - /* Inline schedule marker: chain via enif_schedule_nif with local_env */ - inline_continuation_t *cont = create_inline_continuation(ctx, penv, py_result, 0); - Py_DECREF(py_result); - - if (cont == NULL) { - result = make_error(env, "create_continuation_failed"); - } else { - ERL_NIF_TERM cont_ref = enif_make_resource(env, cont); - enif_release_resource(cont); - - /* Restore thread-local state before scheduling */ - tl_allow_suspension = prev_allow_suspension; - tl_current_context = prev_context; - tl_current_local_env = prev_local_env; - clear_pending_callback_tls(); - enif_free(code); - py_context_release(&guard); - - return enif_schedule_nif(env, "inline_continuation", - ERL_NIF_DIRTY_JOB_IO_BOUND, nif_inline_continuation, 1, &cont_ref); - } - } else if (is_schedule_marker(py_result)) { - /* Schedule marker: release dirty scheduler, continue via callback */ - ScheduleMarkerObject *marker = (ScheduleMarkerObject *)py_result; - ERL_NIF_TERM callback_name = py_to_term(env, marker->callback_name); - ERL_NIF_TERM callback_args = py_to_term(env, marker->args); - Py_DECREF(py_result); - result = enif_make_tuple3(env, ATOM_SCHEDULE, callback_name, callback_args); - } else { - ERL_NIF_TERM term_result = py_to_term(env, py_result); - Py_DECREF(py_result); - result = enif_make_tuple2(env, ATOM_OK, term_result); + if (!ctx_uses_async_thread(ctx)) { + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); } - - /* Restore thread-local state */ - tl_allow_suspension = prev_allow_suspension; - tl_current_context = prev_context; - tl_current_local_env = prev_local_env; - - clear_pending_callback_tls(); - enif_free(code); - py_context_release(&guard); - - return result; + ERL_NIF_TERM locals = (argc > 2 && enif_is_map(env, argv[2])) + ? argv[2] : enif_make_new_map(env); + ERL_NIF_TERM request = enif_make_tuple2(env, argv[1], locals); + return ctx_dispatch(env, ctx, CTX_REQ_EVAL_WITH_ENV, request, penv); } /** @@ -5701,187 +4430,14 @@ static ERL_NIF_TERM nif_context_call_with_env(ErlNifEnv *env, int argc, const ER return make_error(env, "invalid_env"); } -#ifdef HAVE_SUBINTERPRETERS - /* OWN_GIL mode: dispatch to the dedicated thread */ - if (ctx->uses_own_gil) { - return dispatch_call_with_env_to_owngil(env, ctx, argv[1], argv[2], argv[3], argv[4], penv); - } -#endif - - /* Worker thread mode: dispatch to dedicated thread with local env */ - if (ctx->uses_worker_thread) { - /* Build request tuple: {Module, Func, Args, Kwargs} */ - ERL_NIF_TERM kwargs = (argc > 4 && enif_is_map(env, argv[4])) - ? argv[4] : enif_make_new_map(env); - ERL_NIF_TERM request = enif_make_tuple4(env, - argv[1], /* Module */ - argv[2], /* Func */ - argv[3], /* Args */ - kwargs); - return dispatch_to_worker_thread_impl(env, ctx, CTX_REQ_CALL_WITH_ENV, request, penv); - } - - char *module_name = binary_to_string(&module_bin); - char *func_name = binary_to_string(&func_bin); - if (module_name == NULL || func_name == NULL) { - enif_free(module_name); - enif_free(func_name); - return make_error(env, "alloc_failed"); - } - - ERL_NIF_TERM result; - - /* Acquire thread state */ - py_context_guard_t guard = py_context_acquire(ctx); - if (!guard.acquired) { - enif_free(module_name); - enif_free(func_name); - return make_error(env, "acquire_failed"); - } - - /* Set thread-local context and env for callback/reentrant support */ - py_context_t *prev_context = tl_current_context; - tl_current_context = ctx; - py_env_resource_t *prev_local_env = tl_current_local_env; - tl_current_local_env = penv; - - /* Enable suspension for callback support */ - bool prev_allow_suspension = tl_allow_suspension; - tl_allow_suspension = true; - - /* Always use process-local environment */ - PyObject *lookup_globals = penv->globals; - - PyObject *module = NULL; - PyObject *func = NULL; - - /* Special handling for __main__ module - look up in process-local globals */ - if (strcmp(module_name, "__main__") == 0) { - func = PyDict_GetItemString(lookup_globals, func_name); /* Borrowed ref */ - if (func != NULL) { - Py_INCREF(func); - } - } - - if (func == NULL) { - /* Get or import module from context cache */ - module = context_get_module(ctx, module_name); - if (module == NULL) { - result = make_py_error(env); - goto cleanup; - } - - /* Get function */ - func = PyObject_GetAttrString(module, func_name); - if (func == NULL) { - result = make_py_error(env); - goto cleanup; - } - } - - /* Convert args */ - unsigned int args_len; - if (!enif_get_list_length(env, argv[3], &args_len)) { - Py_DECREF(func); - result = make_error(env, "invalid_args"); - goto cleanup; - } - - PyObject *args = PyTuple_New(args_len); - if (args == NULL) { - Py_DECREF(func); - result = make_error(env, "alloc_failed"); - goto cleanup; - } - ERL_NIF_TERM head, tail = argv[3]; - for (unsigned int i = 0; i < args_len; i++) { - enif_get_list_cell(env, tail, &head, &tail); - PyObject *arg = term_to_py(env, head); - if (arg == NULL) { - Py_DECREF(args); - Py_DECREF(func); - result = make_error(env, "arg_conversion_failed"); - goto cleanup; - } - PyTuple_SET_ITEM(args, i, arg); - } - - /* Convert kwargs */ - PyObject *kwargs = NULL; - if (argc > 4 && enif_is_map(env, argv[4])) { - kwargs = term_to_py(env, argv[4]); - } - - /* Call the function */ - PyObject *py_result = PyObject_Call(func, args, kwargs); - Py_DECREF(func); - Py_DECREF(args); - Py_XDECREF(kwargs); - - if (py_result == NULL) { - /* Check for pending callback */ - if (tl_pending_callback) { - PyErr_Clear(); - suspended_context_state_t *suspended = create_suspended_context_state_for_call( - env, ctx, &module_bin, &func_bin, argv[3], - argc > 4 ? argv[4] : enif_make_new_map(env)); - if (suspended == NULL) { - tl_pending_callback = false; - Py_CLEAR(tl_pending_args); - result = make_error(env, "create_suspended_state_failed"); - } else { - result = build_suspended_context_result(env, suspended); - } - } else { - result = make_py_error(env); - } - } else if (is_inline_schedule_marker(py_result)) { - /* Inline schedule marker: chain via enif_schedule_nif with local_env */ - inline_continuation_t *cont = create_inline_continuation(ctx, penv, py_result, 0); - Py_DECREF(py_result); - - if (cont == NULL) { - result = make_error(env, "create_continuation_failed"); - } else { - ERL_NIF_TERM cont_ref = enif_make_resource(env, cont); - enif_release_resource(cont); - - /* Restore thread-local state before scheduling */ - tl_allow_suspension = prev_allow_suspension; - tl_current_context = prev_context; - tl_current_local_env = prev_local_env; - clear_pending_callback_tls(); - enif_free(module_name); - enif_free(func_name); - py_context_release(&guard); - - return enif_schedule_nif(env, "inline_continuation", - ERL_NIF_DIRTY_JOB_IO_BOUND, nif_inline_continuation, 1, &cont_ref); - } - } else if (is_schedule_marker(py_result)) { - ScheduleMarkerObject *marker = (ScheduleMarkerObject *)py_result; - ERL_NIF_TERM callback_name = py_to_term(env, marker->callback_name); - ERL_NIF_TERM callback_args = py_to_term(env, marker->args); - Py_DECREF(py_result); - result = enif_make_tuple3(env, ATOM_SCHEDULE, callback_name, callback_args); - } else { - ERL_NIF_TERM term_result = py_to_term(env, py_result); - Py_DECREF(py_result); - result = enif_make_tuple2(env, ATOM_OK, term_result); + if (!ctx_uses_async_thread(ctx)) { + /* Every context created by nif_context_create has a thread */ + return make_error(env, "context_has_no_thread"); } - -cleanup: - /* Restore thread-local state */ - tl_allow_suspension = prev_allow_suspension; - tl_current_context = prev_context; - tl_current_local_env = prev_local_env; - - clear_pending_callback_tls(); - enif_free(module_name); - enif_free(func_name); - py_context_release(&guard); - - return result; + ERL_NIF_TERM kwargs = (argc > 4 && enif_is_map(env, argv[4])) + ? argv[4] : enif_make_new_map(env); + ERL_NIF_TERM request = enif_make_tuple4(env, argv[1], argv[2], argv[3], kwargs); + return ctx_dispatch(env, ctx, CTX_REQ_CALL_WITH_ENV, request, penv); } /** @@ -7330,20 +5886,11 @@ static ERL_NIF_TERM nif_os_kill(ErlNifEnv *env, int argc, const ERL_NIF_TERM arg } static ErlNifFunc nif_funcs[] = { - /* Initialization */ + /* py_nif.c: runtime, contexts, process-local envs, py_ref */ {"init", 0, nif_py_init, 0}, {"init", 1, nif_py_init, 0}, {"finalize", 0, nif_finalize, 0}, - - - /* Python execution - dirty I/O NIFs */ - - /* Module operations */ - - /* Info */ {"version", 0, nif_version, 0}, - - /* Memory and GC */ {"memory_stats", 0, nif_memory_stats, 0}, {"get_debug_counters", 0, nif_get_debug_counters, 0}, {"gc", 0, nif_gc, 0}, @@ -7351,123 +5898,20 @@ static ErlNifFunc nif_funcs[] = { {"tracemalloc_start", 0, nif_tracemalloc_start, 0}, {"tracemalloc_start", 1, nif_tracemalloc_start, 0}, {"tracemalloc_stop", 0, nif_tracemalloc_stop, 0}, - - /* Callback support */ - - /* Async worker management */ - - /* Async execution - dirty I/O NIFs */ - - /* Subinterpreter capability probes */ {"subinterp_supported", 0, nif_subinterp_supported, 0}, {"owngil_supported", 0, nif_owngil_supported, 0}, - - /* OWN_GIL thread pool (used internally by py_event_loop_pool) */ {"subinterp_thread_pool_start", 0, nif_subinterp_thread_pool_start, 0}, {"subinterp_thread_pool_start", 1, nif_subinterp_thread_pool_start, 0}, {"subinterp_thread_pool_stop", 0, nif_subinterp_thread_pool_stop, 0}, {"subinterp_thread_pool_ready", 0, nif_subinterp_thread_pool_ready, 0}, {"subinterp_thread_pool_stats", 0, nif_subinterp_thread_pool_stats, 0}, - - /* OWN_GIL session management for event loop pool */ {"owngil_create_session", 1, nif_owngil_create_session, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"owngil_submit_task", 7, nif_owngil_submit_task, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"owngil_destroy_session", 2, nif_owngil_destroy_session, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"owngil_apply_imports", 3, nif_owngil_apply_imports, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"owngil_apply_paths", 3, nif_owngil_apply_paths, ERL_NIF_DIRTY_JOB_IO_BOUND}, - - /* Execution mode info */ {"execution_mode", 0, nif_execution_mode, 0}, - - /* Thread worker support (ThreadPoolExecutor). - * Writes are ERL_NIF_DIRTY_JOB_IO_BOUND because the response pipe - * has a non-blocking write end and the looped write may briefly - * wait for write-readiness when the Python reader is slow. */ - {"thread_worker_set_coordinator", 1, nif_thread_worker_set_coordinator, 0}, - {"thread_worker_write_with_id", 3, nif_thread_worker_write_with_id, - ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"thread_worker_signal_ready", 1, nif_thread_worker_signal_ready, 0}, - - /* Async callback support (for erlang.async_call). Same dirty-IO - * rationale as thread_worker_write_with_id above. */ - {"async_callback_response", 3, nif_async_callback_response, - ERL_NIF_DIRTY_JOB_IO_BOUND}, - - /* Callback name registry (prevents torch introspection issues) */ - {"register_callback_name", 1, nif_register_callback_name, 0}, - {"unregister_callback_name", 1, nif_unregister_callback_name, 0}, - - /* Logging and tracing */ - {"set_log_receiver", 2, nif_set_log_receiver, 0}, - {"clear_log_receiver", 0, nif_clear_log_receiver, 0}, - {"set_trace_receiver", 1, nif_set_trace_receiver, 0}, - {"clear_trace_receiver", 0, nif_clear_trace_receiver, 0}, - - /* Erlang-native event loop NIFs */ - {"set_event_loop_priv_dir", 1, nif_set_event_loop_priv_dir, 0}, - {"event_loop_new", 0, nif_event_loop_new, 0}, - {"event_loop_destroy", 1, nif_event_loop_destroy, 0}, - {"event_loop_set_router", 2, nif_event_loop_set_router, 0}, - {"event_loop_set_worker", 2, nif_event_loop_set_worker, 0}, - {"event_loop_set_id", 2, nif_event_loop_set_id, 0}, - {"event_loop_wakeup", 1, nif_event_loop_wakeup, 0}, - {"event_loop_run_async", 7, nif_event_loop_run_async, ERL_NIF_DIRTY_JOB_IO_BOUND}, - /* Async task queue NIFs (uvloop-inspired) */ - {"submit_task", 7, nif_submit_task, 0}, /* Thread-safe, no GIL needed */ - {"submit_task_with_env", 8, nif_submit_task_with_env, 0}, /* With process-local env */ - {"process_ready_tasks", 1, nif_process_ready_tasks, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"event_loop_set_py_loop", 2, nif_event_loop_set_py_loop, 0}, - /* Per-process namespace NIFs */ - {"event_loop_exec", 2, nif_event_loop_exec, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"event_loop_eval", 2, nif_event_loop_eval, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"add_reader", 3, nif_add_reader, 0}, - {"remove_reader", 2, nif_remove_reader, 0}, - {"add_writer", 3, nif_add_writer, 0}, - {"remove_writer", 2, nif_remove_writer, 0}, - {"call_later", 3, nif_call_later, 0}, - {"cancel_timer", 2, nif_cancel_timer, 0}, - {"poll_events", 2, nif_poll_events, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"get_pending", 1, nif_get_pending, 0}, - {"dispatch_callback", 3, nif_dispatch_callback, 0}, - {"dispatch_timer", 2, nif_dispatch_timer, 0}, - {"get_fd_callback_id", 2, nif_get_fd_callback_id, 0}, - {"reselect_reader", 2, nif_reselect_reader, 0}, - {"reselect_writer", 2, nif_reselect_writer, 0}, - {"reselect_reader_fd", 1, nif_reselect_reader_fd, 0}, - {"reselect_writer_fd", 1, nif_reselect_writer_fd, 0}, - /* FD lifecycle management (uvloop-like API) */ - {"handle_fd_event", 2, nif_handle_fd_event, 0}, - {"handle_fd_event_and_reselect", 2, nif_handle_fd_event_and_reselect, 0}, - {"fd_arm", 2, nif_fd_arm, 0}, - {"stop_reader", 1, nif_stop_reader, 0}, - {"start_reader", 1, nif_start_reader, 0}, - {"stop_writer", 1, nif_stop_writer, 0}, - {"start_writer", 1, nif_start_writer, 0}, - {"close_fd", 1, nif_close_fd, 0}, - /* Test helpers for fd monitoring (using pipes) */ - {"create_test_pipe", 0, nif_create_test_pipe, 0}, - {"close_test_fd", 1, nif_close_test_fd, 0}, - {"dup_fd", 1, nif_dup_fd, 0}, {"os_kill", 2, nif_os_kill, 0}, - {"write_test_fd", 2, nif_write_test_fd, 0}, - {"read_test_fd", 2, nif_read_test_fd, 0}, - /* TCP test helpers */ - {"create_test_tcp_listener", 1, nif_create_test_tcp_listener, 0}, - {"accept_test_tcp", 1, nif_accept_test_tcp, 0}, - {"connect_test_tcp", 2, nif_connect_test_tcp, 0}, - /* UDP test helpers */ - {"create_test_udp_socket", 1, nif_create_test_udp_socket, 0}, - {"recvfrom_test_udp", 2, nif_recvfrom_test_udp, 0}, - {"sendto_test_udp", 4, nif_sendto_test_udp, 0}, - {"set_udp_broadcast", 2, nif_set_udp_broadcast, 0}, - /* Python event loop integration */ - {"set_python_event_loop", 1, nif_set_python_event_loop, 0}, - {"set_isolation_mode", 1, nif_set_isolation_mode, 0}, - {"set_shared_worker", 1, nif_set_shared_worker, 0}, - - /* Worker pool */ - - /* Process-per-context API (no mutex) */ {"context_create", 1, nif_context_create, 0}, {"context_destroy", 1, nif_context_destroy, 0}, {"context_interrupt", 1, nif_context_interrupt, ERL_NIF_DIRTY_JOB_IO_BOUND}, @@ -7479,7 +5923,6 @@ static ErlNifFunc nif_funcs[] = { {"context_exec", 3, nif_context_exec_with_env, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"context_eval", 4, nif_context_eval_with_env, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"context_call", 6, nif_context_call_with_env, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - /* Async dispatch - non-blocking, returns immediately */ {"context_call_async", 7, nif_context_call_async, 0}, {"context_eval_async", 5, nif_context_eval_async, 0}, {"context_exec_async", 4, nif_context_exec_async, 0}, @@ -7497,9 +5940,6 @@ static ErlNifFunc nif_funcs[] = { {"context_write_callback_response", 2, nif_context_write_callback_response, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"context_resume", 3, nif_context_resume, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"context_cancel_resume", 2, nif_context_cancel_resume, 0}, - {"context_get_event_loop", 1, nif_context_get_event_loop, 0}, - - /* py_ref API (Python object references with interp_id) */ {"ref_wrap", 2, nif_ref_wrap, 0}, {"is_ref", 1, nif_is_ref, 0}, {"ref_interp_id", 1, nif_ref_interp_id, 0}, @@ -7507,54 +5947,14 @@ static ErlNifFunc nif_funcs[] = { {"ref_getattr", 2, nif_ref_getattr, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"ref_call_method", 3, nif_ref_call_method, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - /* Reactor NIFs - Erlang-as-Reactor architecture */ - {"reactor_register_fd", 3, nif_reactor_register_fd, 0}, - {"reactor_reselect_read", 1, nif_reactor_reselect_read, 0}, - {"reactor_select_write", 1, nif_reactor_select_write, 0}, - {"get_fd_from_resource", 1, nif_get_fd_from_resource, 0}, - {"reactor_on_read_ready", 2, nif_reactor_on_read_ready, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"reactor_on_write_ready", 2, nif_reactor_on_write_ready, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"reactor_init_connection", 3, nif_reactor_init_connection, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"reactor_close_fd", 2, nif_reactor_close_fd, 0}, - - /* Direct FD operations */ - {"fd_read", 2, nif_fd_read, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"fd_write", 2, nif_fd_write, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"fd_select_read", 1, nif_fd_select_read, 0}, - {"fd_select_write", 1, nif_fd_select_write, 0}, - {"fd_close", 1, nif_fd_close, 0}, - {"socketpair", 0, nif_socketpair, 0}, - - /* Channel API - bidirectional message passing */ - {"channel_create", 0, nif_channel_create, 0}, - {"channel_create", 1, nif_channel_create, 0}, - {"channel_send", 2, nif_channel_send, 0}, - {"channel_receive", 2, nif_channel_receive, 0}, - {"channel_try_receive", 1, nif_channel_try_receive, 0}, - {"channel_reply", 3, nif_channel_reply, 0}, - {"channel_close", 1, nif_channel_close, 0}, - {"channel_info", 1, nif_channel_info, 0}, - {"channel_wait", 3, nif_channel_wait, 0}, - {"channel_cancel_wait", 2, nif_channel_cancel_wait, 0}, - {"channel_register_sync_waiter", 1, nif_channel_register_sync_waiter, 0}, - - /* ByteChannel API - raw bytes, no term conversion */ - {"byte_channel_send_bytes", 2, nif_byte_channel_send_bytes, 0}, - {"byte_channel_try_receive_bytes", 1, nif_byte_channel_try_receive_bytes, 0}, - {"byte_channel_wait_bytes", 3, nif_byte_channel_wait_bytes, 0}, - - /* PyBuffer API - zero-copy input */ - {"py_buffer_create", 1, nif_py_buffer_create, 0}, - {"py_buffer_write", 2, nif_py_buffer_write, 0}, - {"py_buffer_close", 1, nif_py_buffer_close, 0}, - - /* SharedDict API - process-scoped shared dictionary */ - {"shared_dict_new", 0, nif_shared_dict_new, 0}, - {"shared_dict_get", 3, nif_shared_dict_get, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"shared_dict_set", 3, nif_shared_dict_set, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"shared_dict_del", 2, nif_shared_dict_del, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"shared_dict_keys", 1, nif_shared_dict_keys, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"shared_dict_destroy", 1, nif_shared_dict_destroy, 0} + /* One macro per area, defined at the end of the file that owns it */ + PY_CALLBACK_NIFS, + PY_THREAD_WORKER_NIFS, + PY_LOGGING_NIFS, + PY_EVENT_LOOP_NIFS, + PY_CHANNEL_NIFS, + PY_BUFFER_NIFS, + PY_SHARED_DICT_NIFS }; ERL_NIF_INIT(py_nif, nif_funcs, load, NULL, upgrade, unload) diff --git a/c_src/py_nif.h b/c_src/py_nif.h index 6832c40..5dab26c 100644 --- a/c_src/py_nif.h +++ b/c_src/py_nif.h @@ -645,21 +645,21 @@ typedef struct { * @brief One Python execution environment served by one Erlang process * * A context has exactly one pthread that runs Python for it: the context - * thread (worker_context_thread_main for worker mode, - * owngil_context_thread_main for owngil mode). Erlang processes never run + * thread (ctx_thread_main_worker for worker mode, + * ctx_thread_main_owngil for owngil mode). Erlang processes never run * Python on a context; NIFs enqueue a ctx_request_t and return, the * context thread dequeues, executes and replies with `{py_result, Id, R}` * through msg_env. Isolated mode does not use this struct at all. * * Lock and ownership contract, by field group: * - * - Identity and lifecycle (interp_id, is_subinterp, uses_worker_thread, + * - Identity and lifecycle (interp_id, is_subinterp, has_thread, * uses_own_gil): written once by nif_context_create before the thread - * starts, read-only afterwards. destroyed, leaked, worker_running, + * starts, read-only afterwards. destroyed, leaked, thread_running, * shutdown_requested and init_error are atomics; any thread may read * them, the writers are nif_context_destroy (destroyed, leaked), the * shutdown helpers (shutdown_requested) and the context thread - * (worker_running, init_error). + * (thread_running, init_error). * * - Callback handler (has_callback_handler, callback_handler, * callback_pipe): set by the owning Erlang process through @@ -734,16 +734,16 @@ struct py_context { /* ========== Context thread (worker and owngil modes) ========== */ /** @brief Dedicated pthread for this context */ - pthread_t worker_thread; + pthread_t thread; /** @brief True when worker thread is running */ - _Atomic bool worker_running; + _Atomic bool thread_running; /** @brief True when shutdown has been requested */ _Atomic bool shutdown_requested; /** @brief True if this context uses a dedicated worker thread (worker mode) */ - bool uses_worker_thread; + bool has_thread; /** @brief True if thread initialization failed */ _Atomic bool init_error; @@ -2050,40 +2050,13 @@ static inline void log_and_clear_python_error(const char *context) { #ifdef HAVE_SUBINTERPRETERS -/** - * @brief Dispatch reactor on_read_ready to OWN_GIL thread - * - * @param env Caller's NIF environment - * @param ctx OWN_GIL context - * @param fd File descriptor - * @param buffer_ptr Reactor buffer resource (ownership transferred) - * @return Result term - */ -ERL_NIF_TERM dispatch_reactor_read_to_owngil(ErlNifEnv *env, py_context_t *ctx, - int fd, void *buffer_ptr); - -/** - * @brief Dispatch reactor on_write_ready to OWN_GIL thread - * - * @param env Caller's NIF environment - * @param ctx OWN_GIL context - * @param fd File descriptor - * @return Result term - */ -ERL_NIF_TERM dispatch_reactor_write_to_owngil(ErlNifEnv *env, py_context_t *ctx, - int fd); - -/** - * @brief Dispatch reactor init_connection to OWN_GIL thread - * - * @param env Caller's NIF environment - * @param ctx OWN_GIL context - * @param fd File descriptor - * @param client_info Client info map term - * @return Result term - */ -ERL_NIF_TERM dispatch_reactor_init_to_owngil(ErlNifEnv *env, py_context_t *ctx, - int fd, ERL_NIF_TERM client_info); +/* Reactor callbacks run on the context thread through the request queue + * (ctx_dispatch_wait in py_nif.c); py_event_loop.c calls these. */ +ERL_NIF_TERM dispatch_reactor_read(ErlNifEnv *env, py_context_t *ctx, + int fd, void *buffer_ptr); +ERL_NIF_TERM dispatch_reactor_write(ErlNifEnv *env, py_context_t *ctx, int fd); +ERL_NIF_TERM dispatch_reactor_init(ErlNifEnv *env, py_context_t *ctx, + int fd, ERL_NIF_TERM client_info); #endif /* HAVE_SUBINTERPRETERS */ diff --git a/c_src/py_shared_dict.c b/c_src/py_shared_dict.c index a55c3d0..265b860 100644 --- a/c_src/py_shared_dict.c +++ b/c_src/py_shared_dict.c @@ -813,3 +813,13 @@ static PyObject *py_shared_dict_keys_impl(PyObject *self, PyObject *args) { pthread_mutex_unlock(&sd->mutex); return result; } + +/* NIF table entries of this file; py_nif.c concatenates them into nif_funcs[]. + * Flags: ERL_NIF_DIRTY_JOB_* for anything that can block or run Python. */ +#define PY_SHARED_DICT_NIFS \ + {"shared_dict_new", 0, nif_shared_dict_new, 0}, \ + {"shared_dict_get", 3, nif_shared_dict_get, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"shared_dict_set", 3, nif_shared_dict_set, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"shared_dict_del", 2, nif_shared_dict_del, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"shared_dict_keys", 1, nif_shared_dict_keys, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"shared_dict_destroy", 1, nif_shared_dict_destroy, 0} diff --git a/c_src/py_thread_worker.c b/c_src/py_thread_worker.c index 31fde37..7656583 100644 --- a/c_src/py_thread_worker.c +++ b/c_src/py_thread_worker.c @@ -844,3 +844,11 @@ static ERL_NIF_TERM nif_async_callback_response(ErlNifEnv *env, int argc, } return make_error(env, "write_failed"); } + +/* NIF table entries of this file; py_nif.c concatenates them into nif_funcs[]. + * Flags: ERL_NIF_DIRTY_JOB_* for anything that can block or run Python. */ +#define PY_THREAD_WORKER_NIFS \ + {"thread_worker_set_coordinator", 1, nif_thread_worker_set_coordinator, 0}, \ + {"thread_worker_write_with_id", 3, nif_thread_worker_write_with_id, ERL_NIF_DIRTY_JOB_IO_BOUND}, \ + {"thread_worker_signal_ready", 1, nif_thread_worker_signal_ready, 0}, \ + {"async_callback_response", 3, nif_async_callback_response, ERL_NIF_DIRTY_JOB_IO_BOUND} diff --git a/docs/architecture.md b/docs/architecture.md index 7c30bf3..3ae029f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -50,7 +50,7 @@ one Python execution environment and serves calls in order. Pools | | `worker` | `owngil` | `isolated` | |---|---|---|---| | Python runs in | the VM, main interpreter | the VM, a sub-interpreter with its own GIL | a child process | -| Thread | one pthread per context (`worker_context_thread_main`) | one pthread per context (`owngil_context_thread_main`) | the child's main thread | +| Thread | one pthread per context (`ctx_thread_main_worker`) | one pthread per context (`ctx_thread_main_owngil`) | the child's main thread | | Erlang process loop | the receive loop in `py_context` | the receive loop in `py_context` | `py_isolated` (`gen_statem`) | | Transport | NIF request queue on `py_context_t` | same | Unix socket, frames of the callback pipe format | | Python -> Erlang | suspension protocol | blocking callback pipe | socket frames | @@ -71,9 +71,9 @@ one Python execution environment and serves calls in order. Pools arguments (`term_to_py`, `c_src/py_convert.c`) into a request, enqueues it on the context's queue (`ctx_queue_enqueue`) and returns `{enqueued, Ref}` at once. The Erlang process is now free to serve callbacks. -3. The context's pthread (`worker_context_thread_main` or - `owngil_context_thread_main`, `c_src/py_nif.c`) dequeues the request and - runs it through `owngil_execute_request` (despite its name it serves both +3. The context's pthread (`ctx_thread_main_worker` or + `ctx_thread_main_owngil`, `c_src/py_nif.c`) dequeues the request and + runs it through `ctx_execute_request` (one function for both modes), which calls into Python with the GIL held. 4. The thread converts the result (`py_to_term`) and sends `{py_result, Ref, Result}` to the `py_context` process, which replies diff --git a/docs/code-map.md b/docs/code-map.md index 5b237d8..534ce81 100644 --- a/docs/code-map.md +++ b/docs/code-map.md @@ -9,8 +9,12 @@ exercised by suites). Guides are in `docs/`, suites in `test/`. Start with | Module | Owns | Status | Guide | Suites | |---|---|---|---|---| -| `py` | Public API facade: call/eval/exec, streams, async helpers, venvs, memory, function registration | live | README, getting-started | `py_SUITE`, `py_api_SUITE`, `py_stream_SUITE`, `py_venv_SUITE` | -| `py_context` | The context process for embedded modes and the API every mode answers (`call/eval/exec`, `interrupt`, `kill`, loops, `pass_fd`); dispatch to `py_isolated` for isolated mode | live | context-affinity, workers, interrupts | `py_context_SUITE`, `py_context_process_SUITE`, `py_interrupt_SUITE`, `py_worker_loop_SUITE` | +| `py` | Public API facade: call/eval/exec, async helpers, memory, function registration; delegates streams, venvs and shared dicts | live | README, getting-started | `py_SUITE`, `py_api_SUITE` | +| `py_stream` | Generator streaming behind `py:stream*` | live | streaming | `py_stream_SUITE` | +| `py_venv` | Virtual environments behind `py:ensure_venv` and friends | live | README (venvs) | `py_venv_SUITE` | +| `py_shared_dict` | `py:shared_dict_*` over the shared dict NIFs | live | shared-dict | `py_SUITE` | +| `py_context` | The API every mode answers (`call/eval/exec`, `interrupt`, `kill`, loops, `pass_fd`), the reply protocol and the pid to NIF reference table; `init/4` hands the process to `py_context_embedded` or `py_isolated` | live | context-affinity, workers, interrupts | `py_context_SUITE`, `py_context_process_SUITE`, `py_interrupt_SUITE`, `py_worker_loop_SUITE` | +| `py_context_embedded` | Process body for `worker` and `owngil` mode: the receive loop, callbacks (suspension and pipe), worker loops | live | architecture, state-machines | same | | `py_isolated` | `gen_statem` driving a child process over the socket; restart policy | live | isolated | `py_isolated_*_SUITE` | | `py_context_router` | Pools and scheduler-affinity routing | live | pools, context-affinity | `py_context_router_SUITE`, `py_pool_SUITE` | | `py_context_sup`, `py_context_init` | Supervisor of contexts; starts the default pool at boot | live | pools | (through the above) | diff --git a/docs/contributing.md b/docs/contributing.md index 52c1226..756ed2e 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -111,8 +111,9 @@ unreleased version. 1. Implement `static ERL_NIF_TERM nif_x(ErlNifEnv*, int, const ERL_NIF_TERM[])` in the `c_src` file that owns the area (see `c_src/README.md`). -2. Add `{"x", Arity, nif_x, Flags}` to `nif_funcs[]` at the end of - `c_src/py_nif.c`. `Flags` is `ERL_NIF_DIRTY_JOB_CPU_BOUND` or +2. Add `{"x", Arity, nif_x, Flags}` to the `PY_*_NIFS` macro at the end of + that file (`nif_funcs[]` in `c_src/py_nif.c` concatenates them; NIFs + that live in `py_nif.c` go in its own block there). `Flags` is `ERL_NIF_DIRTY_JOB_CPU_BOUND` or `ERL_NIF_DIRTY_JOB_IO_BOUND` when the NIF can block or run Python, `0` otherwise. 3. Add the stub, its `-spec` and a `@doc` to `src/py_nif.erl`, and the diff --git a/docs/decisions/0001-one-thread-per-context.md b/docs/decisions/0001-one-thread-per-context.md index 33fe2bc..45618ed 100644 --- a/docs/decisions/0001-one-thread-per-context.md +++ b/docs/decisions/0001-one-thread-per-context.md @@ -1,6 +1,6 @@ # 0001: One pthread per context, NIFs only enqueue -Since 3.0.0. Code: `worker_context_thread_main`, `owngil_context_thread_main`, +Since 3.0.0. Code: `ctx_thread_main_worker`, `ctx_thread_main_owngil`, the request queue on `py_context_t` (`c_src/py_nif.c`, `c_src/py_nif.h`). ## Situation diff --git a/docs/glossary.md b/docs/glossary.md index 439622e..7e675ca 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -22,7 +22,7 @@ descriptors for the reactor; "coordinator context" in C comments means the per context, shared GIL), `owngil` (a sub-interpreter with its own GIL per context, one pthread), `isolated` (a child process). `py_context:new(#{mode => ...})`. -Related flags on `py_context_t`: `uses_worker_thread` (has its own pthread; +Related flags on `py_context_t`: `has_thread` (has its own pthread; true for worker and owngil contexts created today), `is_subinterp` (has its own sub-interpreter), `uses_own_gil` (that sub-interpreter has its own GIL). `subinterp` in file and NIF names (`py_subinterp_thread.c`, @@ -39,7 +39,7 @@ The most overloaded word. Meanings, by file: | Where | Meaning | Prefer to say | |---|---|---| | `py_context:new(#{mode => worker})` | the context mode above | worker mode | -| `worker_context_thread_main`, `uses_worker_thread` (`py_nif.c`) | the pthread that serves a context's queue | context thread | +| `ctx_thread_main_worker`, `has_thread` (`py_nif.c`) | the pthread that serves a context's queue | context thread | | `thread_worker`, `thread_worker_call` (`py_thread_worker.c`), `py_thread_handler` | the channel a Python thread uses to call Erlang | thread callback bridge | | `py_event_worker` | the Erlang process that drives one asyncio loop (readiness, timers) | loop driver | | `docs/workers.md`, "worker loop" | a long-running asyncio loop on a context thread, gunicorn-style | worker loop | diff --git a/docs/owngil_internals.md b/docs/owngil_internals.md index 8d3b2e5..c7ae84f 100644 --- a/docs/owngil_internals.md +++ b/docs/owngil_internals.md @@ -68,7 +68,7 @@ All major erlang_python features work with OWN_GIL mode: │ │ │ │ └──────────┼───────────────────────────┼──────────────────────────────┘ │ │ - │ dispatch_to_owngil_thread │ + │ ctx_dispatch │ ▼ ▼ ┌──────────────────────┐ ┌──────────────────────┐ │ OWN_GIL Thread 1 │ │ OWN_GIL Thread 2 │ @@ -150,8 +150,8 @@ nif_context_create(env, "owngil") └── owngil_context_init(ctx) ├── Initialize mutex/condvars ├── Create shared_env - └── pthread_create(owngil_context_thread_main) - └── owngil_context_thread_main(ctx) + └── pthread_create(ctx_thread_main_owngil) + └── ctx_thread_main_owngil(ctx) ├── Py_NewInterpreterFromConfig(OWN_GIL) ├── Initialize globals/locals ├── Register py_event_loop module @@ -164,7 +164,7 @@ nif_context_create(env, "owngil") nif_context_call(env, ctx, module, func, args, kwargs) │ ├── [ctx->uses_own_gil == true] - │ └── dispatch_to_owngil_thread(env, ctx, CTX_REQ_CALL, request) + │ └── ctx_dispatch(env, ctx, CTX_REQ_CALL, request, NULL) │ ├── pthread_mutex_lock(&ctx->request_mutex) │ ├── Copy request term to shared_env │ ├── Set ctx->request_type = CTX_REQ_CALL @@ -180,15 +180,15 @@ nif_context_call(env, ctx, module, func, args, kwargs) ### 3. Request Processing (OWN_GIL Thread) ``` -owngil_context_thread_main(ctx) +ctx_thread_main_owngil(ctx) while (!shutdown_requested) { pthread_cond_wait(&ctx->request_ready) - owngil_execute_request(ctx) + ctx_execute_request(ctx) switch (ctx->request_type) { - case CTX_REQ_CALL: owngil_execute_call(ctx); break; - case CTX_REQ_EVAL: owngil_execute_eval(ctx); break; - case CTX_REQ_EXEC: owngil_execute_exec(ctx); break; + case CTX_REQ_CALL: ctx_execute_call(ctx); break; + case CTX_REQ_EVAL: ctx_execute_eval(ctx); break; + case CTX_REQ_EXEC: ctx_execute_exec(ctx); break; // ... other cases } @@ -224,8 +224,8 @@ OWN_GIL contexts support process-local environments for namespace isolation: ``` py_context:create_local_env(Ctx) └── nif_create_local_env(CtxRef) - └── dispatch_create_local_env_to_owngil(env, ctx, res) - └── owngil_execute_create_local_env(ctx) + └── ctx_dispatch_wait(env, ctx, req) (CTX_REQ_CREATE_LOCAL_ENV) + └── ctx_execute_create_local_env(ctx) ├── res->globals = PyDict_New() ├── res->locals = PyDict_New() └── res->interp_id = ctx->interp_id @@ -262,7 +262,7 @@ while (!shutdown_requested) { if (shutdown_requested) break; // Process request (GIL already held within subinterpreter) - owngil_execute_request(ctx); + ctx_execute_request(ctx); pthread_cond_signal(&response_ready); pthread_mutex_unlock(&request_mutex); @@ -336,7 +336,7 @@ Each Python subinterpreter has its own module namespace. The `py_event_loop` mod │ [ctx->uses_own_gil == true] ▼ ┌────────────────────────────────────────────────────────────────────────┐ -│ dispatch_reactor_read_to_owngil(env, ctx, fd, buffer_ptr) │ +│ dispatch_reactor_read(env, ctx, fd, buffer_ptr) │ │ │ │ │ ├── ctx->reactor_buffer_ptr = buffer_ptr │ │ ├── ctx->request_type = CTX_REQ_REACTOR_READ │ @@ -349,7 +349,7 @@ Each Python subinterpreter has its own module namespace. The `py_event_loop` mod │ OWN_GIL Thread │ ├────────────────────────────────────────────────────────────────────────┤ │ │ -│ owngil_execute_reactor_read(ctx) │ +│ ctx_execute_reactor_read(ctx) │ │ │ │ │ ├── Create ReactorBuffer Python object │ │ │ │ @@ -390,9 +390,9 @@ The `ensure_reactor_cached_for_interp()` function lazily imports `erlang.reactor | Request Type | Dispatch Function | Execute Function | |--------------|-------------------|------------------| -| `CTX_REQ_REACTOR_READ` | `dispatch_reactor_read_to_owngil` | `owngil_execute_reactor_read` | -| `CTX_REQ_REACTOR_WRITE` | `dispatch_reactor_write_to_owngil` | `owngil_execute_reactor_write` | -| `CTX_REQ_REACTOR_INIT` | `dispatch_reactor_init_to_owngil` | `owngil_execute_reactor_init` | +| `CTX_REQ_REACTOR_READ` | `dispatch_reactor_read` | `ctx_execute_reactor_read` | +| `CTX_REQ_REACTOR_WRITE` | `dispatch_reactor_write` | `ctx_execute_reactor_write` | +| `CTX_REQ_REACTOR_INIT` | `dispatch_reactor_init` | `ctx_execute_reactor_init` | ### Buffer Handling diff --git a/docs/state-machines.md b/docs/state-machines.md index 871bab1..f2fee76 100644 --- a/docs/state-machines.md +++ b/docs/state-machines.md @@ -25,7 +25,7 @@ UNINIT --init--> INITING --ok--> RUNNING --finalize--> SHUTTING_DOWN --> STOPPED Two cooperating machines: the Erlang process and the pthread in C. -Context thread (`worker_context_thread_main`, `owngil_context_thread_main` +Context thread (`ctx_thread_main_worker`, `ctx_thread_main_owngil` in `c_src/py_nif.c`): ``` @@ -38,11 +38,11 @@ starting --namespaces created--> waiting --dequeue--> executing --reply--> waiti - `executing` is bracketed by `py_context_exec_enter` / `exec_leave` (interrupt bookkeeping) around the GIL; the request mirror on `py_context_t` is valid only here. -- `exited` sets `worker_running = false`; `nif_context_destroy` joins with a +- `exited` sets `thread_running = false`; `nif_context_destroy` joins with a timeout and, if the join fails, marks the context `leaked` and pins the resource instead of freeing it. -Erlang process (`loop/1` in `py_context`): +Erlang process (`loop/1` in `py_context_embedded`): ``` idle --{call|eval|exec|submit}--> in_request --{py_result}--> idle diff --git a/src/py.erl b/src/py.erl index 8b9218a..f2c682e 100644 --- a/src/py.erl +++ b/src/py.erl @@ -145,7 +145,7 @@ -type py_args() :: [term()]. -type py_kwargs() :: #{atom() | binary() => term()}. --export_type([py_result/0, py_ref/0]). +-export_type([py_result/0, py_ref/0, py_module/0, py_func/0, py_args/0, py_kwargs/0]). %% Default timeout for synchronous calls (30 seconds) -define(DEFAULT_TIMEOUT, 30000). @@ -404,83 +404,24 @@ await(Ref, Timeout) -> %% @doc Stream results from a Python generator. %% Returns a list of all yielded values. -spec stream(py_module(), py_func(), py_args()) -> py_result(). -stream(Module, Func, Args) -> - stream(Module, Func, Args, #{}). +stream(A1, A2, A3) -> + py_stream:stream(A1, A2, A3). %% @doc Stream results from a Python generator with kwargs. -spec stream(py_module(), py_func(), py_args(), py_kwargs()) -> py_result(). -stream(Module, Func, Args, Kwargs) when map_size(Kwargs) == 0 -> - %% No kwargs - use stream_start and collect results - {ok, Ref} = stream_start(Module, Func, Args), - collect_stream(Ref, []); -stream(Module, Func, Args, Kwargs) -> - %% With kwargs - use eval approach - Ctx = py_context_router:get_context(), - ModuleBin = valid_py_module(ensure_binary(Module)), - FuncBin = valid_py_ident(ensure_binary(Func)), - KwargsCode = format_kwargs(Kwargs), - ArgsCode = format_args(Args), - Code = iolist_to_binary([ - <<"list(__import__('">>, ModuleBin, <<"').">>, FuncBin, - <<"(">>, ArgsCode, KwargsCode, <<"))">> - ]), - py_context:eval(Ctx, Code, #{}). - -%% @private Collect all stream events into a list -collect_stream(Ref, Acc) -> - receive - {py_stream, Ref, {data, Value}} -> - collect_stream(Ref, [Value | Acc]); - {py_stream, Ref, done} -> - {ok, lists:reverse(Acc)}; - {py_stream, Ref, {error, Reason}} -> - {error, Reason} - after 30000 -> - {error, timeout} - end. - -%% @private Format arguments for Python code -format_args([]) -> <<>>; -format_args(Args) -> - ArgStrs = [format_arg(A) || A <- Args], - iolist_to_binary(lists:join(<<", ">>, ArgStrs)). - -%% @private Format a single argument -format_arg(A) when is_integer(A) -> integer_to_binary(A); -format_arg(A) when is_float(A) -> float_to_binary(A); -format_arg(A) when is_binary(A) -> <<"'", (escape_py_literal(A))/binary, "'">>; -format_arg(A) when is_atom(A) -> <<"'", (escape_py_literal(atom_to_binary(A)))/binary, "'">>; -format_arg(A) when is_list(A) -> iolist_to_binary([<<"[">>, format_args(A), <<"]">>]); -format_arg(_) -> <<"None">>. - -%% @private Format kwargs for Python code -format_kwargs(Kwargs) when map_size(Kwargs) == 0 -> <<>>; -format_kwargs(Kwargs) -> - KwList = maps:fold(fun(K, V, Acc) -> - KB = valid_py_ident(if is_atom(K) -> atom_to_binary(K); is_binary(K) -> K end), - [<>, lists:join(<<", ">>, KwList)]). +stream(A1, A2, A3, A4) -> + py_stream:stream(A1, A2, A3, A4). %% @doc Stream results from a Python generator expression. %% Evaluates the expression and if it returns a generator, streams all values. -spec stream_eval(string() | binary()) -> py_result(). -stream_eval(Code) -> - stream_eval(Code, #{}). +stream_eval(A1) -> + py_stream:stream_eval(A1). %% @doc Stream results from a Python generator expression with local variables. -spec stream_eval(string() | binary(), map()) -> py_result(). -stream_eval(Code, Locals) -> - %% Route through the new process-per-context system - %% Wrap the code in list() to collect generator values - Ctx = py_context_router:get_context(), - CodeBin = ensure_binary(Code), - WrappedCode = <<"list(", CodeBin/binary, ")">>, - py_context:eval(Ctx, WrappedCode, Locals). - -%%% ============================================================================ -%%% True Streaming API (Event-driven) -%%% ============================================================================ +stream_eval(A1, A2) -> + py_stream:stream_eval(A1, A2). %% @doc Start a true streaming iteration from a Python generator. %% @@ -516,8 +457,8 @@ stream_eval(Code, Locals) -> %% end. %% ''' -spec stream_start(py_module(), py_func(), py_args()) -> {ok, reference()}. -stream_start(Module, Func, Args) -> - stream_start(Module, Func, Args, #{}). +stream_start(A1, A2, A3) -> + py_stream:stream_start(A1, A2, A3). %% @doc Start a true streaming iteration with options. %% @@ -530,83 +471,8 @@ stream_start(Module, Func, Args) -> %% @param Opts Options map %% @returns {ok, Ref} where Ref is used to identify stream events -spec stream_start(py_module(), py_func(), py_args(), map()) -> {ok, reference()}. -stream_start(Module, Func, Args, Opts) -> - Owner = maps:get(owner, Opts, self()), - Ref = make_ref(), - ModuleBin = ensure_binary(Module), - FuncBin = ensure_binary(Func), - RefHash = erlang:phash2(Ref), - %% Store owner and ref for Python to retrieve - %% Use binary keys because Python strings become binaries - py_state:store({<<"stream_owner">>, RefHash}, Owner), - py_state:store({<<"stream_ref">>, RefHash}, Ref), - py_state:store({<<"stream_args">>, RefHash}, Args), - %% Spawn an Erlang process to run the streaming iteration - spawn(fun() -> - stream_run_python(ModuleBin, FuncBin, RefHash) - end), - {ok, Ref}. - -%% @private Run the streaming via Python code -stream_run_python(ModuleBin0, FuncBin0, RefHash) -> - ModuleBin = valid_py_module(ModuleBin0), - FuncBin = valid_py_ident(FuncBin0), - RefHashBin = integer_to_binary(RefHash), - %% Build Python code that streams values using callbacks - Code = iolist_to_binary([ - <<"import erlang\n">>, - <<"_rh = ">>, RefHashBin, <<"\n">>, - <<"_args = erlang.call('state_get', ('stream_args', _rh))\n">>, - <<"if _args is None:\n">>, - <<" _args = []\n">>, - <<"try:\n">>, - <<" _mod = __import__('">>, ModuleBin, <<"')\n">>, - <<" _fn = getattr(_mod, '">>, FuncBin, <<"')\n">>, - <<" _gen = _fn(*_args) if _args else _fn()\n">>, - %% Async generators are driven on a private event loop. erlang.call is - %% a blocking pipe read, so it stalls that loop between yields, which - %% is fine for a sequential stream. - <<" if hasattr(_gen, '__anext__'):\n">>, - <<" import asyncio\n">>, - <<" async def _drive():\n">>, - <<" async for _val in _gen:\n">>, - <<" if erlang.call('_py_stream_cancelled', _rh):\n">>, - <<" erlang.call('_py_stream_send', _rh, 'error', 'cancelled')\n">>, - <<" return\n">>, - <<" erlang.call('_py_stream_send', _rh, 'data', _val)\n">>, - <<" erlang.call('_py_stream_send', _rh, 'done', None)\n">>, - <<" asyncio.run(_drive())\n">>, - <<" else:\n">>, - <<" for _val in _gen:\n">>, - <<" if erlang.call('_py_stream_cancelled', _rh):\n">>, - <<" erlang.call('_py_stream_send', _rh, 'error', 'cancelled')\n">>, - <<" break\n">>, - <<" erlang.call('_py_stream_send', _rh, 'data', _val)\n">>, - <<" else:\n">>, - <<" erlang.call('_py_stream_send', _rh, 'done', None)\n">>, - <<"except Exception as _e:\n">>, - <<" erlang.call('_py_stream_send', _rh, 'error', str(_e))\n">>, - <<"finally:\n">>, - <<" erlang.call('_py_stream_cleanup', _rh)\n">> - ]), - %% Execute the streaming code - case exec(Code) of - ok -> ok; - {error, Reason} -> - %% Try to notify owner of error - case py_state:fetch({<<"stream_owner">>, RefHash}) of - {ok, Owner} -> - case py_state:fetch({<<"stream_ref">>, RefHash}) of - {ok, Ref} -> - Owner ! {py_stream, Ref, {error, Reason}}, - py_state:remove({<<"stream_owner">>, RefHash}), - py_state:remove({<<"stream_ref">>, RefHash}), - py_state:remove({<<"stream_args">>, RefHash}); - _ -> ok - end; - _ -> ok - end - end. +stream_start(A1, A2, A3, A4) -> + py_stream:stream_start(A1, A2, A3, A4). %% @doc Cancel an active stream. %% @@ -616,13 +482,8 @@ stream_run_python(ModuleBin0, FuncBin0, RefHash) -> %% @param Ref The stream reference from stream_start/3,4 %% @returns ok -spec stream_cancel(reference()) -> ok. -stream_cancel(Ref) when is_reference(Ref) -> - %% Store cancellation flag that the streaming task checks - %% Use hash because we can't pass Erlang refs to Python callbacks easily - %% Use binary key because Python strings become binaries - RefHash = erlang:phash2(Ref), - py_state:store({<<"stream_cancelled_hash">>, RefHash}, true), - ok. +stream_cancel(A1) -> + py_stream:stream_cancel(A1). %%% ============================================================================ %%% Info @@ -844,6 +705,11 @@ parallel(Calls) when is_list(Calls) -> %%% Virtual Environment Support %%% ============================================================================ +%% @doc Kill the child of an isolated context. See py_context:kill/1. +-spec kill(pid()) -> ok | {error, not_isolated}. +kill(Ctx) when is_pid(Ctx) -> + py_context:kill(Ctx). + %% @doc Ensure a virtual environment exists and activate it. %% %% Creates a venv at `Path' if it doesn't exist, installs dependencies from @@ -858,8 +724,8 @@ parallel(Calls) when is_list(Calls) -> %% ok = py:ensure_venv("priv/venv", "requirements.txt"). %% ''' -spec ensure_venv(string() | binary(), string() | binary()) -> ok | {error, term()}. -ensure_venv(Path, RequirementsFile) -> - ensure_venv(Path, RequirementsFile, []). +ensure_venv(A1, A2) -> + py_venv:ensure_venv(A1, A2). %% @doc Ensure a virtual environment exists with options. %% @@ -882,50 +748,8 @@ ensure_venv(Path, RequirementsFile) -> %% ]). %% ''' -spec ensure_venv(string() | binary(), string() | binary(), list()) -> ok | {error, term()}. -ensure_venv(Path, RequirementsFile, Opts) -> - PathStr = to_string(Path), - ReqFileStr = to_string(RequirementsFile), - Force = proplists:get_bool(force, Opts), - %% Create venv if needed - VenvReady = case venv_exists(PathStr) of - true when not Force -> - ok; - _ -> - create_venv(PathStr, Opts) - end, - case VenvReady of - ok -> - %% Always install/update dependencies (pip/uv skip existing) - case install_deps(PathStr, ReqFileStr, Opts) of - ok -> - activate_venv(PathStr); - {error, _} = Err -> - Err - end; - {error, _} = Err -> - Err - end. - -%% @private Check if venv exists by looking for pyvenv.cfg --spec venv_exists(string()) -> boolean(). -venv_exists(Path) -> - filelib:is_file(filename:join(Path, "pyvenv.cfg")). - -%% @private Create a new virtual environment --spec create_venv(string(), list()) -> ok | {error, term()}. -create_venv(Path, Opts) -> - Installer = detect_installer(Opts), - Python = case proplists:get_value(python, Opts, undefined) of - undefined -> get_python_executable(); - P -> P - end, - case Installer of - uv -> - %% uv venv is faster, use --python to match the running interpreter - run_cmd(uv_exe(), ["venv", "--python", Python, Path], []); - pip -> - run_cmd(Python, ["-m", "venv", Path], []) - end. +ensure_venv(A1, A2, A3) -> + py_venv:ensure_venv(A1, A2, A3). %% @private Get the Python executable path %% When embedded, sys.executable returns the embedding app (beam.smp) @@ -936,138 +760,7 @@ create_venv(Path, Opts) -> %% VM). Used as the default interpreter of isolated contexts and for venvs. -spec python_executable() -> string(). python_executable() -> - get_python_executable(). - -%% @doc Kill the child of an isolated context. See py_context:kill/1. --spec kill(pid()) -> ok | {error, not_isolated}. -kill(Ctx) when is_pid(Ctx) -> - py_context:kill(Ctx). - --spec get_python_executable() -> string(). -get_python_executable() -> - %% Use a single expression to find the Python executable - %% Searches for pythonX.Y, python3, python in sys.prefix/bin (Unix) - %% or python.exe in sys.prefix (Windows) - Expr = <<"(lambda: (__import__('os').path.join(__import__('sys').prefix, 'python.exe') if __import__('sys').platform == 'win32' and __import__('os').path.isfile(__import__('os').path.join(__import__('sys').prefix, 'python.exe')) else next((p for p in [__import__('os').path.join(__import__('sys').prefix, 'bin', f'python{__import__(\"sys\").version_info.major}.{__import__(\"sys\").version_info.minor}'), __import__('os').path.join(__import__('sys').prefix, 'bin', 'python3'), __import__('os').path.join(__import__('sys').prefix, 'bin', 'python')] if __import__('os').path.isfile(p)), 'python3')))()">>, - case eval(Expr) of - {ok, Path} when is_binary(Path) -> binary_to_list(Path); - _ -> "python3" - end. - -%% @private Install dependencies from requirements file --spec install_deps(string(), string(), list()) -> ok | {error, term()}. -install_deps(Path, RequirementsFile, Opts) -> - Installer = detect_installer(Opts), - {Exe, BaseArgs, PortOpts} = pip_command(Path, Installer), - Extras = proplists:get_value(extras, Opts, []), - - %% Determine file type and build the install argument list (no shell). - Args = case filename:extension(RequirementsFile) of - ".txt" -> - BaseArgs ++ ["install", "-r", RequirementsFile]; - ".toml" -> - %% pyproject.toml - install as editable. - %% filename:dirname returns "." for files without directory component - InstallPath = filename:dirname(RequirementsFile), - case Extras of - [] -> - BaseArgs ++ ["install", "-e", InstallPath]; - _ -> - ExtrasStr = string:join(Extras, ","), - BaseArgs ++ ["install", "-e", InstallPath ++ "[" ++ ExtrasStr ++ "]"] - end; - _ -> - BaseArgs ++ ["install", "-r", RequirementsFile] - end, - run_cmd(Exe, Args, PortOpts). - -%% @private Detect which installer to use (uv or pip) --spec detect_installer(list()) -> uv | pip. -detect_installer(Opts) -> - case proplists:get_value(installer, Opts, auto) of - auto -> - case os:find_executable("uv") of - false -> pip; - _ -> uv - end; - Installer -> - Installer - end. - -%% @private Resolve the installer into {Executable, BaseArgs, PortOpts}. -%% For uv the venv is selected via the VIRTUAL_ENV port env option (not a shell -%% prefix); for pip we use the venv's own pip binary. --spec pip_command(string(), uv | pip) -> {string(), [string()], list()}. -pip_command(VenvPath, uv) -> - {uv_exe(), ["pip"], [{env, [{"VIRTUAL_ENV", VenvPath}]}]}; -pip_command(VenvPath, pip) -> - PipExe = case os:type() of - {win32, _} -> - filename:join([VenvPath, "Scripts", "pip"]); - _ -> - filename:join([VenvPath, "bin", "pip"]) - end, - {PipExe, [], []}. - -%% @private Full path to the uv executable (falls back to the bare name). --spec uv_exe() -> string(). -uv_exe() -> - case os:find_executable("uv") of - false -> "uv"; - P -> P - end. - -%% @private Run an executable with an argv list (no shell) and return ok or error. --spec run_cmd(string(), [string()], list()) -> ok | {error, term()}. -run_cmd(Exe, Args, ExtraOpts) -> - case resolve_exe(Exe) of - {error, _} = Err -> - Err; - ExeFull -> - try open_port({spawn_executable, ExeFull}, - [exit_status, stderr_to_stdout, binary, {args, Args} | ExtraOpts]) of - Port -> collect_port(Port, []) - catch - error:Reason -> {error, {spawn_failed, Exe, Reason}} - end - end. - -%% @private Resolve an executable name/path to a full path (spawn_executable does -%% not search PATH). --spec resolve_exe(string()) -> string() | {error, term()}. -resolve_exe(Exe) -> - case filename:pathtype(Exe) of - absolute -> - case filelib:is_file(Exe) of - true -> Exe; - false -> {error, {executable_not_found, Exe}} - end; - _ -> - case os:find_executable(Exe) of - false -> {error, {executable_not_found, Exe}}; - Found -> Found - end - end. - -%% @private Collect a spawned port's output and exit status. --spec collect_port(port(), [binary()]) -> ok | {error, term()}. -collect_port(Port, Acc) -> - receive - {Port, {data, Data}} -> - collect_port(Port, [Data | Acc]); - {Port, {exit_status, 0}} -> - ok; - {Port, {exit_status, Code}} -> - {error, {exit_code, Code, iolist_to_binary(lists:reverse(Acc))}} - after 300000 -> - try port_close(Port) catch _:_ -> ok end, - {error, timeout} - end. - -%% @private Convert to string --spec to_string(string() | binary()) -> string(). -to_string(B) when is_binary(B) -> binary_to_list(B); -to_string(S) when is_list(S) -> S. + py_venv:python_executable(). %% @doc Activate a Python virtual environment. %% This modifies sys.path to use packages from the specified venv. @@ -1084,132 +777,20 @@ to_string(S) when is_list(S) -> S. %% {ok, _} = py:call(sentence_transformers, 'SentenceTransformer', [<<"all-MiniLM-L6-v2">>]). %% ''' -spec activate_venv(string() | binary()) -> ok | {error, term()}. -activate_venv(VenvPath) -> - VenvBin = ensure_binary(VenvPath), - %% Find site-packages directory dynamically (venv may use different Python version) - %% Uses a single expression to avoid multiline code issues - FindSitePackages = <<"(lambda vp: __import__('os').path.join(vp, 'Lib', 'site-packages') if __import__('os').path.exists(__import__('os').path.join(vp, 'Lib', 'site-packages')) else next((sp for name in (__import__('os').listdir(__import__('os').path.join(vp, 'lib')) if __import__('os').path.isdir(__import__('os').path.join(vp, 'lib')) else []) if name.startswith('python') for sp in [__import__('os').path.join(vp, 'lib', name, 'site-packages')] if __import__('os').path.isdir(sp)), None))(_venv_path)">>, - case eval(FindSitePackages, #{<<"_venv_path">> => VenvBin}) of - {ok, SitePackages} when SitePackages =/= none, SitePackages =/= null -> - activate_venv_with_site_packages(VenvBin, SitePackages); - {ok, _} -> - {error, {invalid_venv, no_site_packages_found}}; - Error -> - Error - end. - -%% @private Activate venv with known site-packages path -activate_venv_with_site_packages(VenvBin, SitePackages) -> - %% Verify site-packages exists - case eval(<<"__import__('os').path.isdir(sp)">>, #{sp => SitePackages}) of - {ok, true} -> - %% Save original path if not already saved - {ok, _} = eval(<<"setattr(__import__('sys'), '_original_path', __import__('sys').path.copy()) if not hasattr(__import__('sys'), '_original_path') else None">>), - %% Set venv info - {ok, _} = eval(<<"setattr(__import__('sys'), '_active_venv', vp)">>, #{vp => VenvBin}), - {ok, _} = eval(<<"setattr(__import__('sys'), '_venv_site_packages', sp)">>, #{sp => SitePackages}), - %% Add site-packages and process .pth files (editable installs) - %% Note: We embed the site-packages path directly since exec doesn't support - %% variables and sys attributes may not persist across calls in subinterpreters - SitePackagesStr = binary_to_list(SitePackages), - ExecCode = iolist_to_binary([ - <<"import site as _site, sys as _sys\n">>, - <<"_sp = '">>, escape_python_string(SitePackagesStr), <<"'\n">>, - <<"_b = frozenset(_sys.path)\n">>, - <<"_site.addsitedir(_sp)\n">>, - <<"_sys.path[:] = [p for p in _sys.path if p not in _b] + [p for p in _sys.path if p in _b]\n">>, - <<"del _site, _sys, _b, _sp\n">> - ]), - ok = exec(ExecCode), - ok; - {ok, false} -> - {error, {invalid_venv, SitePackages}}; - Error -> - Error - end. - -%% @private Escape a string for embedding in Python code -escape_python_string(Str) -> - lists:flatmap(fun($') -> "\\'"; - ($\\) -> "\\\\"; - (C) -> [C] - end, Str). - -%% @private Escape a binary for safe embedding inside a single-quoted Python -%% string literal: quote, backslash, and newline/CR/tab/other control bytes that -%% would otherwise break out of or corrupt the literal. -escape_py_literal(Bin) when is_binary(Bin) -> - << <<(escape_py_byte(B))/binary>> || <> <= Bin >>. - -escape_py_byte($') -> <<"\\'">>; -escape_py_byte($\\) -> <<"\\\\">>; -escape_py_byte($\n) -> <<"\\n">>; -escape_py_byte($\r) -> <<"\\r">>; -escape_py_byte($\t) -> <<"\\t">>; -escape_py_byte(B) when B < 16#20; B =:= 16#7f -> - list_to_binary(io_lib:format("\\x~2.16.0b", [B])); -escape_py_byte(B) -> <>. - -%% @private Validate a Python identifier ([A-Za-z_][A-Za-z0-9_]*). Crashes on a -%% non-conforming value so an attacker-controlled module/func/kwarg name can't -%% inject code at an identifier position (where quoting is meaningless). -valid_py_ident(Bin) when is_binary(Bin), byte_size(Bin) > 0 -> - case ident_ok(Bin, first) of - true -> Bin; - false -> error({invalid_python_identifier, Bin}) - end; -valid_py_ident(Other) -> - error({invalid_python_identifier, Other}). - -%% @private Validate a dotted Python module path (each segment an identifier). -valid_py_module(Bin) when is_binary(Bin), byte_size(Bin) > 0 -> - Segments = binary:split(Bin, <<".">>, [global]), - lists:foreach(fun valid_py_ident/1, Segments), - Bin; -valid_py_module(Other) -> - error({invalid_python_identifier, Other}). - -ident_ok(<<>>, first) -> false; %% empty segment (leading/trailing/double dot) -ident_ok(<<>>, rest) -> true; -ident_ok(<>, first) - when (C >= $A andalso C =< $Z); (C >= $a andalso C =< $z); C =:= $_ -> - ident_ok(Rest, rest); -ident_ok(<>, rest) - when (C >= $A andalso C =< $Z); (C >= $a andalso C =< $z); - (C >= $0 andalso C =< $9); C =:= $_ -> - ident_ok(Rest, rest); -ident_ok(_, _) -> false. +activate_venv(A1) -> + py_venv:activate_venv(A1). %% @doc Deactivate the current virtual environment. %% Restores sys.path to its original state. -spec deactivate_venv() -> ok | {error, term()}. deactivate_venv() -> - case eval(<<"hasattr(__import__('sys'), '_original_path')">>) of - {ok, true} -> - ok = exec(<<"import sys as _sys\n" - "_sys.path[:] = _sys._original_path\n" - "del _sys\n">>), - {ok, _} = eval(<<"delattr(__import__('sys'), '_original_path')">>), - {ok, _} = eval(<<"delattr(__import__('sys'), '_active_venv') if hasattr(__import__('sys'), '_active_venv') else None">>), - {ok, _} = eval(<<"delattr(__import__('sys'), '_venv_site_packages') if hasattr(__import__('sys'), '_venv_site_packages') else None">>), - ok; - {ok, false} -> - ok; - Error -> - Error - end. + py_venv:deactivate_venv(). %% @doc Get information about the currently active virtual environment. %% Returns a map with venv_path and site_packages, or none if no venv is active. -spec venv_info() -> {ok, map() | none} | {error, term()}. venv_info() -> - %% Check both attributes exist to handle partial activation/deactivation state - Code = <<"({'active': True, 'venv_path': __import__('sys')._active_venv, 'site_packages': __import__('sys')._venv_site_packages, 'sys_path': __import__('sys').path} if (hasattr(__import__('sys'), '_active_venv') and hasattr(__import__('sys'), '_venv_site_packages')) else {'active': False})">>, - eval(Code). - -%% @private -ensure_binary(S) -> - py_util:to_binary(S). + py_venv:venv_info(). %%% ============================================================================ %%% Execution Info @@ -1305,7 +886,7 @@ state_decr(Key, Amount) -> %% if any contexts failed. -spec reload(py_module()) -> ok | {error, [{context, term()}]}. reload(Module) -> - ModuleBin = ensure_binary(Module), + ModuleBin = py_util:to_binary(Module), %% Build Python code that: %% 1. Checks if module is loaded in sys.modules %% 2. If yes, reloads it with importlib.reload() @@ -1372,13 +953,13 @@ configure_logging(Opts) -> iolist_to_binary([ "__import__('erlang').setup_logging(", integer_to_binary(LevelInt), - ", '", escape_py_literal(F), "')" + ", '", py_util:escape_py_literal(F), "')" ]); F when is_list(F) -> iolist_to_binary([ "__import__('erlang').setup_logging(", integer_to_binary(LevelInt), - ", '", escape_py_literal(iolist_to_binary(F)), "')" + ", '", py_util:escape_py_literal(iolist_to_binary(F)), "')" ]) end, case eval(Code) of @@ -1529,7 +1110,7 @@ interrupt(Ctx) when is_pid(Ctx) -> %% @returns {ok, Result} | {error, Reason} -spec call_method(reference(), atom() | binary(), list()) -> py_result(). call_method(Ref, Method, Args) -> - MethodBin = ensure_binary(Method), + MethodBin = py_util:to_binary(Method), py_nif:ref_call_method(Ref, MethodBin, Args). %% @doc Get an attribute from a Python object reference. @@ -1539,7 +1120,7 @@ call_method(Ref, Method, Args) -> %% @returns {ok, Value} | {error, Reason} -spec getattr(reference(), atom() | binary()) -> py_result(). getattr(Ref, Name) -> - NameBin = ensure_binary(Name), + NameBin = py_util:to_binary(Name), py_nif:ref_getattr(Ref, NameBin). %% @doc Convert a Python object reference to an Erlang term. @@ -1645,7 +1226,7 @@ unregister_pool({Module, Func}) when is_atom(Module), is_atom(Func) -> %% @returns {ok, Reference} on success, {error, Reason} on failure -spec shared_dict_new() -> {ok, reference()} | {error, term()}. shared_dict_new() -> - py_nif:shared_dict_new(). + py_shared_dict:shared_dict_new(). %% @doc Get a value from SharedDict with default undefined. %% @@ -1653,8 +1234,8 @@ shared_dict_new() -> %% @param Key Binary key %% @returns Value or undefined if key not found -spec shared_dict_get(reference(), binary()) -> term(). -shared_dict_get(Handle, Key) -> - shared_dict_get(Handle, Key, undefined). +shared_dict_get(A1, A2) -> + py_shared_dict:shared_dict_get(A1, A2). %% @doc Get a value from SharedDict with custom default. %% @@ -1663,8 +1244,8 @@ shared_dict_get(Handle, Key) -> %% @param Default Default value if key not found %% @returns Value or Default -spec shared_dict_get(reference(), binary(), term()) -> term(). -shared_dict_get(Handle, Key, Default) when is_binary(Key) -> - py_nif:shared_dict_get(Handle, Key, Default). +shared_dict_get(A1, A2, A3) -> + py_shared_dict:shared_dict_get(A1, A2, A3). %% @doc Set a value in SharedDict. %% @@ -1675,8 +1256,8 @@ shared_dict_get(Handle, Key, Default) when is_binary(Key) -> %% @param Value Erlang term value (will be pickled) %% @returns ok on success -spec shared_dict_set(reference(), binary(), term()) -> ok | {error, term()}. -shared_dict_set(Handle, Key, Value) when is_binary(Key) -> - py_nif:shared_dict_set(Handle, Key, Value). +shared_dict_set(A1, A2, A3) -> + py_shared_dict:shared_dict_set(A1, A2, A3). %% @doc Delete a key from SharedDict. %% @@ -1684,16 +1265,16 @@ shared_dict_set(Handle, Key, Value) when is_binary(Key) -> %% @param Key Binary key %% @returns ok (even if key didn't exist) -spec shared_dict_del(reference(), binary()) -> ok. -shared_dict_del(Handle, Key) when is_binary(Key) -> - py_nif:shared_dict_del(Handle, Key). +shared_dict_del(A1, A2) -> + py_shared_dict:shared_dict_del(A1, A2). %% @doc Get all keys from SharedDict. %% %% @param Handle SharedDict reference %% @returns List of binary keys -spec shared_dict_keys(reference()) -> [binary()]. -shared_dict_keys(Handle) -> - py_nif:shared_dict_keys(Handle). +shared_dict_keys(A1) -> + py_shared_dict:shared_dict_keys(A1). %% @doc Explicitly destroy a SharedDict. %% @@ -1705,6 +1286,5 @@ shared_dict_keys(Handle) -> %% @param Handle SharedDict reference %% @returns ok -spec shared_dict_destroy(reference()) -> ok. -shared_dict_destroy(Handle) -> - py_nif:shared_dict_destroy(Handle). - +shared_dict_destroy(A1) -> + py_shared_dict:shared_dict_destroy(A1). diff --git a/src/py_context.erl b/src/py_context.erl index 385dff4..757df51 100644 --- a/src/py_context.erl +++ b/src/py_context.erl @@ -30,10 +30,10 @@ %%% back. Nested requests from the callback are served inline, so callbacks %%% can call Python again to any depth. %%% -%%% Owns: the context resource, the request in flight, its timeout and -%%% the process-local envs (`py:call(Ctx, ...)'). -%%% Talks to: `py_nif' (context NIFs), `py_isolated', `py_callback', -%%% `py_context_sup'. +%%% Owns: the public API, the reply protocol (`{MRef, Reply}', timeouts, +%%% interrupt on timeout) and the pid to NIF reference table. +%%% Talks to: `py_context_embedded' (the process body for worker and owngil +%%% mode), `py_isolated' (isolated mode), `py_nif' (interrupt). %%% Never: runs Python on a scheduler thread; the context thread does. %%% %%% @end @@ -78,9 +78,9 @@ -export([kill/1, pass_fd/2, child_info/1]). -export([init/3, init/4, init_ref_tab/0]). +%% Used by py_context_embedded +-export([register_nif_ref/1, unregister_nif_ref/0]). -%% Exported for py_reactor_context --export([extend_erlang_module_in_context/1]). %% Maps context pid -> NIF context reference. Read by interrupt/1, which must %% reach the NIF reference while the context process is blocked in a NIF and @@ -90,29 +90,17 @@ %% How long to wait for an interrupted call to unwind and reply, so the late %% reply is drained instead of being left in the caller's mailbox. -define(INTERRUPT_GRACE_MS, 1000). +%% Time given to a running loop to exit after interrupt/1 (also in +%% py_context_embedded, which drives the stop) +-define(LOOP_INTERRUPT_GRACE_MS, 3000). -type context_mode() :: worker | owngil | isolated. -type context() :: pid(). -export_type([context_mode/0, context/0]). --record(state, { - ref :: reference(), - id :: pos_integer(), - interp_id :: non_neg_integer(), - event_state = #{} :: map(), %% #{loop_ref => ref(), worker_pid => pid()} - callback_handler :: pid() | undefined, %% For thread-model callback handling - %% Worker loop (start_loop/1): request id of the run_forever exec, the - %% owner that gets {py_loop_exit, Ctx, Result}, its monitor, and the - %% callers waiting in stop_loop/2 - loop_req :: reference() | undefined, - loop_owner :: pid() | undefined, - loop_owner_mon :: reference() | undefined, - loop_stop_waiters = [] :: [{pid(), reference()}] -}). -%% Time given to a running loop to exit after py_context:interrupt/1 --define(LOOP_INTERRUPT_GRACE_MS, 3000). + %% ============================================================================ %% API @@ -735,867 +723,7 @@ init(Parent, Id, Mode) -> init(Parent, Id, isolated, Opts) -> py_isolated:init(Parent, Id, isolated, Opts); init(Parent, Id, Mode, Opts) -> - process_flag(trap_exit, true), - case create_context(Mode) of - {ok, Ref, InterpId} -> - %% Publish the NIF reference so interrupt/1 can reach it while - %% this process is blocked in a NIF - register_nif_ref(Ref), - case apply_memory_limit(Ref, Opts) of - ok -> - init_started(Parent, Id, Ref, InterpId, Opts); - {error, LimitError} -> - unregister_nif_ref(), - try py_nif:context_destroy(Ref) catch _:_ -> ok end, - Parent ! {self(), {error, LimitError}} - end; - {error, Reason} -> - Parent ! {self(), {error, Reason}} - end. - -%% @private -apply_memory_limit(Ref, Opts) -> - case maps:get(memory_limit, Opts, undefined) of - undefined -> - ok; - Bytes when is_integer(Bytes), Bytes >= 0 -> - py_nif:context_set_memory_limit(Ref, Bytes); - Other -> - {error, {invalid_memory_limit, Other}} - end. - -%% @private -init_started(Parent, Id, Ref, InterpId, Opts) -> - %% Apply all registered imports and paths to this interpreter - apply_registered_imports(Ref), - apply_registered_paths(Ref), - %% Apply preload code (populates globals for process-local envs) - apply_preload(Ref), - %% Per-context preload from new/1 (imports the app once per worker) - case maps:get(preload, Opts, undefined) of - undefined -> ok; - PreCode when is_binary(PreCode); is_list(PreCode) -> - case handle_exec_with_async(Ref, iolist_to_binary(PreCode)) of - ok -> ok; - {error, PreErr} -> - error_logger:warning_msg( - "py_context ~p: preload failed: ~p~n", [InterpId, PreErr]) - end - end, - %% For subinterpreters, create a dedicated event worker - EventState = setup_event_worker(Ref, InterpId), - %% For thread-model subinterpreters, spawn a dedicated callback handler - %% because the main context process will be blocked in the NIF - CallbackHandler = case maps:get(mode, EventState, normal) of - thread_model -> - Handler = spawn_callback_handler(Ref), - ok = py_nif:context_set_callback_handler(Ref, Handler), - Handler; - _ -> - undefined - end, - Parent ! {self(), started}, - State = #state{ - ref = Ref, - id = Id, - interp_id = InterpId, - event_state = EventState, - callback_handler = CallbackHandler - }, - loop(State). - -%% @private Create event worker for subinterpreter contexts -setup_event_worker(Ref, InterpId) -> - case py_nif:context_get_event_loop(Ref) of - {ok, LoopRef} -> - %% This is a subinterpreter - create dedicated event worker - WorkerId = iolist_to_binary(["ctx_", integer_to_list(InterpId)]), - case py_event_worker:start_link(WorkerId, LoopRef) of - {ok, WorkerPid} -> - ok = py_nif:event_loop_set_worker(LoopRef, WorkerPid), - %% Extend erlang module with event loop functions - extend_erlang_module_in_context(Ref), - #{loop_ref => LoopRef, worker_pid => WorkerPid}; - {error, WorkerError} -> - error_logger:warning_msg( - "py_context ~p: Failed to start event worker: ~p~n", - [InterpId, WorkerError]), - #{} - end; - {error, not_subinterp} -> - %% Worker mode - uses shared router (lazy initialization) - #{}; - {error, event_loop_owned_by_thread} -> - %% Thread-model subinterpreter: event loop is managed by dedicated thread. - %% This is expected behavior, not a failure. - #{mode => thread_model}; - {error, Reason} -> - error_logger:warning_msg( - "py_context ~p: Failed to get event loop: ~p~n", - [InterpId, Reason]), - #{} - end. - -%% @private Extend the erlang module with event loop functions in a subinterpreter -extend_erlang_module_in_context(Ref) -> - PrivDir = code:priv_dir(erlang_python), - Code = iolist_to_binary([ - "import sys\n", - "priv_dir = '", PrivDir, "'\n", - "if priv_dir not in sys.path:\n", - " sys.path.insert(0, priv_dir)\n", - "import erlang\n", - "if hasattr(erlang, '_extend_erlang_module'):\n", - " erlang._extend_erlang_module(priv_dir)\n" - ]), - case py_nif:context_exec(Ref, Code) of - ok -> ok; - {error, Reason} -> - error_logger:warning_msg( - "py_context: Failed to extend erlang module: ~p~n", [Reason]), - ok - end. - -%% @private Apply all imports from the global registry to this interpreter. -%% -%% Called when a new interpreter is created to pre-warm the module cache -%% with all modules registered via py_import:ensure_imported/1,2. -apply_registered_imports(Ref) -> - case py_import:all_imports() of - [] -> ok; - Imports -> py_nif:interp_apply_imports(Ref, Imports) - end. - -%% @private Apply all paths from the global registry to this interpreter. -%% -%% Called when a new interpreter is created to add all registered paths -%% to sys.path. -apply_registered_paths(Ref) -> - case py_import:all_paths() of - [] -> ok; - Paths -> py_nif:interp_apply_paths(Ref, Paths) - end. - -%% @private Apply preload code to the interpreter's globals. -%% -%% Called when a new interpreter is created. The preload code populates -%% the context's globals dict, which process-local environments inherit. -apply_preload(Ref) -> - py_preload:apply_preload(Ref). - -%% @private -create_context(worker) -> - py_nif:context_create(worker); -create_context(owngil) -> - %% OWN_GIL mode requires Python 3.14+ due to C extension bugs in earlier versions - case py_nif:owngil_supported() of - true -> py_nif:context_create(owngil); - false -> {error, owngil_requires_python314} - end. - -%% @private -%% Main context loop. Handles requests and uses suspension-based callback support. -loop(#state{ref = Ref, interp_id = InterpId, loop_req = LoopReq} = State) -> - receive - %% ---- worker loop management (start_loop/stop_loop/loop_ref) ---- - {start_loop, From, MRef, _Owner} when LoopReq =/= undefined -> - From ! {MRef, {error, already_running}}, - loop(State); - - {start_loop, From, MRef, Owner} -> - {Reply, NewState} = do_start_loop(Owner, State), - From ! {MRef, Reply}, - loop(NewState); - - {stop_loop, From, MRef, _GraceMs} when LoopReq =:= undefined -> - From ! {MRef, {error, no_loop}}, - loop(State); - - {stop_loop, From, MRef, GraceMs} -> - loop(begin_stop_loop(From, MRef, GraceMs, State)); - - {loop_ref, From, MRef} -> - From ! {MRef, context_loop_ref(State)}, - loop(State); - - {py_result, LoopReq, Result} when LoopReq =/= undefined -> - loop(loop_exited(Result, State)); - - {loop_stop_deadline, LoopReq} when LoopReq =/= undefined -> - %% Cooperative stop did not land: interrupt the thread - _ = py_nif:context_interrupt(Ref), - erlang:send_after(?LOOP_INTERRUPT_GRACE_MS, self(), - {loop_interrupt_deadline, LoopReq}), - loop(State); - - {loop_interrupt_deadline, LoopReq} when LoopReq =/= undefined -> - [W ! {M, {error, timeout}} || {W, M} <- State#state.loop_stop_waiters], - loop(State#state{loop_stop_waiters = []}); - - {loop_stop_deadline, _} -> - loop(State); - {loop_interrupt_deadline, _} -> - loop(State); - - {'DOWN', Mon, process, _Owner, _Reason} - when Mon =:= State#state.loop_owner_mon, LoopReq =/= undefined -> - %% Owner is gone: nobody will hear the exit, stop the loop - loop(begin_stop_loop(undefined, undefined, 5000, - State#state{loop_owner_mon = undefined})); - - {async_result, _TaskRef, _} -> - %% Result of a coroutine this process submitted (loop stop) - drop - loop(State); - - %% ---- while a worker loop runs, the thread is not available ---- - {call, From, MRef, _, _, _, _} when LoopReq =/= undefined -> - From ! {MRef, {error, loop_running}}, loop(State); - {call, From, MRef, _, _, _, _, _} when LoopReq =/= undefined -> - From ! {MRef, {error, loop_running}}, loop(State); - {eval, From, MRef, _, _} when LoopReq =/= undefined -> - From ! {MRef, {error, loop_running}}, loop(State); - {eval, From, MRef, _, _, _} when LoopReq =/= undefined -> - From ! {MRef, {error, loop_running}}, loop(State); - {exec, From, MRef, _} when LoopReq =/= undefined -> - From ! {MRef, {error, loop_running}}, loop(State); - {exec, From, MRef, _, _} when LoopReq =/= undefined -> - From ! {MRef, {error, loop_running}}, loop(State); - {call_method, From, MRef, _, _, _} when LoopReq =/= undefined -> - From ! {MRef, {error, loop_running}}, loop(State); - - {stop, From, MRef} when LoopReq =/= undefined -> - %% Get the thread out of the loop before destroying the context, - %% otherwise context_destroy waits for a thread that never returns - terminate(normal, stop_running_loop(State)), - From ! {MRef, ok}; - - {'EXIT', _Pid, Reason} = Exit when LoopReq =/= undefined, - (Reason =:= shutdown orelse Reason =:= kill orelse - (is_tuple(Reason) andalso element(1, Reason) =:= shutdown)) -> - self() ! Exit, - loop(stop_running_loop(State)); - - {call, From, MRef, Module, Func, Args, Kwargs} -> - Result = handle_call_with_suspension(Ref, Module, Func, Args, Kwargs), - From ! {MRef, Result}, - loop(State); - - %% Call with process-local environment (worker mode) - {call, From, MRef, Module, Func, Args, Kwargs, EnvRef} -> - Result = handle_call_with_suspension_and_env(Ref, Module, Func, Args, Kwargs, EnvRef), - From ! {MRef, Result}, - loop(State); - - {eval, From, MRef, Code, Locals} -> - Result = handle_eval_with_suspension(Ref, Code, Locals), - From ! {MRef, Result}, - loop(State); - - %% Eval with process-local environment (worker mode) - {eval, From, MRef, Code, Locals, EnvRef} -> - Result = handle_eval_with_suspension_and_env(Ref, Code, Locals, EnvRef), - From ! {MRef, Result}, - loop(State); - - {exec, From, MRef, Code} -> - Result = handle_exec_with_async(Ref, Code), - From ! {MRef, Result}, - loop(State); - - %% Exec with process-local environment (worker mode). - %% Async dispatch with sync fallback (mirrors call/eval). - {exec, From, MRef, Code, EnvRef} -> - Result = handle_exec_with_async_and_env(Ref, Code, EnvRef), - From ! {MRef, Result}, - loop(State); - - {call_method, From, MRef, ObjRef, Method, Args} -> - Result = py_nif:context_call_method(Ref, ObjRef, Method, Args), - From ! {MRef, Result}, - loop(State); - - {get_interp_id, From, MRef} -> - From ! {MRef, {ok, InterpId}}, - loop(State); - - {is_subinterp, From, MRef} -> - %% Check the interp_id to determine if this is a subinterpreter - %% Subinterpreters have interp_id > 0 (main interpreter is 0) - %% But actually we need to check the mode, not just interp_id - IsSubinterp = is_context_subinterp(Ref), - From ! {MRef, IsSubinterp}, - loop(State); - - {create_local_env, From, MRef} -> - %% Create env inside this context's interpreter - Result = py_nif:create_local_env(Ref), - From ! {MRef, Result}, - loop(State); - - {get_nif_ref, From, MRef} -> - From ! {MRef, Ref}, - loop(State); - - {stop, From, MRef} -> - terminate(normal, State), - From ! {MRef, ok}; - - {'EXIT', Pid, Reason} -> - %% Handle EXIT from linked processes - case State#state.callback_handler of - Pid -> - %% Callback handler died - restart it for thread-model contexts - error_logger:warning_msg( - "py_context ~p: Callback handler died: ~p, restarting~n", - [InterpId, Reason]), - NewHandler = spawn_callback_handler(Ref), - ok = py_nif:context_set_callback_handler(Ref, NewHandler), - NewState = State#state{callback_handler = NewHandler}, - loop(NewState); - _ -> - case State#state.event_state of - #{worker_pid := Pid} -> - %% Event worker died - log and continue (degraded asyncio support) - error_logger:warning_msg( - "py_context ~p: Event worker died: ~p~n", - [InterpId, Reason]), - NewState = State#state{event_state = #{}}, - loop(NewState); - _ when Reason =:= shutdown; Reason =:= kill -> - %% Supervisor shutdown or kill signal - clean exit - terminate(Reason, State); - _ when is_tuple(Reason), element(1, Reason) =:= shutdown -> - %% Supervisor shutdown with extra info: {shutdown, _} - terminate(Reason, State); - _ -> - %% Ignore EXIT from other processes - loop(State) - end - end - end. - -%% ============================================================================ -%% Worker loop helpers -%% ============================================================================ - -%% @private Loop reference: the context's own loop (owngil) or the shared -%% main-interpreter loop (worker mode) -context_loop_ref(#state{event_state = #{loop_ref := LoopRef}}) -> - {ok, LoopRef}; -context_loop_ref(_State) -> - py_event_loop:get_loop(). - -%% @private Start run_forever on the context thread through the async exec -%% path, so this process stays free to serve loop_ref/stop_loop and the -%% dirty schedulers are not held. -do_start_loop(Owner, #state{ref = Ref} = State) -> - case context_loop_ref(State) of - {ok, _} -> - LoopReq = make_ref(), - case py_nif:context_call_async(Ref, self(), LoopReq, <<"erlang">>, - <<"_run_loop_forever">>, [self()], #{}) of - {enqueued, LoopReq} -> - %% Wait for the loop to actually run before answering, so - %% a submit right after start_loop finds it - receive - {py_loop_started} -> - Mon = case is_pid(Owner) of - true -> erlang:monitor(process, Owner); - false -> undefined - end, - {ok, State#state{loop_req = LoopReq, loop_owner = Owner, - loop_owner_mon = Mon, loop_stop_waiters = []}}; - {py_result, LoopReq, {error, Reason}} -> - {{error, Reason}, State}; - {py_result, LoopReq, Other} -> - {{error, {loop_exited, Other}}, State} - after 10000 -> - {{error, loop_start_timeout}, State} - end; - {error, Reason} -> - {{error, Reason}, State} - end; - {error, Reason} -> - {{error, Reason}, State} - end. - -%% @private Ask the running loop to stop from inside, arm the interrupt -%% deadline, and remember who to answer once it has exited. -begin_stop_loop(From, MRef, GraceMs, #state{loop_req = LoopReq} = State) -> - Waiters = case From of - undefined -> State#state.loop_stop_waiters; - _ -> [{From, MRef} | State#state.loop_stop_waiters] - end, - case context_loop_ref(State) of - {ok, LoopRef} -> - _ = py_nif:submit_task(LoopRef, self(), make_ref(), - <<"erlang">>, <<"_stop_loop">>, [], #{}); - _ -> - ok - end, - erlang:send_after(GraceMs, self(), {loop_stop_deadline, LoopReq}), - State#state{loop_stop_waiters = Waiters}. - -%% @private The exec running the loop returned: tell the owner and the -%% stop_loop callers, clear the loop state. -loop_exited(Result, #state{loop_owner = Owner, loop_owner_mon = Mon, - loop_stop_waiters = Waiters} = State) -> - case Mon of - undefined -> ok; - _ -> erlang:demonitor(Mon, [flush]) - end, - case is_pid(Owner) of - true -> Owner ! {py_loop_exit, self(), Result}; - false -> ok - end, - [W ! {M, ok} || {W, M} <- Waiters], - State#state{loop_req = undefined, loop_owner = undefined, - loop_owner_mon = undefined, loop_stop_waiters = []}. - -%% @private Synchronous stop used before terminate: interrupt and wait a -%% bounded time for the exec to return. -stop_running_loop(#state{ref = Ref, loop_req = LoopReq} = State) -> - _ = py_nif:context_interrupt(Ref), - receive - {py_result, LoopReq, Result} -> - loop_exited(Result, State) - after ?LOOP_INTERRUPT_GRACE_MS -> - loop_exited({error, timeout}, State) - end. - -%% @private Clean up resources on termination -terminate(_Reason, #state{ref = Ref, event_state = EventState, callback_handler = CallbackHandler}) -> - unregister_nif_ref(), - %% Stop the callback handler if it exists - case CallbackHandler of - Pid when is_pid(Pid) -> - Pid ! stop; - _ -> - ok - end, - %% Stop the event worker first (if it exists and is still alive) - case EventState of - #{worker_pid := WorkerPid} -> - try gen_server:stop(WorkerPid, normal, 5000) catch _:_ -> ok end; - _ -> - ok - end, - %% Destroy the Python context - try py_nif:context_destroy(Ref) catch _:_ -> ok end, - ok. - -%% ============================================================================ -%% Blocking callback handling (for thread-model subinterpreters) -%% ============================================================================ -%% -%% Thread-model subinterpreters use blocking pipe-based callbacks because -%% the suspension mechanism doesn't work when Python runs in a dedicated thread. -%% The Python thread blocks waiting for a response on the callback pipe. -%% -%% A separate callback handler process is spawned because the main context -%% process is blocked in the NIF (dispatch_to_thread) and cannot receive messages. - -%% @private -%% Spawn a dedicated callback handler process for thread-model subinterpreters. -spawn_callback_handler(Ref) -> - spawn_link(fun() -> callback_handler_loop(Ref) end). - -%% @private -%% Callback handler loop - receives erlang_callback messages and responds. -callback_handler_loop(Ref) -> - receive - {erlang_callback, _CallbackId, FuncName, Args} -> - handle_blocking_callback(Ref, FuncName, Args), - callback_handler_loop(Ref); - stop -> - ok - end. - -%% @private -%% Handle a blocking callback from a thread-model subinterpreter. -%% Executes the callback and writes the response to the callback pipe. -handle_blocking_callback(Ref, FuncName, Args) -> - %% Convert Args from tuple to list if needed - ArgsList = case Args of - T when is_tuple(T) -> tuple_to_list(T); - L when is_list(L) -> L; - _ -> [Args] - end, - %% Execute the registered function - Response = case py_callback:execute(FuncName, ArgsList) of - {ok, Result} -> - %% Format: status_byte (2=ok, ETF) + external term format - <<2, (term_to_binary(Result))/binary>>; - {error, {not_found, Name}} -> - ErrMsg = iolist_to_binary( - io_lib:format("Function '~s' not registered", [Name])), - <<1, ErrMsg/binary>>; - {error, {Class, Reason, _Stack}} -> - ErrMsg = iolist_to_binary( - io_lib:format("~p: ~p", [Class, Reason])), - <<1, ErrMsg/binary>> - end, - %% Write response to context's callback pipe - py_nif:context_write_callback_response(Ref, Response). - -%% ============================================================================ -%% Suspension-based callback handling -%% ============================================================================ -%% -%% When Python calls erlang.call(), the NIF returns {suspended, ...} instead of -%% blocking. We handle the callback inline and then resume Python execution. -%% This enables unlimited nesting depth without deadlock. - -%% @private -%% Handle call with potential suspension for callbacks -%% Uses async dispatch to avoid blocking dirty schedulers when possible. -handle_call_with_suspension(Ref, Module, Func, Args, Kwargs) -> - RequestId = make_ref(), - case py_nif:context_call_async(Ref, self(), RequestId, Module, Func, Args, Kwargs) of - {enqueued, RequestId} -> - %% Async dispatch succeeded - wait for result message - wait_for_async_result(Ref, RequestId); - {error, async_requires_worker_thread} -> - %% Fall back to blocking call for non-worker-thread contexts - handle_call_blocking(Ref, Module, Func, Args, Kwargs); - {error, Reason} -> - {error, Reason} - end. - -%% @private -%% Blocking call handler (used when async is not available) -handle_call_blocking(Ref, Module, Func, Args, Kwargs) -> - case py_nif:context_call(Ref, Module, Func, Args, Kwargs) of - {suspended, _CallbackId, StateRef, {FuncName, CallbackArgs}} -> - %% Callback needed - handle it with recursive receive - CallbackResult = handle_callback_with_nested_receive(Ref, FuncName, CallbackArgs), - %% Resume and potentially get more suspensions - resume_and_continue(Ref, StateRef, CallbackResult); - {schedule, CallbackName, CallbackArgs} -> - %% Schedule marker: Python returned erlang.schedule() - %% Execute the callback and return its result - handle_schedule(Ref, CallbackName, CallbackArgs); - Result -> - Result - end. - -%% @private -%% Handle eval with potential suspension for callbacks -%% Uses async dispatch to avoid blocking dirty schedulers when possible. -handle_eval_with_suspension(Ref, Code, Locals) -> - RequestId = make_ref(), - case py_nif:context_eval_async(Ref, self(), RequestId, Code, Locals) of - {enqueued, RequestId} -> - %% Async dispatch succeeded - wait for result message - wait_for_async_result(Ref, RequestId); - {error, async_requires_worker_thread} -> - %% Fall back to blocking call for non-worker-thread contexts - handle_eval_blocking(Ref, Code, Locals); - {error, Reason} -> - {error, Reason} - end. - -%% @private -%% Handle exec with async dispatch -handle_exec_with_async(Ref, Code) -> - RequestId = make_ref(), - case py_nif:context_exec_async(Ref, self(), RequestId, Code) of - {enqueued, RequestId} -> - wait_for_async_result(Ref, RequestId); - {error, async_requires_worker_thread} -> - py_nif:context_exec(Ref, Code); - {error, Reason} -> - {error, Reason} - end. - -%% @private -%% Blocking eval handler (used when async is not available) -handle_eval_blocking(Ref, Code, Locals) -> - case py_nif:context_eval(Ref, Code, Locals) of - {suspended, _CallbackId, StateRef, {FuncName, CallbackArgs}} -> - %% Callback needed - handle it with recursive receive - CallbackResult = handle_callback_with_nested_receive(Ref, FuncName, CallbackArgs), - %% Resume and potentially get more suspensions - resume_and_continue(Ref, StateRef, CallbackResult); - {schedule, CallbackName, CallbackArgs} -> - %% Schedule marker: Python returned erlang.schedule() - %% Execute the callback and return its result - handle_schedule(Ref, CallbackName, CallbackArgs); - Result -> - Result - end. - -%% @private -%% Wait for async result from worker thread -%% The worker thread sends {py_result, RequestId, Result} when done. -%% -%% Drains stale {py_result, _, _} messages from prior timed-out -%% requests before the matching receive so a context that experiences -%% repeat timeouts doesn't grow an unbounded mailbox: when -%% wait_for_async_result/2 returns {error, async_timeout}, the C -%% worker can still finish later and deliver the result; without the -%% drain those messages would accumulate forever. -%% -%% Safe because the context process is the sole receiver for its own -%% async results and only one wait_for_async_result/2 is in flight at -%% a time, so the drain cannot consume the result of a concurrent live -%% request. -wait_for_async_result(Ref, RequestId) -> - drain_stale_async_results(RequestId), - receive - {py_result, RequestId, Result} -> - process_async_result(Ref, Result) - after 300000 -> %% 5 minute timeout - {error, async_timeout} - end. - -%% @private -drain_stale_async_results(CurrentId) -> - receive - {py_result, OldId, _} when OldId =/= CurrentId -> - drain_stale_async_results(CurrentId) - after 0 -> - ok - end. - -%% @private -%% Process the result from async dispatch -%% Handles suspension, schedule markers, and normal results. -process_async_result(Ref, {suspended, _CallbackId, StateRef, {FuncName, CallbackArgs}}) -> - CallbackResult = handle_callback_with_nested_receive(Ref, FuncName, CallbackArgs), - resume_and_continue(Ref, StateRef, CallbackResult); -process_async_result(Ref, {schedule, CallbackName, CallbackArgs}) -> - handle_schedule(Ref, CallbackName, CallbackArgs); -process_async_result(_Ref, Result) -> - Result. - -%% @private -%% Handle call with process-local environment. -%% Tries async dispatch first (no 30 s NIF timeout); falls back to the -%% blocking NIF only when the worker thread isn't available. -handle_call_with_suspension_and_env(Ref, Module, Func, Args, Kwargs, EnvRef) -> - RequestId = make_ref(), - case py_nif:context_call_with_env_async(Ref, self(), RequestId, - Module, Func, Args, Kwargs, - EnvRef) of - {enqueued, RequestId} -> - wait_for_async_result(Ref, RequestId); - {error, async_requires_worker_thread} -> - handle_call_with_env_blocking(Ref, Module, Func, Args, Kwargs, EnvRef); - {error, Reason} -> - {error, Reason} - end. - -%% @private -handle_call_with_env_blocking(Ref, Module, Func, Args, Kwargs, EnvRef) -> - case py_nif:context_call(Ref, Module, Func, Args, Kwargs, EnvRef) of - {suspended, _CallbackId, StateRef, {FuncName, CallbackArgs}} -> - CallbackResult = handle_callback_with_nested_receive(Ref, FuncName, CallbackArgs), - resume_and_continue(Ref, StateRef, CallbackResult); - {schedule, CallbackName, CallbackArgs} -> - handle_schedule(Ref, CallbackName, CallbackArgs); - Result -> - Result - end. - -%% @private -%% Handle eval with process-local environment. -%% Tries async dispatch first; falls back to the blocking NIF only when -%% the worker thread isn't available. -handle_eval_with_suspension_and_env(Ref, Code, Locals, EnvRef) -> - RequestId = make_ref(), - case py_nif:context_eval_with_env_async(Ref, self(), RequestId, - Code, Locals, EnvRef) of - {enqueued, RequestId} -> - wait_for_async_result(Ref, RequestId); - {error, async_requires_worker_thread} -> - handle_eval_with_env_blocking(Ref, Code, Locals, EnvRef); - {error, Reason} -> - {error, Reason} - end. - -%% @private -handle_eval_with_env_blocking(Ref, Code, Locals, EnvRef) -> - case py_nif:context_eval(Ref, Code, Locals, EnvRef) of - {suspended, _CallbackId, StateRef, {FuncName, CallbackArgs}} -> - CallbackResult = handle_callback_with_nested_receive(Ref, FuncName, CallbackArgs), - resume_and_continue(Ref, StateRef, CallbackResult); - {schedule, CallbackName, CallbackArgs} -> - handle_schedule(Ref, CallbackName, CallbackArgs); - Result -> - Result - end. - -%% @private -%% Handle exec with process-local environment via the same async-first -%% path used for call/eval. -handle_exec_with_async_and_env(Ref, Code, EnvRef) -> - RequestId = make_ref(), - case py_nif:context_exec_with_env_async(Ref, self(), RequestId, - Code, EnvRef) of - {enqueued, RequestId} -> - wait_for_async_result(Ref, RequestId); - {error, async_requires_worker_thread} -> - py_nif:context_exec(Ref, Code, EnvRef); - {error, Reason} -> - {error, Reason} - end. - -%% @private -%% Check if a context is a subinterpreter (has interp_id > 0) -is_context_subinterp(Ref) -> - py_nif:context_interp_id(Ref) > 0. - -%% @private -%% Handle schedule marker - Python returned erlang.schedule() or schedule_py() -%% Execute the callback and return its result transparently to the caller. -%% -%% Special case for _execute_py: this callback is used by schedule_py() to -%% call back into Python with a different function. We handle it directly -%% using context_call to avoid recursion through py:call. -handle_schedule(Ref, <<"_execute_py">>, {Module, Func, Args, Kwargs}) -> - %% schedule_py callback: call Python function via context - CallArgs = case Args of - none -> []; - undefined -> []; - List when is_list(List) -> List; - Tuple when is_tuple(Tuple) -> tuple_to_list(Tuple); - _ -> [Args] - end, - CallKwargs = case Kwargs of - none -> #{}; - undefined -> #{}; - Map when is_map(Map) -> Map; - _ -> #{} - end, - handle_call_with_suspension(Ref, Module, Func, CallArgs, CallKwargs); -handle_schedule(_Ref, CallbackName, CallbackArgs) when is_binary(CallbackName) -> - %% Regular callback: execute via py_callback:execute - ArgsList = tuple_to_list(CallbackArgs), - case py_callback:execute(CallbackName, ArgsList) of - {ok, Result} -> - {ok, Result}; - {error, Reason} -> - {error, Reason} - end. - -%% @private -%% Handle callback, allowing nested py:eval/call to be processed. -%% We spawn a process to execute the callback so we can stay in a receive loop -%% for nested calls while the callback runs. -handle_callback_with_nested_receive(Ref, FuncName, CallbackArgs) -> - Parent = self(), - CallbackPid = spawn_link(fun() -> - Result = try - ArgsList = tuple_to_list(CallbackArgs), - case py_callback:execute(FuncName, ArgsList) of - {ok, Value} -> - {ok, <<2, (term_to_binary(Value))/binary>>}; - {error, Reason} -> - ErrMsg = iolist_to_binary(io_lib:format("~p", [Reason])), - {ok, <<1, ErrMsg/binary>>} - end - catch - Class:ExcReason:Stacktrace -> - ErrorMsg = iolist_to_binary(io_lib:format("~p:~p~n~p", - [Class, ExcReason, Stacktrace])), - {ok, <<1, ErrorMsg/binary>>} - end, - Parent ! {callback_result, self(), Result} - end), - %% Wait for callback, processing nested requests - wait_for_callback(Ref, CallbackPid). - -%% @private -%% Wait for callback result while processing nested py:call/eval requests. -%% This enables arbitrarily deep callback nesting. -wait_for_callback(Ref, CallbackPid) -> - receive - {callback_result, CallbackPid, Result} -> - Result; - - %% Handle nested py:call while waiting for callback - {call, From, MRef, Module, Func, Args, Kwargs} -> - NestedResult = handle_call_with_suspension(Ref, Module, Func, Args, Kwargs), - From ! {MRef, NestedResult}, - wait_for_callback(Ref, CallbackPid); - - %% Handle nested py:call while waiting for callback (with EnvRef) - {call, From, MRef, Module, Func, Args, Kwargs, EnvRef} -> - NestedResult = handle_call_with_suspension_and_env(Ref, Module, Func, Args, Kwargs, EnvRef), - From ! {MRef, NestedResult}, - wait_for_callback(Ref, CallbackPid); - - %% Handle nested py:eval while waiting for callback (without EnvRef) - {eval, From, MRef, Code, Locals} -> - NestedResult = handle_eval_with_suspension(Ref, Code, Locals), - From ! {MRef, NestedResult}, - wait_for_callback(Ref, CallbackPid); - - %% Handle nested py:eval while waiting for callback (with EnvRef) - {eval, From, MRef, Code, Locals, EnvRef} -> - NestedResult = handle_eval_with_suspension_and_env(Ref, Code, Locals, EnvRef), - From ! {MRef, NestedResult}, - wait_for_callback(Ref, CallbackPid); - - %% Handle nested py:exec while waiting for callback - {exec, From, MRef, Code} -> - NestedResult = py_nif:context_exec(Ref, Code), - From ! {MRef, NestedResult}, - wait_for_callback(Ref, CallbackPid); - - %% Handle nested py:exec while waiting for callback (with EnvRef) - {exec, From, MRef, Code, EnvRef} -> - NestedResult = py_nif:context_exec(Ref, Code, EnvRef), - From ! {MRef, NestedResult}, - wait_for_callback(Ref, CallbackPid); - - %% Handle nested call_method while waiting for callback - {call_method, From, MRef, ObjRef, Method, Args} -> - NestedResult = py_nif:context_call_method(Ref, ObjRef, Method, Args), - From ! {MRef, NestedResult}, - wait_for_callback(Ref, CallbackPid); - - %% Handle get_interp_id while waiting - {get_interp_id, From, MRef} -> - InterpId = py_nif:context_interp_id(Ref), - From ! {MRef, {ok, InterpId}}, - wait_for_callback(Ref, CallbackPid); - - %% Handle create_local_env while waiting - {create_local_env, From, MRef} -> - Result = py_nif:create_local_env(Ref), - From ! {MRef, Result}, - wait_for_callback(Ref, CallbackPid); - - {get_nif_ref, From, MRef} -> - From ! {MRef, Ref}, - wait_for_callback(Ref, CallbackPid) - end. - -%% @private -%% Resume suspended state, handle additional suspensions (nested callbacks) -resume_and_continue(Ref, StateRef, {ok, ResultBin}) -> - case py_nif:context_resume(Ref, StateRef, ResultBin) of - {suspended, _CallbackId2, StateRef2, {FuncName2, Args2}} -> - %% Another callback during resume - recursive handling - CallbackResult2 = handle_callback_with_nested_receive(Ref, FuncName2, Args2), - resume_and_continue(Ref, StateRef2, CallbackResult2); - FinalResult -> - FinalResult - end; -resume_and_continue(Ref, StateRef, {error, _} = Err) -> - _ = py_nif:context_cancel_resume(Ref, StateRef), - Err. - -%% ============================================================================ -%% Utility functions -%% ============================================================================ - -%% Callback results cross to Python as external term format (status byte 2) -%% and are decoded by term_to_py() in c_src/py_convert.c, the same -%% converter used for call arguments. The former Python-repr encoder was -%% removed in favour of it. + py_context_embedded:init(Parent, Id, Mode, Opts). %% @private to_binary(Atom) when is_atom(Atom) -> diff --git a/src/py_context_embedded.erl b/src/py_context_embedded.erl new file mode 100644 index 0000000..fddeb48 --- /dev/null +++ b/src/py_context_embedded.erl @@ -0,0 +1,847 @@ +%% Copyright 2026 Benoit Chesneau +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%% @doc Process body of a context in `worker' or `owngil' mode. +%%% +%%% `py_context:init/4' hands the process here once the mode is known. The +%%% NIF context is created, the registry imports, paths and preload are +%%% applied, and the process enters `loop/1': one request at a time, +%%% forwarded to the context thread with the `*_async' NIFs and answered by +%%% `{py_result, Ref, Result}'. While the thread waits for an Erlang +%%% callback the process runs it here, serving nested requests, so +%%% callbacks can call Python again to any depth. +%%% +%%% Messages and replies are those documented in `py_context'; callers never +%%% address this module. +%%% +%%% Owns: the NIF context resource, the request in flight, the worker loop +%%% state and the callback handler process. +%%% Talks to: `py_nif' (context NIFs), `py_callback' (registered funs), +%%% `py_event_worker' (owngil loops), `py_import', `py_preload'. +%%% Never: answers a caller directly with anything but `{MRef, Reply}'. +%%% +%%% @private +-module(py_context_embedded). + +-export([init/4]). +%% Used by py_reactor_context +-export([extend_erlang_module_in_context/1]). + +-record(state, { + ref :: reference(), + id :: pos_integer(), + interp_id :: non_neg_integer(), + event_state = #{} :: map(), %% #{loop_ref => ref(), worker_pid => pid()} + callback_handler :: pid() | undefined, %% For thread-model callback handling + %% Worker loop (start_loop/1): request id of the run_forever exec, the + %% owner that gets {py_loop_exit, Ctx, Result}, its monitor, and the + %% callers waiting in stop_loop/2 + loop_req :: reference() | undefined, + loop_owner :: pid() | undefined, + loop_owner_mon :: reference() | undefined, + loop_stop_waiters = [] :: [{pid(), reference()}] +}). + +%% Time given to a running loop to exit after py_context:interrupt/1 +-define(LOOP_INTERRUPT_GRACE_MS, 3000). + +%% @private +init(Parent, Id, Mode, Opts) -> + process_flag(trap_exit, true), + case create_context(Mode) of + {ok, Ref, InterpId} -> + %% Publish the NIF reference so interrupt/1 can reach it while + %% this process is blocked in a NIF + py_context:register_nif_ref(Ref), + case apply_memory_limit(Ref, Opts) of + ok -> + init_started(Parent, Id, Ref, InterpId, Opts); + {error, LimitError} -> + py_context:unregister_nif_ref(), + try py_nif:context_destroy(Ref) catch _:_ -> ok end, + Parent ! {self(), {error, LimitError}} + end; + {error, Reason} -> + Parent ! {self(), {error, Reason}} + end. + +%% @private +apply_memory_limit(Ref, Opts) -> + case maps:get(memory_limit, Opts, undefined) of + undefined -> + ok; + Bytes when is_integer(Bytes), Bytes >= 0 -> + py_nif:context_set_memory_limit(Ref, Bytes); + Other -> + {error, {invalid_memory_limit, Other}} + end. + +%% @private +init_started(Parent, Id, Ref, InterpId, Opts) -> + %% Apply all registered imports and paths to this interpreter + apply_registered_imports(Ref), + apply_registered_paths(Ref), + %% Apply preload code (populates globals for process-local envs) + apply_preload(Ref), + %% Per-context preload from new/1 (imports the app once per worker) + case maps:get(preload, Opts, undefined) of + undefined -> ok; + PreCode when is_binary(PreCode); is_list(PreCode) -> + case handle_exec_with_async(Ref, iolist_to_binary(PreCode)) of + ok -> ok; + {error, PreErr} -> + error_logger:warning_msg( + "py_context ~p: preload failed: ~p~n", [InterpId, PreErr]) + end + end, + %% For subinterpreters, create a dedicated event worker + EventState = setup_event_worker(Ref, InterpId), + %% For thread-model subinterpreters, spawn a dedicated callback handler + %% because the main context process will be blocked in the NIF + CallbackHandler = case maps:get(mode, EventState, normal) of + thread_model -> + Handler = spawn_callback_handler(Ref), + ok = py_nif:context_set_callback_handler(Ref, Handler), + Handler; + _ -> + undefined + end, + Parent ! {self(), started}, + State = #state{ + ref = Ref, + id = Id, + interp_id = InterpId, + event_state = EventState, + callback_handler = CallbackHandler + }, + loop(State). + +%% @private Create event worker for subinterpreter contexts +setup_event_worker(Ref, InterpId) -> + case py_nif:context_get_event_loop(Ref) of + {ok, LoopRef} -> + %% This is a subinterpreter - create dedicated event worker + WorkerId = iolist_to_binary(["ctx_", integer_to_list(InterpId)]), + case py_event_worker:start_link(WorkerId, LoopRef) of + {ok, WorkerPid} -> + ok = py_nif:event_loop_set_worker(LoopRef, WorkerPid), + %% Extend erlang module with event loop functions + extend_erlang_module_in_context(Ref), + #{loop_ref => LoopRef, worker_pid => WorkerPid}; + {error, WorkerError} -> + error_logger:warning_msg( + "py_context ~p: Failed to start event worker: ~p~n", + [InterpId, WorkerError]), + #{} + end; + {error, not_subinterp} -> + %% Worker mode - uses shared router (lazy initialization) + #{}; + {error, event_loop_owned_by_thread} -> + %% Thread-model subinterpreter: event loop is managed by dedicated thread. + %% This is expected behavior, not a failure. + #{mode => thread_model}; + {error, Reason} -> + error_logger:warning_msg( + "py_context ~p: Failed to get event loop: ~p~n", + [InterpId, Reason]), + #{} + end. + +%% @private Extend the erlang module with event loop functions in a subinterpreter +extend_erlang_module_in_context(Ref) -> + PrivDir = code:priv_dir(erlang_python), + Code = iolist_to_binary([ + "import sys\n", + "priv_dir = '", PrivDir, "'\n", + "if priv_dir not in sys.path:\n", + " sys.path.insert(0, priv_dir)\n", + "import erlang\n", + "if hasattr(erlang, '_extend_erlang_module'):\n", + " erlang._extend_erlang_module(priv_dir)\n" + ]), + case py_nif:context_exec(Ref, Code) of + ok -> ok; + {error, Reason} -> + error_logger:warning_msg( + "py_context: Failed to extend erlang module: ~p~n", [Reason]), + ok + end. + +%% @private Apply all imports from the global registry to this interpreter. +%% +%% Called when a new interpreter is created to pre-warm the module cache +%% with all modules registered via py_import:ensure_imported/1,2. +apply_registered_imports(Ref) -> + case py_import:all_imports() of + [] -> ok; + Imports -> py_nif:interp_apply_imports(Ref, Imports) + end. + +%% @private Apply all paths from the global registry to this interpreter. +%% +%% Called when a new interpreter is created to add all registered paths +%% to sys.path. +apply_registered_paths(Ref) -> + case py_import:all_paths() of + [] -> ok; + Paths -> py_nif:interp_apply_paths(Ref, Paths) + end. + +%% @private Apply preload code to the interpreter's globals. +%% +%% Called when a new interpreter is created. The preload code populates +%% the context's globals dict, which process-local environments inherit. +apply_preload(Ref) -> + py_preload:apply_preload(Ref). + +%% @private +create_context(worker) -> + py_nif:context_create(worker); +create_context(owngil) -> + %% OWN_GIL mode requires Python 3.14+ due to C extension bugs in earlier versions + case py_nif:owngil_supported() of + true -> py_nif:context_create(owngil); + false -> {error, owngil_requires_python314} + end. + +%% @private +%% Main context loop. Handles requests and uses suspension-based callback support. +loop(#state{ref = Ref, interp_id = InterpId, loop_req = LoopReq} = State) -> + receive + %% ---- worker loop management (start_loop/stop_loop/loop_ref) ---- + {start_loop, From, MRef, _Owner} when LoopReq =/= undefined -> + From ! {MRef, {error, already_running}}, + loop(State); + + {start_loop, From, MRef, Owner} -> + {Reply, NewState} = do_start_loop(Owner, State), + From ! {MRef, Reply}, + loop(NewState); + + {stop_loop, From, MRef, _GraceMs} when LoopReq =:= undefined -> + From ! {MRef, {error, no_loop}}, + loop(State); + + {stop_loop, From, MRef, GraceMs} -> + loop(begin_stop_loop(From, MRef, GraceMs, State)); + + {loop_ref, From, MRef} -> + From ! {MRef, context_loop_ref(State)}, + loop(State); + + {py_result, LoopReq, Result} when LoopReq =/= undefined -> + loop(loop_exited(Result, State)); + + {loop_stop_deadline, LoopReq} when LoopReq =/= undefined -> + %% Cooperative stop did not land: interrupt the thread + _ = py_nif:context_interrupt(Ref), + erlang:send_after(?LOOP_INTERRUPT_GRACE_MS, self(), + {loop_interrupt_deadline, LoopReq}), + loop(State); + + {loop_interrupt_deadline, LoopReq} when LoopReq =/= undefined -> + [W ! {M, {error, timeout}} || {W, M} <- State#state.loop_stop_waiters], + loop(State#state{loop_stop_waiters = []}); + + {loop_stop_deadline, _} -> + loop(State); + {loop_interrupt_deadline, _} -> + loop(State); + + {'DOWN', Mon, process, _Owner, _Reason} + when Mon =:= State#state.loop_owner_mon, LoopReq =/= undefined -> + %% Owner is gone: nobody will hear the exit, stop the loop + loop(begin_stop_loop(undefined, undefined, 5000, + State#state{loop_owner_mon = undefined})); + + {async_result, _TaskRef, _} -> + %% Result of a coroutine this process submitted (loop stop) - drop + loop(State); + + %% ---- while a worker loop runs, the thread is not available ---- + {call, From, MRef, _, _, _, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + {call, From, MRef, _, _, _, _, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + {eval, From, MRef, _, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + {eval, From, MRef, _, _, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + {exec, From, MRef, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + {exec, From, MRef, _, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + {call_method, From, MRef, _, _, _} when LoopReq =/= undefined -> + From ! {MRef, {error, loop_running}}, loop(State); + + {stop, From, MRef} when LoopReq =/= undefined -> + %% Get the thread out of the loop before destroying the context, + %% otherwise context_destroy waits for a thread that never returns + terminate(normal, stop_running_loop(State)), + From ! {MRef, ok}; + + {'EXIT', _Pid, Reason} = Exit when LoopReq =/= undefined, + (Reason =:= shutdown orelse Reason =:= kill orelse + (is_tuple(Reason) andalso element(1, Reason) =:= shutdown)) -> + self() ! Exit, + loop(stop_running_loop(State)); + + {call, From, MRef, Module, Func, Args, Kwargs} -> + Result = handle_call_with_suspension(Ref, Module, Func, Args, Kwargs), + From ! {MRef, Result}, + loop(State); + + %% Call with process-local environment (worker mode) + {call, From, MRef, Module, Func, Args, Kwargs, EnvRef} -> + Result = handle_call_with_suspension_and_env(Ref, Module, Func, Args, Kwargs, EnvRef), + From ! {MRef, Result}, + loop(State); + + {eval, From, MRef, Code, Locals} -> + Result = handle_eval_with_suspension(Ref, Code, Locals), + From ! {MRef, Result}, + loop(State); + + %% Eval with process-local environment (worker mode) + {eval, From, MRef, Code, Locals, EnvRef} -> + Result = handle_eval_with_suspension_and_env(Ref, Code, Locals, EnvRef), + From ! {MRef, Result}, + loop(State); + + {exec, From, MRef, Code} -> + Result = handle_exec_with_async(Ref, Code), + From ! {MRef, Result}, + loop(State); + + %% Exec with process-local environment (worker mode). + %% Async dispatch with sync fallback (mirrors call/eval). + {exec, From, MRef, Code, EnvRef} -> + Result = handle_exec_with_async_and_env(Ref, Code, EnvRef), + From ! {MRef, Result}, + loop(State); + + {call_method, From, MRef, ObjRef, Method, Args} -> + Result = py_nif:context_call_method(Ref, ObjRef, Method, Args), + From ! {MRef, Result}, + loop(State); + + {get_interp_id, From, MRef} -> + From ! {MRef, {ok, InterpId}}, + loop(State); + + {is_subinterp, From, MRef} -> + %% Check the interp_id to determine if this is a subinterpreter + %% Subinterpreters have interp_id > 0 (main interpreter is 0) + %% But actually we need to check the mode, not just interp_id + IsSubinterp = is_context_subinterp(Ref), + From ! {MRef, IsSubinterp}, + loop(State); + + {create_local_env, From, MRef} -> + %% Create env inside this context's interpreter + Result = py_nif:create_local_env(Ref), + From ! {MRef, Result}, + loop(State); + + {get_nif_ref, From, MRef} -> + From ! {MRef, Ref}, + loop(State); + + {stop, From, MRef} -> + terminate(normal, State), + From ! {MRef, ok}; + + {'EXIT', Pid, Reason} -> + %% Handle EXIT from linked processes + case State#state.callback_handler of + Pid -> + %% Callback handler died - restart it for thread-model contexts + error_logger:warning_msg( + "py_context ~p: Callback handler died: ~p, restarting~n", + [InterpId, Reason]), + NewHandler = spawn_callback_handler(Ref), + ok = py_nif:context_set_callback_handler(Ref, NewHandler), + NewState = State#state{callback_handler = NewHandler}, + loop(NewState); + _ -> + case State#state.event_state of + #{worker_pid := Pid} -> + %% Event worker died - log and continue (degraded asyncio support) + error_logger:warning_msg( + "py_context ~p: Event worker died: ~p~n", + [InterpId, Reason]), + NewState = State#state{event_state = #{}}, + loop(NewState); + _ when Reason =:= shutdown; Reason =:= kill -> + %% Supervisor shutdown or kill signal - clean exit + terminate(Reason, State); + _ when is_tuple(Reason), element(1, Reason) =:= shutdown -> + %% Supervisor shutdown with extra info: {shutdown, _} + terminate(Reason, State); + _ -> + %% Ignore EXIT from other processes + loop(State) + end + end + end. + +%% ============================================================================ +%% Worker loop helpers +%% ============================================================================ + +%% @private Loop reference: the context's own loop (owngil) or the shared +%% main-interpreter loop (worker mode) +context_loop_ref(#state{event_state = #{loop_ref := LoopRef}}) -> + {ok, LoopRef}; +context_loop_ref(_State) -> + py_event_loop:get_loop(). + +%% @private Start run_forever on the context thread through the async exec +%% path, so this process stays free to serve loop_ref/stop_loop and the +%% dirty schedulers are not held. +do_start_loop(Owner, #state{ref = Ref} = State) -> + case context_loop_ref(State) of + {ok, _} -> + LoopReq = make_ref(), + case py_nif:context_call_async(Ref, self(), LoopReq, <<"erlang">>, + <<"_run_loop_forever">>, [self()], #{}) of + {enqueued, LoopReq} -> + %% Wait for the loop to actually run before answering, so + %% a submit right after start_loop finds it + receive + {py_loop_started} -> + Mon = case is_pid(Owner) of + true -> erlang:monitor(process, Owner); + false -> undefined + end, + {ok, State#state{loop_req = LoopReq, loop_owner = Owner, + loop_owner_mon = Mon, loop_stop_waiters = []}}; + {py_result, LoopReq, {error, Reason}} -> + {{error, Reason}, State}; + {py_result, LoopReq, Other} -> + {{error, {loop_exited, Other}}, State} + after 10000 -> + {{error, loop_start_timeout}, State} + end; + {error, Reason} -> + {{error, Reason}, State} + end; + {error, Reason} -> + {{error, Reason}, State} + end. + +%% @private Ask the running loop to stop from inside, arm the interrupt +%% deadline, and remember who to answer once it has exited. +begin_stop_loop(From, MRef, GraceMs, #state{loop_req = LoopReq} = State) -> + Waiters = case From of + undefined -> State#state.loop_stop_waiters; + _ -> [{From, MRef} | State#state.loop_stop_waiters] + end, + case context_loop_ref(State) of + {ok, LoopRef} -> + _ = py_nif:submit_task(LoopRef, self(), make_ref(), + <<"erlang">>, <<"_stop_loop">>, [], #{}); + _ -> + ok + end, + erlang:send_after(GraceMs, self(), {loop_stop_deadline, LoopReq}), + State#state{loop_stop_waiters = Waiters}. + +%% @private The exec running the loop returned: tell the owner and the +%% stop_loop callers, clear the loop state. +loop_exited(Result, #state{loop_owner = Owner, loop_owner_mon = Mon, + loop_stop_waiters = Waiters} = State) -> + case Mon of + undefined -> ok; + _ -> erlang:demonitor(Mon, [flush]) + end, + case is_pid(Owner) of + true -> Owner ! {py_loop_exit, self(), Result}; + false -> ok + end, + [W ! {M, ok} || {W, M} <- Waiters], + State#state{loop_req = undefined, loop_owner = undefined, + loop_owner_mon = undefined, loop_stop_waiters = []}. + +%% @private Synchronous stop used before terminate: interrupt and wait a +%% bounded time for the exec to return. +stop_running_loop(#state{ref = Ref, loop_req = LoopReq} = State) -> + _ = py_nif:context_interrupt(Ref), + receive + {py_result, LoopReq, Result} -> + loop_exited(Result, State) + after ?LOOP_INTERRUPT_GRACE_MS -> + loop_exited({error, timeout}, State) + end. + +%% @private Clean up resources on termination +terminate(_Reason, #state{ref = Ref, event_state = EventState, callback_handler = CallbackHandler}) -> + py_context:unregister_nif_ref(), + %% Stop the callback handler if it exists + case CallbackHandler of + Pid when is_pid(Pid) -> + Pid ! stop; + _ -> + ok + end, + %% Stop the event worker first (if it exists and is still alive) + case EventState of + #{worker_pid := WorkerPid} -> + try gen_server:stop(WorkerPid, normal, 5000) catch _:_ -> ok end; + _ -> + ok + end, + %% Destroy the Python context + try py_nif:context_destroy(Ref) catch _:_ -> ok end, + ok. + +%% ============================================================================ +%% Blocking callback handling (for thread-model subinterpreters) +%% ============================================================================ +%% +%% Thread-model subinterpreters use blocking pipe-based callbacks because +%% the suspension mechanism doesn't work when Python runs in a dedicated thread. +%% The Python thread blocks waiting for a response on the callback pipe. +%% +%% A separate callback handler process is spawned because the main context +%% process is blocked in the NIF (dispatch_to_thread) and cannot receive messages. + +%% @private +%% Spawn a dedicated callback handler process for thread-model subinterpreters. +spawn_callback_handler(Ref) -> + spawn_link(fun() -> callback_handler_loop(Ref) end). + +%% @private +%% Callback handler loop - receives erlang_callback messages and responds. +callback_handler_loop(Ref) -> + receive + {erlang_callback, _CallbackId, FuncName, Args} -> + handle_blocking_callback(Ref, FuncName, Args), + callback_handler_loop(Ref); + stop -> + ok + end. + +%% @private +%% Handle a blocking callback from a thread-model subinterpreter. +%% Executes the callback and writes the response to the callback pipe. +handle_blocking_callback(Ref, FuncName, Args) -> + %% Convert Args from tuple to list if needed + ArgsList = case Args of + T when is_tuple(T) -> tuple_to_list(T); + L when is_list(L) -> L; + _ -> [Args] + end, + %% Execute the registered function + Response = case py_callback:execute(FuncName, ArgsList) of + {ok, Result} -> + %% Format: status_byte (2=ok, ETF) + external term format + <<2, (term_to_binary(Result))/binary>>; + {error, {not_found, Name}} -> + ErrMsg = iolist_to_binary( + io_lib:format("Function '~s' not registered", [Name])), + <<1, ErrMsg/binary>>; + {error, {Class, Reason, _Stack}} -> + ErrMsg = iolist_to_binary( + io_lib:format("~p: ~p", [Class, Reason])), + <<1, ErrMsg/binary>> + end, + %% Write response to context's callback pipe + py_nif:context_write_callback_response(Ref, Response). + +%% ============================================================================ +%% Suspension-based callback handling +%% ============================================================================ +%% +%% When Python calls erlang.call(), the NIF returns {suspended, ...} instead of +%% blocking. We handle the callback inline and then resume Python execution. +%% This enables unlimited nesting depth without deadlock. + +%% @private +%% Handle call with potential suspension for callbacks +handle_call_with_suspension(Ref, Module, Func, Args, Kwargs) -> + RequestId = make_ref(), + case py_nif:context_call_async(Ref, self(), RequestId, Module, Func, Args, Kwargs) of + {enqueued, RequestId} -> + %% Async dispatch succeeded - wait for result message + wait_for_async_result(Ref, RequestId); + {error, Reason} -> + {error, Reason} + end. + + +%% @private +%% Handle eval with potential suspension for callbacks +handle_eval_with_suspension(Ref, Code, Locals) -> + RequestId = make_ref(), + case py_nif:context_eval_async(Ref, self(), RequestId, Code, Locals) of + {enqueued, RequestId} -> + %% Async dispatch succeeded - wait for result message + wait_for_async_result(Ref, RequestId); + {error, Reason} -> + {error, Reason} + end. + +%% @private +%% Handle exec with async dispatch +handle_exec_with_async(Ref, Code) -> + RequestId = make_ref(), + case py_nif:context_exec_async(Ref, self(), RequestId, Code) of + {enqueued, RequestId} -> + wait_for_async_result(Ref, RequestId); + {error, Reason} -> + {error, Reason} + end. + + +%% @private +%% Wait for async result from worker thread +%% The worker thread sends {py_result, RequestId, Result} when done. +%% +%% Drains stale {py_result, _, _} messages from prior timed-out +%% requests before the matching receive so a context that experiences +%% repeat timeouts doesn't grow an unbounded mailbox: when +%% wait_for_async_result/2 returns {error, async_timeout}, the C +%% worker can still finish later and deliver the result; without the +%% drain those messages would accumulate forever. +%% +%% Safe because the context process is the sole receiver for its own +%% async results and only one wait_for_async_result/2 is in flight at +%% a time, so the drain cannot consume the result of a concurrent live +%% request. +wait_for_async_result(Ref, RequestId) -> + drain_stale_async_results(RequestId), + receive + {py_result, RequestId, Result} -> + process_async_result(Ref, Result) + after 300000 -> %% 5 minute timeout + {error, async_timeout} + end. + +%% @private +drain_stale_async_results(CurrentId) -> + receive + {py_result, OldId, _} when OldId =/= CurrentId -> + drain_stale_async_results(CurrentId) + after 0 -> + ok + end. + +%% @private +%% Process the result from async dispatch +%% Handles suspension, schedule markers, and normal results. +process_async_result(Ref, {suspended, _CallbackId, StateRef, {FuncName, CallbackArgs}}) -> + CallbackResult = handle_callback_with_nested_receive(Ref, FuncName, CallbackArgs), + resume_and_continue(Ref, StateRef, CallbackResult); +process_async_result(Ref, {schedule, CallbackName, CallbackArgs}) -> + handle_schedule(Ref, CallbackName, CallbackArgs); +process_async_result(_Ref, Result) -> + Result. + +%% @private +%% Handle call with process-local environment. +handle_call_with_suspension_and_env(Ref, Module, Func, Args, Kwargs, EnvRef) -> + RequestId = make_ref(), + case py_nif:context_call_with_env_async(Ref, self(), RequestId, + Module, Func, Args, Kwargs, + EnvRef) of + {enqueued, RequestId} -> + wait_for_async_result(Ref, RequestId); + {error, Reason} -> + {error, Reason} + end. + + +%% @private +%% Handle eval with process-local environment. +handle_eval_with_suspension_and_env(Ref, Code, Locals, EnvRef) -> + RequestId = make_ref(), + case py_nif:context_eval_with_env_async(Ref, self(), RequestId, + Code, Locals, EnvRef) of + {enqueued, RequestId} -> + wait_for_async_result(Ref, RequestId); + {error, Reason} -> + {error, Reason} + end. + + +%% @private +%% Handle exec with process-local environment via the same async-first +%% path used for call/eval. +handle_exec_with_async_and_env(Ref, Code, EnvRef) -> + RequestId = make_ref(), + case py_nif:context_exec_with_env_async(Ref, self(), RequestId, + Code, EnvRef) of + {enqueued, RequestId} -> + wait_for_async_result(Ref, RequestId); + {error, Reason} -> + {error, Reason} + end. + +%% @private +%% Check if a context is a subinterpreter (has interp_id > 0) +is_context_subinterp(Ref) -> + py_nif:context_interp_id(Ref) > 0. + +%% @private +%% Handle schedule marker - Python returned erlang.schedule() or schedule_py() +%% Execute the callback and return its result transparently to the caller. +%% +%% Special case for _execute_py: this callback is used by schedule_py() to +%% call back into Python with a different function. We handle it directly +%% using context_call to avoid recursion through py:call. +handle_schedule(Ref, <<"_execute_py">>, {Module, Func, Args, Kwargs}) -> + %% schedule_py callback: call Python function via context + CallArgs = case Args of + none -> []; + undefined -> []; + List when is_list(List) -> List; + Tuple when is_tuple(Tuple) -> tuple_to_list(Tuple); + _ -> [Args] + end, + CallKwargs = case Kwargs of + none -> #{}; + undefined -> #{}; + Map when is_map(Map) -> Map; + _ -> #{} + end, + handle_call_with_suspension(Ref, Module, Func, CallArgs, CallKwargs); +handle_schedule(_Ref, CallbackName, CallbackArgs) when is_binary(CallbackName) -> + %% Regular callback: execute via py_callback:execute + ArgsList = tuple_to_list(CallbackArgs), + case py_callback:execute(CallbackName, ArgsList) of + {ok, Result} -> + {ok, Result}; + {error, Reason} -> + {error, Reason} + end. + +%% @private +%% Handle callback, allowing nested py:eval/call to be processed. +%% We spawn a process to execute the callback so we can stay in a receive loop +%% for nested calls while the callback runs. +handle_callback_with_nested_receive(Ref, FuncName, CallbackArgs) -> + Parent = self(), + CallbackPid = spawn_link(fun() -> + Result = try + ArgsList = tuple_to_list(CallbackArgs), + case py_callback:execute(FuncName, ArgsList) of + {ok, Value} -> + {ok, <<2, (term_to_binary(Value))/binary>>}; + {error, Reason} -> + ErrMsg = iolist_to_binary(io_lib:format("~p", [Reason])), + {ok, <<1, ErrMsg/binary>>} + end + catch + Class:ExcReason:Stacktrace -> + ErrorMsg = iolist_to_binary(io_lib:format("~p:~p~n~p", + [Class, ExcReason, Stacktrace])), + {ok, <<1, ErrorMsg/binary>>} + end, + Parent ! {callback_result, self(), Result} + end), + %% Wait for callback, processing nested requests + wait_for_callback(Ref, CallbackPid). + +%% @private +%% Wait for callback result while processing nested py:call/eval requests. +%% This enables arbitrarily deep callback nesting. +wait_for_callback(Ref, CallbackPid) -> + receive + {callback_result, CallbackPid, Result} -> + Result; + + %% Handle nested py:call while waiting for callback + {call, From, MRef, Module, Func, Args, Kwargs} -> + NestedResult = handle_call_with_suspension(Ref, Module, Func, Args, Kwargs), + From ! {MRef, NestedResult}, + wait_for_callback(Ref, CallbackPid); + + %% Handle nested py:call while waiting for callback (with EnvRef) + {call, From, MRef, Module, Func, Args, Kwargs, EnvRef} -> + NestedResult = handle_call_with_suspension_and_env(Ref, Module, Func, Args, Kwargs, EnvRef), + From ! {MRef, NestedResult}, + wait_for_callback(Ref, CallbackPid); + + %% Handle nested py:eval while waiting for callback (without EnvRef) + {eval, From, MRef, Code, Locals} -> + NestedResult = handle_eval_with_suspension(Ref, Code, Locals), + From ! {MRef, NestedResult}, + wait_for_callback(Ref, CallbackPid); + + %% Handle nested py:eval while waiting for callback (with EnvRef) + {eval, From, MRef, Code, Locals, EnvRef} -> + NestedResult = handle_eval_with_suspension_and_env(Ref, Code, Locals, EnvRef), + From ! {MRef, NestedResult}, + wait_for_callback(Ref, CallbackPid); + + %% Handle nested py:exec while waiting for callback + {exec, From, MRef, Code} -> + NestedResult = py_nif:context_exec(Ref, Code), + From ! {MRef, NestedResult}, + wait_for_callback(Ref, CallbackPid); + + %% Handle nested py:exec while waiting for callback (with EnvRef) + {exec, From, MRef, Code, EnvRef} -> + NestedResult = py_nif:context_exec(Ref, Code, EnvRef), + From ! {MRef, NestedResult}, + wait_for_callback(Ref, CallbackPid); + + %% Handle nested call_method while waiting for callback + {call_method, From, MRef, ObjRef, Method, Args} -> + NestedResult = py_nif:context_call_method(Ref, ObjRef, Method, Args), + From ! {MRef, NestedResult}, + wait_for_callback(Ref, CallbackPid); + + %% Handle get_interp_id while waiting + {get_interp_id, From, MRef} -> + InterpId = py_nif:context_interp_id(Ref), + From ! {MRef, {ok, InterpId}}, + wait_for_callback(Ref, CallbackPid); + + %% Handle create_local_env while waiting + {create_local_env, From, MRef} -> + Result = py_nif:create_local_env(Ref), + From ! {MRef, Result}, + wait_for_callback(Ref, CallbackPid); + + {get_nif_ref, From, MRef} -> + From ! {MRef, Ref}, + wait_for_callback(Ref, CallbackPid) + end. + +%% @private +%% Resume suspended state, handle additional suspensions (nested callbacks) +resume_and_continue(Ref, StateRef, {ok, ResultBin}) -> + case py_nif:context_resume(Ref, StateRef, ResultBin) of + {suspended, _CallbackId2, StateRef2, {FuncName2, Args2}} -> + %% Another callback during resume - recursive handling + CallbackResult2 = handle_callback_with_nested_receive(Ref, FuncName2, Args2), + resume_and_continue(Ref, StateRef2, CallbackResult2); + FinalResult -> + FinalResult + end; +resume_and_continue(Ref, StateRef, {error, _} = Err) -> + _ = py_nif:context_cancel_resume(Ref, StateRef), + Err. + +%% ============================================================================ +%% Utility functions +%% ============================================================================ + +%% Callback results cross to Python as external term format (status byte 2) +%% and are decoded by term_to_py() in c_src/py_convert.c, the same +%% converter used for call arguments. The former Python-repr encoder was +%% removed in favour of it. diff --git a/src/py_reactor_context.erl b/src/py_reactor_context.erl index 0261f8e..52fe763 100644 --- a/src/py_reactor_context.erl +++ b/src/py_reactor_context.erl @@ -187,7 +187,7 @@ init(Parent, Id, Mode, Opts) -> py_nif:context_set_callback_handler(Ref, self()), %% Extend erlang module to make erlang.reactor available - py_context:extend_erlang_module_in_context(Ref), + py_context_embedded:extend_erlang_module_in_context(Ref), MaxConns = maps:get(max_connections, Opts, ?DEFAULT_MAX_CONNECTIONS), AppModule = maps:get(app_module, Opts, undefined), diff --git a/src/py_shared_dict.erl b/src/py_shared_dict.erl new file mode 100644 index 0000000..8ee103b --- /dev/null +++ b/src/py_shared_dict.erl @@ -0,0 +1,110 @@ +%% Copyright 2026 Benoit Chesneau +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%% @doc Process-scoped shared dictionaries (`py:shared_dict_*'). +%%% Thin wrappers over the `shared_dict_*' NIFs; use the `py' functions. +%%% @private +-module(py_shared_dict). + +-export([ + shared_dict_new/0, + shared_dict_get/2, + shared_dict_get/3, + shared_dict_set/3, + shared_dict_del/2, + shared_dict_keys/1, + shared_dict_destroy/1 +]). + + +%% @doc Create a new process-scoped SharedDict. +%% +%% Creates a SharedDict owned by the calling process. The dict is automatically +%% destroyed when the owning process terminates. Values are stored as pickled +%% bytes for cross-interpreter safety. +%% +%% == Example == +%% ``` +%% {ok, SD} = py:shared_dict_new(). +%% ok = py:shared_dict_set(SD, <<"config">>, #{host => <<"localhost">>}). +%% #{<<"host">> := <<"localhost">>} = py:shared_dict_get(SD, <<"config">>). +%% ''' +%% +%% @returns {ok, Reference} on success, {error, Reason} on failure +-spec shared_dict_new() -> {ok, reference()} | {error, term()}. +shared_dict_new() -> + py_nif:shared_dict_new(). + +%% @doc Get a value from SharedDict with default undefined. +%% +%% @param Handle SharedDict reference +%% @param Key Binary key +%% @returns Value or undefined if key not found +-spec shared_dict_get(reference(), binary()) -> term(). +shared_dict_get(Handle, Key) -> + shared_dict_get(Handle, Key, undefined). + +%% @doc Get a value from SharedDict with custom default. +%% +%% @param Handle SharedDict reference +%% @param Key Binary key +%% @param Default Default value if key not found +%% @returns Value or Default +-spec shared_dict_get(reference(), binary(), term()) -> term(). +shared_dict_get(Handle, Key, Default) when is_binary(Key) -> + py_nif:shared_dict_get(Handle, Key, Default). + +%% @doc Set a value in SharedDict. +%% +%% The value is pickled for cross-interpreter safety. +%% +%% @param Handle SharedDict reference +%% @param Key Binary key +%% @param Value Erlang term value (will be pickled) +%% @returns ok on success +-spec shared_dict_set(reference(), binary(), term()) -> ok | {error, term()}. +shared_dict_set(Handle, Key, Value) when is_binary(Key) -> + py_nif:shared_dict_set(Handle, Key, Value). + +%% @doc Delete a key from SharedDict. +%% +%% @param Handle SharedDict reference +%% @param Key Binary key +%% @returns ok (even if key didn't exist) +-spec shared_dict_del(reference(), binary()) -> ok. +shared_dict_del(Handle, Key) when is_binary(Key) -> + py_nif:shared_dict_del(Handle, Key). + +%% @doc Get all keys from SharedDict. +%% +%% @param Handle SharedDict reference +%% @returns List of binary keys +-spec shared_dict_keys(reference()) -> [binary()]. +shared_dict_keys(Handle) -> + py_nif:shared_dict_keys(Handle). + +%% @doc Explicitly destroy a SharedDict. +%% +%% Marks the SharedDict as destroyed and clears its Python dict. +%% After destruction, any further operations on this SharedDict will +%% return badarg. This is idempotent - calling on an already-destroyed +%% dict returns ok. +%% +%% @param Handle SharedDict reference +%% @returns ok +-spec shared_dict_destroy(reference()) -> ok. +shared_dict_destroy(Handle) -> + py_nif:shared_dict_destroy(Handle). + + diff --git a/src/py_stream.erl b/src/py_stream.erl new file mode 100644 index 0000000..badb8f8 --- /dev/null +++ b/src/py_stream.erl @@ -0,0 +1,255 @@ +%% Copyright 2026 Benoit Chesneau +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%% @doc Streaming results from Python generators into Erlang messages. +%%% Implementation of `py:stream/3,4', `py:stream_eval/1,2', `py:stream_start/3,4' +%%% and `py:stream_cancel/1'; use those. Owns the `{py_stream, Ref, ...}' +%%% event protocol and the Python-side generator driver. +%%% @private +-module(py_stream). + +-export([ + stream/3, + stream/4, + stream_eval/1, + stream_eval/2, + stream_start/3, + stream_start/4, + stream_cancel/1 +]). + + +%% @doc Stream results from a Python generator. +%% Returns a list of all yielded values. +-spec stream(py:py_module(), py:py_func(), py:py_args()) -> py:py_result(). +stream(Module, Func, Args) -> + stream(Module, Func, Args, #{}). + +%% @doc Stream results from a Python generator with kwargs. +-spec stream(py:py_module(), py:py_func(), py:py_args(), py:py_kwargs()) -> py:py_result(). +stream(Module, Func, Args, Kwargs) when map_size(Kwargs) == 0 -> + %% No kwargs - use stream_start and collect results + {ok, Ref} = stream_start(Module, Func, Args), + collect_stream(Ref, []); +stream(Module, Func, Args, Kwargs) -> + %% With kwargs - use eval approach + Ctx = py_context_router:get_context(), + ModuleBin = py_util:valid_py_module(py_util:to_binary(Module)), + FuncBin = py_util:valid_py_ident(py_util:to_binary(Func)), + KwargsCode = format_kwargs(Kwargs), + ArgsCode = format_args(Args), + Code = iolist_to_binary([ + <<"list(__import__('">>, ModuleBin, <<"').">>, FuncBin, + <<"(">>, ArgsCode, KwargsCode, <<"))">> + ]), + py_context:eval(Ctx, Code, #{}). + +%% @private Collect all stream events into a list +collect_stream(Ref, Acc) -> + receive + {py_stream, Ref, {data, Value}} -> + collect_stream(Ref, [Value | Acc]); + {py_stream, Ref, done} -> + {ok, lists:reverse(Acc)}; + {py_stream, Ref, {error, Reason}} -> + {error, Reason} + after 30000 -> + {error, timeout} + end. + +%% @private Format arguments for Python code +format_args([]) -> <<>>; +format_args(Args) -> + ArgStrs = [format_arg(A) || A <- Args], + iolist_to_binary(lists:join(<<", ">>, ArgStrs)). + +%% @private Format a single argument +format_arg(A) when is_integer(A) -> integer_to_binary(A); +format_arg(A) when is_float(A) -> float_to_binary(A); +format_arg(A) when is_binary(A) -> <<"'", (py_util:escape_py_literal(A))/binary, "'">>; +format_arg(A) when is_atom(A) -> <<"'", (py_util:escape_py_literal(atom_to_binary(A)))/binary, "'">>; +format_arg(A) when is_list(A) -> iolist_to_binary([<<"[">>, format_args(A), <<"]">>]); +format_arg(_) -> <<"None">>. + +%% @private Format kwargs for Python code +format_kwargs(Kwargs) when map_size(Kwargs) == 0 -> <<>>; +format_kwargs(Kwargs) -> + KwList = maps:fold(fun(K, V, Acc) -> + KB = py_util:valid_py_ident(if is_atom(K) -> atom_to_binary(K); is_binary(K) -> K end), + [<>, lists:join(<<", ">>, KwList)]). + +%% @doc Stream results from a Python generator expression. +%% Evaluates the expression and if it returns a generator, streams all values. +-spec stream_eval(string() | binary()) -> py:py_result(). +stream_eval(Code) -> + stream_eval(Code, #{}). + +%% @doc Stream results from a Python generator expression with local variables. +-spec stream_eval(string() | binary(), map()) -> py:py_result(). +stream_eval(Code, Locals) -> + %% Route through the new process-per-context system + %% Wrap the code in list() to collect generator values + Ctx = py_context_router:get_context(), + CodeBin = py_util:to_binary(Code), + WrappedCode = <<"list(", CodeBin/binary, ")">>, + py_context:eval(Ctx, WrappedCode, Locals). + +%%% ============================================================================ +%%% True Streaming API (Event-driven) +%%% ============================================================================ + +%% @doc Start a true streaming iteration from a Python generator. +%% +%% Unlike stream/3,4 which collects all values at once, this function +%% returns immediately with a reference and sends values as events +%% to the calling process as they are yielded. +%% +%% Events sent to the owner process: +%% - `{py_stream, Ref, {data, Value}}' - Each yielded value +%% - `{py_stream, Ref, done}' - Stream completed +%% - `{py_stream, Ref, {error, Reason}}' - Stream error +%% +%% Accepts sync generators and async generators. An async generator is driven +%% on a private event loop, one value at a time; delivering a value blocks that +%% loop, so other coroutines on it do not progress between yields. +%% +%% Example: +%% ``` +%% {ok, Ref} = py:stream_start(builtins, iter, [[1,2,3,4,5]]), +%% receive_loop(Ref). +%% +%% receive_loop(Ref) -> +%% receive +%% {py_stream, Ref, {data, Value}} -> +%% io:format("Got: ~p~n", [Value]), +%% receive_loop(Ref); +%% {py_stream, Ref, done} -> +%% io:format("Complete~n"); +%% {py_stream, Ref, {error, Reason}} -> +%% io:format("Error: ~p~n", [Reason]) +%% after 30000 -> +%% timeout +%% end. +%% ''' +-spec stream_start(py:py_module(), py:py_func(), py:py_args()) -> {ok, reference()}. +stream_start(Module, Func, Args) -> + stream_start(Module, Func, Args, #{}). + +%% @doc Start a true streaming iteration with options. +%% +%% Options: +%% - `owner => pid()' - Process to receive events (default: self()) +%% +%% @param Module Python module name +%% @param Func Python function name +%% @param Args Function arguments +%% @param Opts Options map +%% @returns {ok, Ref} where Ref is used to identify stream events +-spec stream_start(py:py_module(), py:py_func(), py:py_args(), map()) -> {ok, reference()}. +stream_start(Module, Func, Args, Opts) -> + Owner = maps:get(owner, Opts, self()), + Ref = make_ref(), + ModuleBin = py_util:to_binary(Module), + FuncBin = py_util:to_binary(Func), + RefHash = erlang:phash2(Ref), + %% Store owner and ref for Python to retrieve + %% Use binary keys because Python strings become binaries + py_state:store({<<"stream_owner">>, RefHash}, Owner), + py_state:store({<<"stream_ref">>, RefHash}, Ref), + py_state:store({<<"stream_args">>, RefHash}, Args), + %% Spawn an Erlang process to run the streaming iteration + spawn(fun() -> + stream_run_python(ModuleBin, FuncBin, RefHash) + end), + {ok, Ref}. + +%% @private Run the streaming via Python code +stream_run_python(ModuleBin0, FuncBin0, RefHash) -> + ModuleBin = py_util:valid_py_module(ModuleBin0), + FuncBin = py_util:valid_py_ident(FuncBin0), + RefHashBin = integer_to_binary(RefHash), + %% Build Python code that streams values using callbacks + Code = iolist_to_binary([ + <<"import erlang\n">>, + <<"_rh = ">>, RefHashBin, <<"\n">>, + <<"_args = erlang.call('state_get', ('stream_args', _rh))\n">>, + <<"if _args is None:\n">>, + <<" _args = []\n">>, + <<"try:\n">>, + <<" _mod = __import__('">>, ModuleBin, <<"')\n">>, + <<" _fn = getattr(_mod, '">>, FuncBin, <<"')\n">>, + <<" _gen = _fn(*_args) if _args else _fn()\n">>, + %% Async generators are driven on a private event loop. erlang.call is + %% a blocking pipe read, so it stalls that loop between yields, which + %% is fine for a sequential stream. + <<" if hasattr(_gen, '__anext__'):\n">>, + <<" import asyncio\n">>, + <<" async def _drive():\n">>, + <<" async for _val in _gen:\n">>, + <<" if erlang.call('_py_stream_cancelled', _rh):\n">>, + <<" erlang.call('_py_stream_send', _rh, 'error', 'cancelled')\n">>, + <<" return\n">>, + <<" erlang.call('_py_stream_send', _rh, 'data', _val)\n">>, + <<" erlang.call('_py_stream_send', _rh, 'done', None)\n">>, + <<" asyncio.run(_drive())\n">>, + <<" else:\n">>, + <<" for _val in _gen:\n">>, + <<" if erlang.call('_py_stream_cancelled', _rh):\n">>, + <<" erlang.call('_py_stream_send', _rh, 'error', 'cancelled')\n">>, + <<" break\n">>, + <<" erlang.call('_py_stream_send', _rh, 'data', _val)\n">>, + <<" else:\n">>, + <<" erlang.call('_py_stream_send', _rh, 'done', None)\n">>, + <<"except Exception as _e:\n">>, + <<" erlang.call('_py_stream_send', _rh, 'error', str(_e))\n">>, + <<"finally:\n">>, + <<" erlang.call('_py_stream_cleanup', _rh)\n">> + ]), + %% Execute the streaming code + case py:exec(Code) of + ok -> ok; + {error, Reason} -> + %% Try to notify owner of error + case py_state:fetch({<<"stream_owner">>, RefHash}) of + {ok, Owner} -> + case py_state:fetch({<<"stream_ref">>, RefHash}) of + {ok, Ref} -> + Owner ! {py_stream, Ref, {error, Reason}}, + py_state:remove({<<"stream_owner">>, RefHash}), + py_state:remove({<<"stream_ref">>, RefHash}), + py_state:remove({<<"stream_args">>, RefHash}); + _ -> ok + end; + _ -> ok + end + end. + +%% @doc Cancel an active stream. +%% +%% Sends a cancellation signal to stop the stream iteration. +%% Any pending values may still be delivered before the stream stops. +%% +%% @param Ref The stream reference from stream_start/3,4 +%% @returns ok +-spec stream_cancel(reference()) -> ok. +stream_cancel(Ref) when is_reference(Ref) -> + %% Store cancellation flag that the streaming task checks + %% Use hash because we can't pass Erlang refs to Python callbacks easily + %% Use binary key because Python strings become binaries + RefHash = erlang:phash2(Ref), + py_state:store({<<"stream_cancelled_hash">>, RefHash}, true), + ok. + diff --git a/src/py_util.erl b/src/py_util.erl index 5ed4319..9ae86b1 100644 --- a/src/py_util.erl +++ b/src/py_util.erl @@ -18,7 +18,10 @@ -module(py_util). -export([ - to_binary/1 + to_binary/1, + escape_py_literal/1, + valid_py_module/1, + valid_py_ident/1 ]). %%% ============================================================================ @@ -33,3 +36,47 @@ to_binary(List) when is_list(List) -> list_to_binary(List); to_binary(Bin) when is_binary(Bin) -> Bin. + +%% @doc Escape a binary for use inside a single-quoted Python string literal. +-spec escape_py_literal(binary()) -> binary(). +escape_py_literal(Bin) when is_binary(Bin) -> + << <<(escape_py_byte(B))/binary>> || <> <= Bin >>. + +escape_py_byte($') -> <<"\\'">>; +escape_py_byte($\\) -> <<"\\\\">>; +escape_py_byte($\n) -> <<"\\n">>; +escape_py_byte($\r) -> <<"\\r">>; +escape_py_byte($\t) -> <<"\\t">>; +escape_py_byte(B) when B < 16#20; B =:= 16#7f -> + list_to_binary(io_lib:format("\\x~2.16.0b", [B])); +escape_py_byte(B) -> <>. + +%% @private Validate a dotted Python module path (each segment an identifier). +valid_py_module(Bin) when is_binary(Bin), byte_size(Bin) > 0 -> + Segments = binary:split(Bin, <<".">>, [global]), + lists:foreach(fun valid_py_ident/1, Segments), + Bin; +valid_py_module(Other) -> + error({invalid_python_identifier, Other}). + +ident_ok(<<>>, first) -> false; %% empty segment (leading/trailing/double dot) +ident_ok(<<>>, rest) -> true; +ident_ok(<>, first) + when (C >= $A andalso C =< $Z); (C >= $a andalso C =< $z); C =:= $_ -> + ident_ok(Rest, rest); +ident_ok(<>, rest) + when (C >= $A andalso C =< $Z); (C >= $a andalso C =< $z); + (C >= $0 andalso C =< $9); C =:= $_ -> + ident_ok(Rest, rest); +ident_ok(_, _) -> false. + +%% @private Validate a Python identifier ([A-Za-z_][A-Za-z0-9_]*). Crashes on a +%% non-conforming value so an attacker-controlled module/func/kwarg name can't +%% inject code at an identifier position (where quoting is meaningless). +valid_py_ident(Bin) when is_binary(Bin), byte_size(Bin) > 0 -> + case ident_ok(Bin, first) of + true -> Bin; + false -> error({invalid_python_identifier, Bin}) + end; +valid_py_ident(Other) -> + error({invalid_python_identifier, Other}). diff --git a/src/py_venv.erl b/src/py_venv.erl new file mode 100644 index 0000000..731f6cd --- /dev/null +++ b/src/py_venv.erl @@ -0,0 +1,350 @@ +%% Copyright 2026 Benoit Chesneau +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%% @doc Virtual environments: create, install dependencies, activate. +%%% Implementation of `py:ensure_venv/2,3', `py:activate_venv/1', +%%% `py:deactivate_venv/0', `py:venv_info/0' and `py:python_executable/0'; +%%% use those. Owns the venv layout under the configured directory and the +%%% pip/uv invocation. +%%% @private +-module(py_venv). + +-export([ + ensure_venv/2, + ensure_venv/3, + python_executable/0, + activate_venv/1, + deactivate_venv/0, + venv_info/0 +]). + + +%% @doc Ensure a virtual environment exists and activate it. +%% +%% Creates a venv at `Path' if it doesn't exist, installs dependencies from +%% `RequirementsFile', and activates the venv. +%% +%% RequirementsFile can be: +%% - `"requirements.txt"' - standard pip requirements file +%% - `"pyproject.toml"' - PEP 621 project file (installs with -e .) +%% +%% Example: +%% ``` +%% ok = py:ensure_venv("priv/venv", "requirements.txt"). +%% ''' +-spec ensure_venv(string() | binary(), string() | binary()) -> ok | {error, term()}. +ensure_venv(Path, RequirementsFile) -> + ensure_venv(Path, RequirementsFile, []). + +%% @doc Ensure a virtual environment exists with options. +%% +%% Options: +%% - `{extras, [string()]}' - Install optional dependencies (pyproject.toml) +%% - `{installer, uv | pip}' - Package installer (default: auto-detect) +%% - `{python, string()}' - Python executable for venv creation +%% - `force' - Recreate venv even if it exists +%% +%% Example: +%% ``` +%% %% With pyproject.toml and dev extras +%% ok = py:ensure_venv("priv/venv", "pyproject.toml", [ +%% {extras, ["dev", "test"]} +%% ]). +%% +%% %% Force uv installer +%% ok = py:ensure_venv("priv/venv", "requirements.txt", [ +%% {installer, uv} +%% ]). +%% ''' +-spec ensure_venv(string() | binary(), string() | binary(), list()) -> ok | {error, term()}. +ensure_venv(Path, RequirementsFile, Opts) -> + PathStr = to_string(Path), + ReqFileStr = to_string(RequirementsFile), + Force = proplists:get_bool(force, Opts), + %% Create venv if needed + VenvReady = case venv_exists(PathStr) of + true when not Force -> + ok; + _ -> + create_venv(PathStr, Opts) + end, + case VenvReady of + ok -> + %% Always install/update dependencies (pip/uv skip existing) + case install_deps(PathStr, ReqFileStr, Opts) of + ok -> + activate_venv(PathStr); + {error, _} = Err -> + Err + end; + {error, _} = Err -> + Err + end. + +%% @private Check if venv exists by looking for pyvenv.cfg +-spec venv_exists(string()) -> boolean(). +venv_exists(Path) -> + filelib:is_file(filename:join(Path, "pyvenv.cfg")). + +%% @private Create a new virtual environment +-spec create_venv(string(), list()) -> ok | {error, term()}. +create_venv(Path, Opts) -> + Installer = detect_installer(Opts), + Python = case proplists:get_value(python, Opts, undefined) of + undefined -> get_python_executable(); + P -> P + end, + case Installer of + uv -> + %% uv venv is faster, use --python to match the running interpreter + run_cmd(uv_exe(), ["venv", "--python", Python, Path], []); + pip -> + run_cmd(Python, ["-m", "venv", Path], []) + end. + +%% @private Get the Python executable path +%% When embedded, sys.executable returns the embedding app (beam.smp) +%% so we reconstruct the path from sys.prefix and version info +%% @doc Path of the Python interpreter matching the embedded runtime. +%% +%% Reconstructed from `sys.prefix' (when embedded, `sys.executable' is the +%% VM). Used as the default interpreter of isolated contexts and for venvs. +-spec python_executable() -> string(). +python_executable() -> + get_python_executable(). + + +-spec get_python_executable() -> string(). +get_python_executable() -> + %% Use a single expression to find the Python executable + %% Searches for pythonX.Y, python3, python in sys.prefix/bin (Unix) + %% or python.exe in sys.prefix (Windows) + Expr = <<"(lambda: (__import__('os').path.join(__import__('sys').prefix, 'python.exe') if __import__('sys').platform == 'win32' and __import__('os').path.isfile(__import__('os').path.join(__import__('sys').prefix, 'python.exe')) else next((p for p in [__import__('os').path.join(__import__('sys').prefix, 'bin', f'python{__import__(\"sys\").version_info.major}.{__import__(\"sys\").version_info.minor}'), __import__('os').path.join(__import__('sys').prefix, 'bin', 'python3'), __import__('os').path.join(__import__('sys').prefix, 'bin', 'python')] if __import__('os').path.isfile(p)), 'python3')))()">>, + case py:eval(Expr) of + {ok, Path} when is_binary(Path) -> binary_to_list(Path); + _ -> "python3" + end. + +%% @private Install dependencies from requirements file +-spec install_deps(string(), string(), list()) -> ok | {error, term()}. +install_deps(Path, RequirementsFile, Opts) -> + Installer = detect_installer(Opts), + {Exe, BaseArgs, PortOpts} = pip_command(Path, Installer), + Extras = proplists:get_value(extras, Opts, []), + + %% Determine file type and build the install argument list (no shell). + Args = case filename:extension(RequirementsFile) of + ".txt" -> + BaseArgs ++ ["install", "-r", RequirementsFile]; + ".toml" -> + %% pyproject.toml - install as editable. + %% filename:dirname returns "." for files without directory component + InstallPath = filename:dirname(RequirementsFile), + case Extras of + [] -> + BaseArgs ++ ["install", "-e", InstallPath]; + _ -> + ExtrasStr = string:join(Extras, ","), + BaseArgs ++ ["install", "-e", InstallPath ++ "[" ++ ExtrasStr ++ "]"] + end; + _ -> + BaseArgs ++ ["install", "-r", RequirementsFile] + end, + run_cmd(Exe, Args, PortOpts). + +%% @private Detect which installer to use (uv or pip) +-spec detect_installer(list()) -> uv | pip. +detect_installer(Opts) -> + case proplists:get_value(installer, Opts, auto) of + auto -> + case os:find_executable("uv") of + false -> pip; + _ -> uv + end; + Installer -> + Installer + end. + +%% @private Resolve the installer into {Executable, BaseArgs, PortOpts}. +%% For uv the venv is selected via the VIRTUAL_ENV port env option (not a shell +%% prefix); for pip we use the venv's own pip binary. +-spec pip_command(string(), uv | pip) -> {string(), [string()], list()}. +pip_command(VenvPath, uv) -> + {uv_exe(), ["pip"], [{env, [{"VIRTUAL_ENV", VenvPath}]}]}; +pip_command(VenvPath, pip) -> + PipExe = case os:type() of + {win32, _} -> + filename:join([VenvPath, "Scripts", "pip"]); + _ -> + filename:join([VenvPath, "bin", "pip"]) + end, + {PipExe, [], []}. + +%% @private Full path to the uv executable (falls back to the bare name). +-spec uv_exe() -> string(). +uv_exe() -> + case os:find_executable("uv") of + false -> "uv"; + P -> P + end. + +%% @private Run an executable with an argv list (no shell) and return ok or error. +-spec run_cmd(string(), [string()], list()) -> ok | {error, term()}. +run_cmd(Exe, Args, ExtraOpts) -> + case resolve_exe(Exe) of + {error, _} = Err -> + Err; + ExeFull -> + try open_port({spawn_executable, ExeFull}, + [exit_status, stderr_to_stdout, binary, {args, Args} | ExtraOpts]) of + Port -> collect_port(Port, []) + catch + error:Reason -> {error, {spawn_failed, Exe, Reason}} + end + end. + +%% @private Resolve an executable name/path to a full path (spawn_executable does +%% not search PATH). +-spec resolve_exe(string()) -> string() | {error, term()}. +resolve_exe(Exe) -> + case filename:pathtype(Exe) of + absolute -> + case filelib:is_file(Exe) of + true -> Exe; + false -> {error, {executable_not_found, Exe}} + end; + _ -> + case os:find_executable(Exe) of + false -> {error, {executable_not_found, Exe}}; + Found -> Found + end + end. + +%% @private Collect a spawned port's output and exit status. +-spec collect_port(port(), [binary()]) -> ok | {error, term()}. +collect_port(Port, Acc) -> + receive + {Port, {data, Data}} -> + collect_port(Port, [Data | Acc]); + {Port, {exit_status, 0}} -> + ok; + {Port, {exit_status, Code}} -> + {error, {exit_code, Code, iolist_to_binary(lists:reverse(Acc))}} + after 300000 -> + try port_close(Port) catch _:_ -> ok end, + {error, timeout} + end. + +%% @private Convert to string +-spec to_string(string() | binary()) -> string(). +to_string(B) when is_binary(B) -> binary_to_list(B); +to_string(S) when is_list(S) -> S. + +%% @doc Activate a Python virtual environment. +%% This modifies sys.path to use packages from the specified venv. +%% The venv path should be the root directory (containing bin/lib folders). +%% +%% `.pth' files in the venv's site-packages directory are processed, so +%% editable installs created by uv, pip, or any PEP 517/660 compliant tool +%% work correctly. New paths are inserted at the front of sys.path so that +%% venv packages take priority over system packages. +%% +%% Example: +%% ``` +%% ok = py:activate_venv(<<"/path/to/myenv">>). +%% {ok, _} = py:call(sentence_transformers, 'SentenceTransformer', [<<"all-MiniLM-L6-v2">>]). +%% ''' +-spec activate_venv(string() | binary()) -> ok | {error, term()}. +activate_venv(VenvPath) -> + VenvBin = py_util:to_binary(VenvPath), + %% Find site-packages directory dynamically (venv may use different Python version) + %% Uses a single expression to avoid multiline code issues + FindSitePackages = <<"(lambda vp: __import__('os').path.join(vp, 'Lib', 'site-packages') if __import__('os').path.exists(__import__('os').path.join(vp, 'Lib', 'site-packages')) else next((sp for name in (__import__('os').listdir(__import__('os').path.join(vp, 'lib')) if __import__('os').path.isdir(__import__('os').path.join(vp, 'lib')) else []) if name.startswith('python') for sp in [__import__('os').path.join(vp, 'lib', name, 'site-packages')] if __import__('os').path.isdir(sp)), None))(_venv_path)">>, + case py:eval(FindSitePackages, #{<<"_venv_path">> => VenvBin}) of + {ok, SitePackages} when SitePackages =/= none, SitePackages =/= null -> + activate_venv_with_site_packages(VenvBin, SitePackages); + {ok, _} -> + {error, {invalid_venv, no_site_packages_found}}; + Error -> + Error + end. + +%% @private Activate venv with known site-packages path +activate_venv_with_site_packages(VenvBin, SitePackages) -> + %% Verify site-packages exists + case py:eval(<<"__import__('os').path.isdir(sp)">>, #{sp => SitePackages}) of + {ok, true} -> + %% Save original path if not already saved + {ok, _} = py:eval(<<"setattr(__import__('sys'), '_original_path', __import__('sys').path.copy()) if not hasattr(__import__('sys'), '_original_path') else None">>), + %% Set venv info + {ok, _} = py:eval(<<"setattr(__import__('sys'), '_active_venv', vp)">>, #{vp => VenvBin}), + {ok, _} = py:eval(<<"setattr(__import__('sys'), '_venv_site_packages', sp)">>, #{sp => SitePackages}), + %% Add site-packages and process .pth files (editable installs) + %% Note: We embed the site-packages path directly since exec doesn't support + %% variables and sys attributes may not persist across calls in subinterpreters + SitePackagesStr = binary_to_list(SitePackages), + ExecCode = iolist_to_binary([ + <<"import site as _site, sys as _sys\n">>, + <<"_sp = '">>, escape_python_string(SitePackagesStr), <<"'\n">>, + <<"_b = frozenset(_sys.path)\n">>, + <<"_site.addsitedir(_sp)\n">>, + <<"_sys.path[:] = [p for p in _sys.path if p not in _b] + [p for p in _sys.path if p in _b]\n">>, + <<"del _site, _sys, _b, _sp\n">> + ]), + ok = py:exec(ExecCode), + ok; + {ok, false} -> + {error, {invalid_venv, SitePackages}}; + Error -> + Error + end. + +%% @private Escape a string for embedding in Python code +escape_python_string(Str) -> + lists:flatmap(fun($') -> "\\'"; + ($\\) -> "\\\\"; + (C) -> [C] + end, Str). + + + + +%% @doc Deactivate the current virtual environment. +%% Restores sys.path to its original state. +-spec deactivate_venv() -> ok | {error, term()}. +deactivate_venv() -> + case py:eval(<<"hasattr(__import__('sys'), '_original_path')">>) of + {ok, true} -> + ok = py:exec(<<"import sys as _sys\n" + "_sys.path[:] = _sys._original_path\n" + "del _sys\n">>), + {ok, _} = py:eval(<<"delattr(__import__('sys'), '_original_path')">>), + {ok, _} = py:eval(<<"delattr(__import__('sys'), '_active_venv') if hasattr(__import__('sys'), '_active_venv') else None">>), + {ok, _} = py:eval(<<"delattr(__import__('sys'), '_venv_site_packages') if hasattr(__import__('sys'), '_venv_site_packages') else None">>), + ok; + {ok, false} -> + ok; + Error -> + Error + end. + +%% @doc Get information about the currently active virtual environment. +%% Returns a map with venv_path and site_packages, or none if no venv is active. +-spec venv_info() -> {ok, map() | none} | {error, term()}. +venv_info() -> + %% Check both attributes exist to handle partial activation/deactivation state + Code = <<"({'active': True, 'venv_path': __import__('sys')._active_venv, 'site_packages': __import__('sys')._venv_site_packages, 'sys_path': __import__('sys').path} if (hasattr(__import__('sys'), '_active_venv') and hasattr(__import__('sys'), '_venv_site_packages')) else {'active': False})">>, + py:eval(Code). + + From 945898157932290d796e791b6a04c6b88aadee6d Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sat, 29 Aug 2026 20:23:42 +0200 Subject: [PATCH 13/15] Check the code map against the tree (#83) A script, run by make and CI, that fails when a source file is missing from docs/code-map.md, an Erlang module has no moduledoc, or a module has no row in the new Modules table of the coverage audit. Running it once found two modules, seven headers and two Python files the map had missed. --- .github/workflows/ci.yml | 3 +++ CHANGELOG.md | 6 ++++++ Makefile | 7 ++++++- docs/code-map.md | 9 ++++++--- docs/contributing.md | 4 +++- scripts/check_code_map.sh | 21 ++++++++++++++++++++ test/coverage_audit.md | 40 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 85 insertions(+), 5 deletions(-) create mode 100755 scripts/check_code_map.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d532fa..a2e6ee9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -326,6 +326,9 @@ jobs: - name: Run dialyzer run: rebar3 dialyzer + - name: Check code map + run: make check-code-map + docs: name: Documentation runs-on: ubuntu-24.04 diff --git a/CHANGELOG.md b/CHANGELOG.md index 37bdd1f..1514ebf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,12 @@ virtual environments and shared dicts to `py_stream`, `py_venv` and `py_shared_dict`. The public API is unchanged. +### Documentation + +- `make check-code-map` (also run by CI) verifies that every source file is + in `docs/code-map.md`, every Erlang module has a moduledoc and a row in + the Modules table of `test/coverage_audit.md`. + ### Removed - The legacy worker API (`py_nif:worker_new/0,1`, `worker_call`, `worker_eval`, diff --git a/Makefile b/Makefile index 925c1ea..b44db69 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all compile test lint-docs clean +.PHONY: all compile test lint-docs check-code-map clean all: compile @@ -16,5 +16,10 @@ test: lint-docs: compile escript scripts/lint_doc_snippets.escript +# Every source file in docs/code-map.md, every module with a moduledoc and +# a row in the Modules table of test/coverage_audit.md. +check-code-map: + sh scripts/check_code_map.sh + clean: rebar3 clean diff --git a/docs/code-map.md b/docs/code-map.md index 534ce81..9da9c9b 100644 --- a/docs/code-map.md +++ b/docs/code-map.md @@ -23,7 +23,8 @@ exercised by suites). Guides are in `docs/`, suites in `test/`. Start with | `py_thread_handler` | Coordinator that gives each Python thread calling Erlang a handler process and a pipe | live | threading | `py_thread_callback_SUITE`, `py_reentrant_SUITE` | | `py_event_loop` | Main-interpreter asyncio loop: `run`, `create_task`, `await`, and the loop callbacks Python needs | live | asyncio | `py_event_loop_SUITE`, `py_async_task_SUITE` | | `py_event_loop_pool` | Several main-interpreter loops with process affinity | live | asyncio | `py_event_loop_pool_SUITE` | -| `py_event_worker`, `_sup`, `_registry` | One process per running loop receiving `enif_select` readiness and timers | live | event_loop_architecture | `py_event_loop_SUITE`, `py_fd_ops_SUITE` | +| `py_event_worker` | One process per running loop receiving `enif_select` readiness and timers | live | event_loop_architecture | `py_event_loop_SUITE`, `py_fd_ops_SUITE` | +| `py_event_worker_sup`, `py_event_worker_registry` | Supervisor and name registry of the event workers | live | event_loop_architecture | `py_event_loop_SUITE` | | `py_reactor_context` | FD-owning context for the protocol-based reactor | live | reactor | `py_reactor_SUITE` | | `py_channel`, `py_byte_channel` | Term and byte queues between Erlang and Python coroutines (NIF resources) | live | channel | `py_channel_SUITE`, `py_byte_channel_SUITE` | | `py_buffer` | Native streaming input buffer; shared variant delegates to `py_shm` | live | buffer, isolated | `py_buffer_SUITE`, `py_isolated_buffer_SUITE` | @@ -56,6 +57,7 @@ files. Editing `py_convert.c` alone does not compile it alone; build with | `py_logging.c` | Logging and tracing NIFs | live | | `py_mem_limit.c` | Per-interpreter memory caps (owngil) | live | | `py_util.c/.h` | Macros and helpers | live | +| `py_nif.h`, `py_event_loop.h`, `py_channel.h`, `py_buffer.h`, `py_reactor_buffer.h`, `py_subinterp_thread.h`, `py_util.h` | Declarations shared between the included files; the struct comments in `py_nif.h` and `py_event_loop.h` carry the lock contracts | live | The only code not on a live path is the "Test Helper Functions" section of `py_event_loop.c` (fd, pipe, TCP and UDP helpers the suites use). @@ -79,7 +81,7 @@ loop, channels and servers. | `_erlang_impl/_isolated.py` | Child runtime: socket frames, reader thread, re-entrant main loop, interrupt signal, asyncio loop, the `erlang` shim | isolated child | | `_erlang_impl/_shm.py` | `SharedMemory` and `SharedBuffer` wrappers over mmap | all | | `py_isolated_child.py` | Child launcher: rlimits, parent-death signal, cgroup join, connect | isolated child | -| `test_erlang_loop.py`, `tests/` | Python-side tests of the loop | test | +| `test_erlang_loop.py`, `test_async_task.py`, `test_channel_ref.py`, `tests/` | Python-side tests of the loop, tasks and channels | test | ## Tests (`test/`) @@ -87,7 +89,8 @@ Suites named `py__SUITE`. Cross-mode suites run the same cases in `worker` and `isolated` groups (`py_isolated_SUITE`, `py_isolated_vm_SUITE`, `py_isolated_shm_SUITE`, `py_isolated_buffer_SUITE`). Python helpers used by suites are `test/py_test_*.py`. `test/coverage_audit.md` maps public APIs to -cases. `test/test.config` holds node-wide settings (memory limits flag). +cases and every module to its suites; `make check-code-map` verifies this +page and that table against the tree. `test/test.config` holds node-wide settings (memory limits flag). ## Build and docs diff --git a/docs/contributing.md b/docs/contributing.md index 756ed2e..44edc01 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -33,6 +33,7 @@ rebar3 ct --suite test/py_isolated_SUITE # one suite rebar3 ct --suite test/py_context_SUITE --case test_call # one case rebar3 dialyzer && rebar3 xref # required before a PR make lint-docs # snippets in README and docs/ +make check-code-map # every file in the code map ``` Notes: @@ -182,7 +183,8 @@ supervisor to carry it. 4. Skip, do not fail, when a platform or interpreter cannot run a case: `{skip, Reason}` with the reason a human can act on. 5. Add the suite to the table in `docs/code-map.md` and the cases that - cover a documented API to `test/coverage_audit.md`. + cover a documented API to `test/coverage_audit.md`; a new module also + needs a row in its Modules table. `make check-code-map` verifies both. 6. Cases that measure time or memory print their numbers with `ct:pal` and assert only on invariants, never on absolute timings. diff --git a/scripts/check_code_map.sh b/scripts/check_code_map.sh new file mode 100755 index 0000000..14e44be --- /dev/null +++ b/scripts/check_code_map.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Every source file must be listed in docs/code-map.md, every Erlang module +# must have a moduledoc, and every Erlang module must have a row in the +# "Modules" table of test/coverage_audit.md. Exit code is the number of +# failures. Run: sh scripts/check_code_map.sh +cd "$(dirname "$0")/.." || exit 2 +fail=0 +note() { echo "check_code_map: $1"; fail=$((fail + 1)); } + +for f in src/*.erl; do + m=$(basename "$f" .erl) + grep -q "\`$m\`" docs/code-map.md || note "$m is not in docs/code-map.md" + grep -q '^%%%\{0,1\} @doc' "$f" || note "$f has no @doc moduledoc" + grep -q "^| \`$m\` |" test/coverage_audit.md || note "$m has no row in the Modules table of test/coverage_audit.md" +done +for f in c_src/*.c c_src/*.h priv/_erlang_impl/*.py priv/*.py; do + b=$(basename "$f") + grep -q "$b" docs/code-map.md || note "$b is not in docs/code-map.md" +done +[ "$fail" -eq 0 ] && echo "check_code_map: clean" +exit $fail diff --git a/test/coverage_audit.md b/test/coverage_audit.md index 2502366..bb8a8ca 100644 --- a/test/coverage_audit.md +++ b/test/coverage_audit.md @@ -5,6 +5,46 @@ README and `docs/*.md` to at least one `*_SUITE.erl` test that exercises it. Update this table whenever a documented API is added, renamed, or removed. +## Modules + +Every Erlang module and the suites that exercise it, so a module without a +suite is visible. `scripts/check_code_map.sh` requires a row per module. + +| Module | Suites | +|---|---| +| `py` | `py_SUITE`, `py_api_SUITE`, `py_stream_SUITE`, `py_venv_SUITE` | +| `py_context` | `py_context_SUITE`, `py_context_process_SUITE`, `py_interrupt_SUITE`, `py_worker_loop_SUITE` | +| `py_context_embedded` | `py_context_SUITE`, `py_context_process_SUITE`, `py_interrupt_SUITE`, `py_worker_loop_SUITE` | +| `py_stream` | `py_stream_SUITE` | +| `py_venv` | `py_venv_SUITE` | +| `py_shared_dict` | `py_SUITE` | +| `py_isolated` | `py_isolated_*_SUITE` | +| `py_context_router` | `py_context_router_SUITE`, `py_pool_SUITE` | +| `py_context_sup` | (through the above) | +| `py_context_init` | (through the above) | +| `py_nif` | all | +| `py_callback` | `py_callback_encoding_SUITE`, `py_thread_callback_SUITE` | +| `py_thread_handler` | `py_thread_callback_SUITE`, `py_reentrant_SUITE` | +| `py_event_loop` | `py_event_loop_SUITE`, `py_async_task_SUITE` | +| `py_event_loop_pool` | `py_event_loop_pool_SUITE` | +| `py_event_worker` | `py_event_loop_SUITE`, `py_fd_ops_SUITE` | +| `py_event_worker_sup` | `py_event_loop_SUITE` | +| `py_event_worker_registry` | `py_event_loop_SUITE` | +| `py_reactor_context` | `py_reactor_SUITE` | +| `py_channel` | `py_channel_SUITE`, `py_byte_channel_SUITE` | +| `py_byte_channel` | `py_channel_SUITE`, `py_byte_channel_SUITE` | +| `py_buffer` | `py_buffer_SUITE`, `py_isolated_buffer_SUITE` | +| `py_shm` | `py_isolated_shm_SUITE` | +| `py_import` | `py_import_SUITE` | +| `py_preload` | `py_preload_SUITE` | +| `py_state` | `py_state_SUITE` | +| `py_semaphore` | (through `py_SUITE`) | +| `py_logger` | `py_logging_SUITE` | +| `py_util` | (through every suite) | +| `py_tracer` | `py_logging_SUITE` | +| `erlang_python_app` | all | +| `erlang_python_sup` | all | + ## Erlang public API (`src/py.erl` exports) | API | Documented in | Test suite | Test case | From 58f0d116ba9d83b73fc7f0812259f7a6c5e05af4 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Tue, 1 Sep 2026 08:08:17 +0200 Subject: [PATCH 14/15] Grant an isolated child what it may reach (#84) Isolated mode bounded what Python could consume but nothing it could reach: the child held every authority the node's user held. The caps option names its directories, environment and addresses, and refuses the rest. The model and its vocabulary come from erlang_wasm's WASI implementation, so a grant means the same thing in both. It is a cooperative policy over Python, not a boundary: built on an audit hook, it binds Python and not a C extension, and covers only what CPython announces. The guide says what holds and what does not, and names the kernel work that would change the answer. --- CHANGELOG.md | 25 + README.md | 11 +- docs/capabilities.md | 238 ++++++++ docs/code-map.md | 2 + docs/decisions/0009-child-capabilities.md | 92 ++++ docs/decisions/overview.md | 1 + docs/isolated.md | 12 +- docs/security.md | 19 +- priv/_erlang_impl/_caps.py | 571 ++++++++++++++++++++ priv/_erlang_impl/_isolated.py | 11 +- priv/py_isolated_child.py | 31 +- priv/tests/test_caps.py | 253 +++++++++ rebar.config | 6 +- src/erlang_python.app.src | 2 +- src/py_caps.erl | 263 +++++++++ src/py_context.erl | 27 +- src/py_isolated.erl | 64 ++- test/coverage_audit.md | 1 + test/py_isolated_caps_SUITE.erl | 628 ++++++++++++++++++++++ test/py_test_caps.py | 171 ++++++ 20 files changed, 2411 insertions(+), 17 deletions(-) create mode 100644 docs/capabilities.md create mode 100644 docs/decisions/0009-child-capabilities.md create mode 100644 priv/_erlang_impl/_caps.py create mode 100644 priv/tests/test_caps.py create mode 100644 src/py_caps.erl create mode 100644 test/py_isolated_caps_SUITE.erl create mode 100644 test/py_test_caps.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1514ebf..d70309d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## 5.1.0 (unreleased) + +### Added + +- **Capabilities for isolated children** - `py_context:new(#{mode => isolated, + caps => ...})` names the directories, environment variables and network + addresses a child may reach; anything not named is refused. Leaving a key + out grants none of it, and omitting `caps` leaves existing behaviour + unchanged. Paths are resolved a component at a time with `openat` and + `O_NOFOLLOW` from the descriptor of the grant, so `..`, absolute paths, + symlinks out of a grant and symlinked directory prefixes are all refused, + and refusals are `PermissionError` rather than `FileNotFoundError` so they + disclose nothing about what exists outside. Network rules name addresses + and never host names, resolution is its own capability covering every + resolver, and binding is checked against `listen` rather than `connect`. + Process creation, `ctypes`, signals to another process and Unix-socket + addresses are refused outright. The model and its vocabulary come from + erlang_wasm's WASI implementation. + + This is a cooperative policy over Python and not a boundary: it is built + on a CPython audit hook, so it binds Python and not a C extension, and it + covers only what CPython announces. `docs/capabilities.md` says what holds + and what does not. Shared memory and capability sets do not combine yet, + because a region reaches the child as a path. + ## 5.0.0 (2026-08-29) ### Added diff --git a/README.md b/README.md index ca06e00..277e715 100644 --- a/README.md +++ b/README.md @@ -622,14 +622,21 @@ When creating Python contexts, you can choose the execution mode: %% segfault only takes the child down, rlimits bound memory and CPU. {ok, Ctx} = py_context:new(#{mode => isolated, kill_after => 1000, rlimits => #{as => 512 * 1024 * 1024}}). + +%% Name what the child may reach, and it reaches nothing else. +{ok, Ctx} = py_context:new(#{mode => isolated, + caps => #{dirs => [{"/srv/models", read}], + net => #{connect => [{tcp, <<"10.0.0.0/8">>, 5432}]}}}). ``` **Isolated mode** is the only mode with a hard bound: `py_context:interrupt/1` stops a blocking C call, and `SIGKILL` is the backstop. It costs a process per context (about 16 MB and 40 ms to start) and roughly twice the call latency. Bulk data crosses through shared memory (`py_shm`, with the optional -[iommap](https://hex.pm/packages/iommap) dependency). See -[Isolated Contexts](docs/isolated.md). +[iommap](https://hex.pm/packages/iommap) dependency), and the `caps` option +names the files, addresses and environment the child may reach, as a +cooperative policy over Python rather than a kernel boundary. See +[Isolated Contexts](docs/isolated.md) and [Capabilities](docs/capabilities.md). **Worker mode is recommended** because it works with any Python version and automatically benefits from free-threaded Python (3.13t+) when available. Each context owns a dedicated pthread, providing stable thread affinity for libraries with thread-local state (numpy, torch, tensorflow). diff --git a/docs/capabilities.md b/docs/capabilities.md new file mode 100644 index 0000000..c1bb0a9 --- /dev/null +++ b/docs/capabilities.md @@ -0,0 +1,238 @@ +# Capabilities + +This guide covers `caps`, the option that says what an isolated child may +reach: which directories, which environment variables, which addresses. +Python that asks for anything else is refused. It is the WASI model, and +the vocabulary is the same as +[erlang_wasm](https://github.com/benoitc/erlang_wasm)'s, so a grant means +the same thing in both. + +**Read this before you rely on it.** `caps` is a cooperative policy over +Python, not a boundary. It stops code that is not trying to get out, and it +makes what a job may touch explicit and reviewable. It does not stop code +that is trying to get out: a C extension calling `open(2)` never reaches +the audit hook it is built on. Use it for code you partly trust; use the +process boundary in [Isolated Contexts](isolated.md) for the rest, and see +[what holds and what does not](#what-holds-and-what-does-not) for the +detail. + +Read [Isolated Contexts](isolated.md) first: `caps` only applies there. + +## Grant what it needs + +```erlang +{ok, Ctx} = py_context:new(#{ + mode => isolated, + caps => #{ + dirs => [{"/srv/models", read}, + {"/var/data/job42", write}], + env => #{<<"MODEL_DIR">> => <<"/srv/models">>}, + net => #{connect => [{tcp, <<"10.0.0.0/8">>, {5432, 5432}}], + resolve => deny} + }}), +{ok, _} = py_context:call(Ctx, scorer, run, [<<"job42">>]). +``` + +Inside the child, ordinary Python works inside the grants and fails outside +them: + +```python +open('/srv/models/weights.bin', 'rb') # granted +open('/var/data/job42/out.csv', 'w') # granted +open('/etc/passwd') # PermissionError +open('/srv/models/w', 'w') # PermissionError: read grant +``` + +Leave a key out and there is none of it. `caps => #{}` grants nothing but the +interpreter's own files, and no `caps` key at all is the behaviour you have +today: the child holds every authority the user running the node holds. + +## What each key grants + +| key | grants | leaving it out means | +| --- | --- | --- | +| `dirs` | directories, `read` or `write` | no filesystem beyond the interpreter's own | +| `env` | environment variables | zero variables, **not** the node's | +| `net` | sockets, to the addresses you name | no network at all | + +`read` covers opening, reading and listing. `write` adds creating, renaming, +unlinking and truncating. Rights apply to everything below the directory, so +a `read` grant yields no writable file however the code opens it. + +Always granted, because nothing works otherwise: the interpreter's own +`sys.path`, `sys.prefix` and `sys.base_prefix` for reading, so imports work, +and `/dev/null`, `/dev/zero`, `/dev/random`, `/dev/urandom`. WASI preopens +its sysroot for the same reason. Directories you list in the `paths` option +are granted for reading too, since that option tells the child to import +from them. + +The `env` option and `caps` cannot be used together. The option adds +variables and a grant says what the whole environment is, so taking both +would let the option quietly win; `caps.env` is the one that names an +environment, and using both is `{error, {bad_caps, +env_option_conflicts_with_caps_env}}`. + +## Name a network + +```erlang +net => #{connect => [{tcp, <<"10.0.0.0/8">>, {8000, 8099}}], + listen => [{tcp, <<"127.0.0.1">>, 8080}], + resolve => allow} +``` + +A rule is `{Proto, Addr, Port}`. `Proto` is `tcp` or `udp`, `Addr` is an +address tuple, a binary address or a binary CIDR, and `Port` is an integer, a +`{Lo, Hi}` range, or `any`. + +Four things to know before writing your first grant: + +- **`connect` and `listen` are separate**, and neither implies the other. + Binding claims a local address, which is what `listen` grants, so code + wanting a particular source port needs a `listen` rule for it. +- **You name addresses, never names.** There is no rule that says + `example.com`: a name would have to be resolved to be checked and resolved + again to be used, and the two answers can differ. `resolve` is its own + capability, off unless granted, and what it returns carries no authority. + Code may learn an address it cannot reach, and the connect is refused then. +- **`::ffff:127.0.0.1` is `127.0.0.1`.** IPv4-mapped addresses are folded + before matching, so the mapped notation cannot walk past an IPv4 rule. +- **Nothing is denied implicitly.** `<<"0.0.0.0/0">>` really does include + link-local and cloud metadata addresses. Name what you mean. + +A socket Erlang opened and handed over with `py_context:pass_fd/2` needs no +rule: the child was given the descriptor, and the descriptor is the +capability. That is how you serve on a port under a capability set. + +```erlang +{ok, LSock} = gen_tcp:listen(8080, [binary, {active, false}]), +{ok, Fd} = inet:getfd(LSock), +{ok, ChildFd} = py_context:pass_fd(Ctx, Fd), +ok = py_context:start_loop(Ctx), +{ok, _} = py_context:submit_await(Ctx, myapp, serve, [ChildFd]). +``` + +## Shared memory does not combine with this yet + +A `py_shm` region reaches the child as a path, so under a capability set it +is refused like any other ungranted path. Granting it would mean granting +the directory the node keeps every region in, which hands over every +region, and an open grant cannot prevent truncation anyway because +`file.truncate()` announces nothing. A truncated region is a `SIGBUS` in +the VM that mapped it, so the half-measure is worse than the refusal. + +The fix is to pass the region's descriptor rather than its name, with +`memfd_create` and `F_SEAL_SHRINK | F_SEAL_GROW | F_SEAL_SEAL` on Linux, +which keeps a writable mapping while making the object unresizable, and +cooperative-only handling where sealing does not exist. Until that lands, +use shared memory or a capability set, not both. + +## What is refused + +Every route below has its own test case in `py_isolated_caps_SUITE`. All are +refused with a `PermissionError` and never a `FileNotFoundError`, so the +error cannot be used to find out what exists outside a grant. + +| the code asks for | it gets | +| --- | --- | +| `note.txt`, `./note.txt`, `sub/deep.txt` | opened | +| `sub/../note.txt` | opened: it never leaves the grant | +| `../secret/key.txt` | refused | +| `/etc/passwd` | refused | +| `escape.txt`, a symlink out of the grant | refused | +| `outdir/key.txt`, through a symlinked directory | refused | +| `sub/../../secret/key.txt` | refused | +| a symlink cycle | refused | +| `missing.txt` inside a grant | `FileNotFoundError` | +| `subprocess.run`, `os.fork`, `os.exec*` | refused | +| `ctypes.CDLL` | refused | +| `os.kill` at another process, `os.killpg` | refused | +| `socket.gethostbyname` and every other resolver | refused unless `resolve` | +| `os.mkfifo`, `os.mknod` | not there: see below | +| a Unix-socket address, connect or bind | refused | + +Signalling is refused because the child usually shares the node's user and +its parent is the BEAM, so an unchecked `os.kill` is a way to take the node +down. Signalling itself is allowed. Every resolver is gated, not only +`getaddrinfo`: a name lookup is a message to whoever answers it, so gating +one of them would leave the rest as a way out. A Unix-socket address is +refused rather than checked as a file, because reaching one is talking to +whatever is behind it, which a directory grant says nothing about; a +descriptor Erlang passed over with `py_context:pass_fd/2` is unaffected. + +Subprocesses are refused outright: a capability set names what may be +reached, and another process is not something you granted. `ctypes` is +refused because it reaches libc directly, which would make every rule above +advisory. A library that needs `ctypes` cannot run under a capability set. + +## What holds and what does not + +Enforcement is a CPython audit hook, so the whole of what it can see is +what CPython announces, and everything below follows from that. + +**What holds.** Python code that asks for a path, an address or a process +outside the grants is refused, whether it asks through `open`, `os.open`, +`pathlib`, numpy or any other library, because the event is raised by the +interpreter rather than by the caller. Path containment is resolved by the +kernel one component at a time, so `..`, absolute paths, symlinks out of a +grant and symlinked directory prefixes are all refused rather than +lexically guessed at. Nothing on the decision path is reachable by name: +the grants, the tables and even the `os` functions the check uses are bound +into the hook's closure when it is installed, so assigning to this module +changes nothing. + +**What does not hold.** + +- A C extension calling `open(2)` or `connect(2)` never reaches an audit + hook. Neither does `file.truncate()` or `mmap.resize()`, which CPython + does not announce, so a writable descriptor can always shorten its own + file. That is part of what a `write` grant grants. +- A thread that replaces a path between the check and the kernel's own + resolution is not stopped. Do not point a `write` grant at a directory + another party writes to concurrently. +- `os.stat`, `os.access` and the rest of the calls that observe without + reaching are left alone, so what exists outside a grant stays visible + even though reading it does not. +- Closure state is a bar, not a wall. Python exposes its own object graph, + and code that goes looking can reach a hook's cells. + +**What would make it hold.** A kernel. Landlock on Linux takes the same +grant table and enforces it below the interpreter, which is the point at +which a C extension stops being an exception; moving the state and the hook +into the NIF would take the rest. Neither is here yet. Until then, the +boundary you have is the process: `rlimits`, `kill_after`, and the +supervision in [Isolated Contexts](isolated.md). + +## Cost + +The check is an audit hook, so it runs on every open, and paths are resolved +one component at a time against the descriptor of the grant. Measured on +macOS with Python 3.14, an open inside a grant costs about 11 microseconds +more than an unguarded one, and the cost grows with the depth of the path +below its grant. Grant close to what the code reads: `/srv/models` rather +than `/`. + +Nothing else changes. Calls, results, shared memory and interrupts are what +they were. + +## Check what a child got + +```erlang +{ok, Info} = py_context:child_info(Ctx), +maps:get(caps, Info). +``` + +```python +import erlang +erlang.caps() # None when no capability set was given +``` + +Both report the grants as the child holds them, including the automatic +ones, and `strict_paths` tells you whether the platform resolved paths +component by component or fell back to a lexical check. + +## See also + +- [Isolated Contexts](isolated.md) for the process boundary itself +- [Security](security.md) for what the embedded modes do instead +- [decision 0009](decisions/0009-child-capabilities.md) for why it is shaped + this way diff --git a/docs/code-map.md b/docs/code-map.md index 9da9c9b..fefea58 100644 --- a/docs/code-map.md +++ b/docs/code-map.md @@ -29,6 +29,7 @@ exercised by suites). Guides are in `docs/`, suites in `test/`. Start with | `py_channel`, `py_byte_channel` | Term and byte queues between Erlang and Python coroutines (NIF resources) | live | channel | `py_channel_SUITE`, `py_byte_channel_SUITE` | | `py_buffer` | Native streaming input buffer; shared variant delegates to `py_shm` | live | buffer, isolated | `py_buffer_SUITE`, `py_isolated_buffer_SUITE` | | `py_shm` | Shared memory regions over iommap and the ring behind shared buffers | live | isolated | `py_isolated_shm_SUITE` | +| `py_caps` | The `caps' option: what an isolated child may reach, and its wire form | live | capabilities | `py_isolated_caps_SUITE` | | `py_import` | Registry of imports and `sys.path` entries applied to every interpreter | live | imports | `py_import_SUITE` | | `py_preload` | Code run once per interpreter at start | live | preload | `py_preload_SUITE` | | `py_state` | Shared key/value store visible from Python as `erlang.state_get/set/delete/keys` | live | README (shared state) | `py_state_SUITE` | @@ -80,6 +81,7 @@ loop, channels and servers. | `_erlang_impl/_etf.py` | Pure-Python ETF codec with the `py_convert.c` mapping | isolated child | | `_erlang_impl/_isolated.py` | Child runtime: socket frames, reader thread, re-entrant main loop, interrupt signal, asyncio loop, the `erlang` shim | isolated child | | `_erlang_impl/_shm.py` | `SharedMemory` and `SharedBuffer` wrappers over mmap | all | +| `_erlang_impl/_caps.py` | Capability enforcement in the child: path containment, address matching, the audit hook | isolated child | | `py_isolated_child.py` | Child launcher: rlimits, parent-death signal, cgroup join, connect | isolated child | | `test_erlang_loop.py`, `test_async_task.py`, `test_channel_ref.py`, `tests/` | Python-side tests of the loop, tasks and channels | test | diff --git a/docs/decisions/0009-child-capabilities.md b/docs/decisions/0009-child-capabilities.md new file mode 100644 index 0000000..682e024 --- /dev/null +++ b/docs/decisions/0009-child-capabilities.md @@ -0,0 +1,92 @@ +# 0009: An isolated child reaches only what it was granted + +Since 5.1.0. Code: `src/py_caps.erl`, `priv/_erlang_impl/_caps.py`, the +prologue of `priv/py_isolated_child.py`. + +## Situation + +Isolated mode bounded what Python could *consume*: memory, CPU, time, and a +crash. It bounded nothing it could *reach*. The child ran as the node's user +with the node's environment, could read and write every file that user +could, dial anywhere, spawn processes, and truncate the shared memory +regions it was handed, which turns the mapping the VM holds into a `SIGBUS`. +The audit hook the embedded modes install was never installed there. + +The first sketch was a deny list of dangerous operations. That is the wrong +shape: it enumerates what to stop, so it is wrong the moment something new +appears, and it says nothing about what a job is supposed to touch. + +## Decision + +Grant capabilities instead. `py_context:new(#{mode => isolated, caps => ...})` +names directories with an access level, environment variables, and network +rules; nothing else is reachable. No `caps` key leaves existing behaviour +alone, so this is additive. + +The model, the option shape, the refusal semantics and the test list are +taken from `erlang_wasm`'s WASI preview 1 implementation rather than +invented: `{Proto, Addr, Port}` rules, addresses and never host names, +resolution as its own capability, binding checked against `listen` and not +`connect`, IPv4-mapped folding, a malformed rule raising where the grant is +written. A grant means the same thing in both projects. + +Enforcement is an audit hook in the child, installed last in the prologue so +the runtime's own imports are outside the grants and everything after them +is inside. Paths are resolved a component at a time with `openat` and +`O_NOFOLLOW` from the descriptor of the grant, following symlinks by hand: +that is erlang_wasm's `native` backend, which needed a C NIF there because +Erlang has no `openat`, and needs none here because Python has one. + +Not chosen: routing every open through Erlang so `wasi_fs` could enforce it +directly. It would share one implementation and close the check-to-use +window, but it makes an `open` a socket round trip from arbitrary code, +including the child's own reader thread, and the deadlock surface is not +worth it at 25 microseconds a call. + +## What this is not + +Three review rounds all found the same shape of defect: a way for Python to +step around a check written in Python. Each was real and each was closed, +but the pattern is the point. An audit hook is a cooperative policy, and +the honest split is: + +* `caps` is for code you partly trust. It stops mistakes and casual misuse, + and it makes what a job may touch reviewable. +* The process, its rlimits and `kill_after` are what hold against code that + is trying to get out. +* Kernel enforcement (Landlock, seccomp, or the state and hook moved into + the NIF) is the point at which `caps` may be described as protection + against adversarial code. It is not there yet, and the guide says so + rather than implying otherwise. + +## Consequences + +- It is a policy over Python, not a boundary. A C extension calling + `open(2)`, or a thread swapping a path between the check and the kernel's + resolution, is not stopped. The guide says so in those words. +- The grants have to live in the hook's closure rather than in module + state, and nothing inside the interpreter may widen them. An earlier + version kept them in a module attribute and let `_shm` add region paths: + both were levers any Python could pull. Regions are now granted from + Erlang, as the directory holding them, opened and nothing more. +- Enforcement can only cover what CPython announces. Calls that create + something and raise no audit event are removed from `os` and `posix` + instead, and the ones that only observe are documented as visible. +- Nothing on the decision path may be reachable by name, including the + re-entrancy guard, the event tables and the `os` functions the check + itself calls. The guard is set around the containment walk alone, since a + wider one would leave a user `__fspath__` running with enforcement off. +- Shared memory does not combine with a capability set. A region arrives as + a path, granting the directory would hand over every region the node + owns, and an open-only grant cannot stop truncation because + `file.truncate()` announces nothing. Passing the descriptor, sealed on + Linux, is the way to make it work. +- `ctypes` must be refused, or every rule is advisory. Libraries that need + it cannot run under a capability set. +- The interpreter's own `sys.path` is granted automatically, or nothing + imports. A capability set therefore always grants reading the standard + library, as WASI's preopened sysroot does. +- An open inside a grant costs about 11 microseconds more, growing with path + depth, so grants should sit close to what is read. +- Landlock on Linux consumes the same table and would make it a boundary. + The table is shaped for that. diff --git a/docs/decisions/overview.md b/docs/decisions/overview.md index 0d1f7eb..9cc5e18 100644 --- a/docs/decisions/overview.md +++ b/docs/decisions/overview.md @@ -16,3 +16,4 @@ what was decided, what it costs, and where the code is. | [0006](0006-shared-memory-over-iommap.md) | Bulk data through iommap regions, handles as plain tuples | 5.0.0 | | [0007](0007-remove-legacy-execution-paths.md) | One execution path per mode; the legacy API is removed | 5.0.0 | | [0008](0008-pipe-io-rules.md) | Pipe I/O is non-blocking, deadlined and waited with poll | 3.1.0, 5.0.0 | +| [0009](0009-child-capabilities.md) | An isolated child reaches only what it was granted | 5.1.0 | diff --git a/docs/isolated.md b/docs/isolated.md index 817fd77..87ac086 100644 --- a/docs/isolated.md +++ b/docs/isolated.md @@ -312,11 +312,19 @@ file on purpose; sealing and syscall filtering are separate hardening work. event worker to step an idle loop). - `erlang.call` from inside a coroutine blocks the loop, as in the embedded modes; use `erlang.async_call`. +- Without `caps` the child holds every authority the user running the node + holds: it reads and writes what that user can, dials anywhere and can + spawn processes. - The child decodes terms with the same rules as the NIF, so atoms sent from Python are created in the VM's atom table. Do not let untrusted code mint unbounded distinct atoms. -- No syscall filtering: process isolation plus rlimits is the boundary. A - seccomp (Linux) or Capsicum (FreeBSD) sandbox is a separate hardening step. +- No syscall filtering: process isolation plus rlimits is the boundary. What + the child may *reach* (files, addresses, environment) is named with the + `caps` option, which is a cooperative policy over Python rather than a + kernel boundary: see [capabilities](capabilities.md). +- Shared memory and `caps` do not combine: a region reaches the child as a + path, and granting it would grant every region the node owns. A seccomp (Linux) or Capsicum (FreeBSD) + sandbox is a separate hardening step. - Each call copies its arguments and result through the socket: a 1 MB binary round-trips in about 1.3 ms, 16 MB in about 27 ms (worker mode: 0.2 ms and 3 ms). For bulk data use shared memory (below). diff --git a/docs/security.md b/docs/security.md index db402eb..640c77f 100644 --- a/docs/security.md +++ b/docs/security.md @@ -158,8 +158,23 @@ child process: ``` A crash kills only the child, `py_context:kill/1` is total, and rlimits or -cgroups bound resources. See [Isolated Contexts](isolated.md). The child is -not sandboxed at the syscall level; that is a separate hardening step. +cgroups bound resources. See [Isolated Contexts](isolated.md). + +That bounds what Python may consume. What it may *reach* is named with the +`caps` option: + +```erlang +{ok, Ctx} = py_context:new(#{mode => isolated, + caps => #{dirs => [{"/srv/models", read}], + net => #{connect => [{tcp, <<"10.0.0.0/8">>, 5432}]}}}). +``` + +Anything not named is refused. That is a cooperative policy over Python: it +binds Python code, not a C extension, because it is built on an audit hook. +[Capabilities](capabilities.md) says what holds and what does not. The child +is not sandboxed at the syscall level; that is a separate hardening step, +and the point at which capability sets would bind an adversary rather than +a mistake. ## Signal Handling Note diff --git a/priv/_erlang_impl/_caps.py b/priv/_erlang_impl/_caps.py new file mode 100644 index 0000000..9032fe2 --- /dev/null +++ b/priv/_erlang_impl/_caps.py @@ -0,0 +1,571 @@ +# Copyright 2026 Benoit Chesneau +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""What an isolated child may reach, enforced over Python. + +Erlang names the directories, environment variables and addresses this +process may reach (`py_caps.erl`); this module refuses the rest. It is +installed once, in the child's prologue, before any user code runs. + +**This is a cooperative policy, not a boundary.** The difference decides +what you may use it for: + +* It stops code that is not trying to get out. Reading the wrong dataset, + writing outside a job's directory, calling home: those stop, and what a + job may touch becomes something you can read off the context options. +* It does not stop code that is trying to get out. A C extension calling + `open(2)` never reaches an audit hook. Neither does `file.truncate()`, + which CPython does not announce. And the grants live in a closure rather + than in a module attribute, which removes the one-line way to switch them + off but not the last one: Python exposes its own object graph, so code + that goes looking can reach a hook's cells. + +Hard isolation is a kernel's job and is not here yet. Landlock on Linux +takes the same grant table; moving the state and the hook into the NIF +would take the rest. Until then, treat `caps` as a policy for partially +trusted code, and the process boundary in `docs/isolated.md` as the thing +that holds against the rest. + +The audit surface, which is the whole of what is enforced: + +* **Refused by rule**: `open`, `os.listdir`, `os.scandir`, `os.mkdir`, + `os.rmdir`, `os.remove`, `os.rename`, `os.link`, `os.symlink`, + `os.truncate`, `os.chmod`, `os.chown`, `os.utime`, the `shutil.*` events, + `socket.connect`, `socket.bind`, `socket.sendto`, and every resolver. +* **Refused outright**: process creation, `ctypes`, signals to anything but + this process, and Unix-socket addresses. +* **Ignored, deliberately**: `os.stat`, `os.access`, `os.statvfs`, + `os.chdir` and the other calls that observe without reaching. What exists + outside a grant stays visible; reading it does not. +* **Removed, because CPython announces nothing**: `os.mkfifo` and + `os.mknod`, which create. `file.truncate()` and `mmap.resize()` announce + nothing either and cannot be removed, so a writable descriptor can always + shorten its own file. That is part of what a `write` grant grants, and it + is why there is no grant that means "open but do not resize". + +Path containment is erlang_wasm's native backend (`c_src/wasi_file_nif.c`), +which needs no C here because Python has `openat`: walk a component at a +time with ``O_NOFOLLOW`` from the descriptor of the grant, follow a symlink +by hand so it is worth what the same text written out is worth, and count +depth so ``..`` moves inside a grant but not out of it. + +Refusals are `PermissionError`, never `FileNotFoundError`, so a refusal +says nothing about what exists outside a grant. +""" + +import errno +import ipaddress +import os +import socket +import sys +import threading + +__all__ = ['install', 'installed', 'grants', 'CapabilityError'] + +# Eight, as Linux allows per path. It has to be a constant a cycle cannot +# outrun; a self-referential link is otherwise not an error but a hang. +_MAX_SYMLINKS = 8 + +_READ, _WRITE = 'read', 'write' + +# Devices that carry nothing about the host and whose absence breaks code in +# ways that are hard to read. Granted for reading with any capability set. +_ALWAYS_READ = ('/dev/null', '/dev/urandom', '/dev/random', '/dev/zero') + +# Process creation, always refused: a capability set names what may be +# reached, and another process is not something it was granted. +_SUBPROCESS_EVENTS = frozenset({ + 'subprocess.Popen', 'os.system', 'os.popen', 'os.fork', 'os.forkpty', + 'os.posix_spawn', 'os.posix_spawnp', +}) +_EXEC_PREFIXES = ('os.exec', 'os.spawn') + +# Signalling, refused except towards this process. The child usually shares +# the node's user and its parent is the BEAM, so an unchecked os.kill is a +# way to take the node down. +_SIGNAL_EVENTS = frozenset({'os.kill', 'os.killpg'}) + +# Resolution, granted by `resolve`. Every one of these reaches a resolver, +# so gating only getaddrinfo would leave the rest as a way out. +_RESOLVE_EVENTS = frozenset({ + 'socket.getaddrinfo', 'socket.gethostbyname', 'socket.gethostbyaddr', + 'socket.getnameinfo', 'socket.getservbyname', 'socket.gethostname', +}) + +# ctypes reaches libc directly, so leaving it open would make every rule +# here advisory. A library that needs it cannot run under a capability set. +_CTYPES_PREFIX = 'ctypes.' + +# Calls that create something and announce nothing, so they are taken away +# rather than refused. +_UNAUDITED_CREATORS = ('mkfifo', 'mknod') + +# Audit events that name a path, and what they need for it. +_PATH_EVENTS = { + 'os.listdir': _READ, + 'os.scandir': _READ, + 'os.mkdir': _WRITE, + 'os.rmdir': _WRITE, + 'os.remove': _WRITE, + 'os.rename': _WRITE, + 'os.link': _WRITE, + 'os.symlink': _WRITE, + 'os.truncate': _WRITE, + 'os.chmod': _WRITE, + 'os.chown': _WRITE, + 'os.utime': _WRITE, + 'shutil.copyfile': _WRITE, + 'shutil.copymode': _WRITE, + 'shutil.copystat': _WRITE, + 'shutil.copytree': _WRITE, + 'shutil.move': _WRITE, + 'shutil.rmtree': _WRITE, + 'shutil.unpack_archive': _WRITE, +} + +# Operations that act on a name and not on what it points at, so the last +# component is not followed: removing a symlink that leads out of a grant +# removes something inside the grant. +_NAME_EVENTS = frozenset({ + 'os.remove', 'os.rename', 'os.symlink', 'os.link', 'os.rmdir', 'os.mkdir', +}) + +# Events that name two paths; both ends are checked. +_TWO_PATH_EVENTS = frozenset({ + 'os.rename', 'os.link', 'os.symlink', 'shutil.copyfile', 'shutil.copymode', + 'shutil.copystat', 'shutil.copytree', 'shutil.move', +}) + +# A summary of what was granted, for `grants()`. It gates nothing: the +# grants themselves are reachable only from the hook's closure. +_summary = None + + +class CapabilityError(PermissionError): + """Raised for anything a capability set does not grant. + + A `PermissionError`, so code that already handles one keeps working, and + never a `FileNotFoundError`: whether a path outside a grant exists is + not something a refusal should disclose. + """ + + +class _Grant: + """One granted directory, held open. + + Opened once and kept: naming the directory by path on every check would + leave it to be resolved again each time, so replacing it would move the + grant. Anchored to the descriptor, a swapped *child* is what gets + refused. + + Both the path as granted and its resolved form are prefixes, because a + grant is often reached through a symlink (`/tmp` is `/private/tmp` on + macOS) and code inside the child will name it either way. + """ + + __slots__ = ('path', 'access', 'fd', 'prefixes') + + def __init__(self, path, access): + self.path = path + self.access = access + self.fd = os.open(path, os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0)) + real = os.path.realpath(path) + self.prefixes = (path,) if real == path else (path, real) + + def writable(self): + return self.access == _WRITE + + def remainder(self, path): + """The part of `path` below this grant, or None if it is not under it.""" + for prefix in self.prefixes: + if path == prefix: + return '' + if path.startswith(prefix.rstrip('/') + '/'): + return path[len(prefix.rstrip('/')) + 1:] + return None + + +class _State: + __slots__ = ('dirs', 'files', 'net', 'lexical') + + def __init__(self): + self.dirs = [] + # Exact paths that may be opened for reading: the devices above. + self.files = {} + self.net = None + self.lexical = False + + +class _Enforcer: + """What `_make_enforcer` returns: the hook, and its parts for tests.""" + + __slots__ = ('hook', 'walk', 'contained', 'check_path', 'writes') + + def __init__(self, **parts): + for name, part in parts.items(): + setattr(self, name, part) + + +def _make_enforcer(st): + """Build the audit hook over `st`. + + Everything on the decision path is bound here rather than looked up when + the hook runs, because a name resolved at call time is a name any Python + in this process can rebind: `_caps._writes = lambda *_: False` would + otherwise turn every open into a read. That includes `os` itself, so the + primitives are bound one by one, and the walk lives here rather than at + module level so no shared function object is left behind whose defaults + could be rewritten. + """ + # syscalls and constants, bound once + _open, _close, _readlink = os.open, os.close, os.readlink + _getcwd, _getpid = os.getcwd, os.getpid + _fspath, _fsdecode = os.fspath, os.fsdecode + _normpath, _abspath = os.path.normpath, os.path.abspath + _O_RDONLY, _O_NOFOLLOW = os.O_RDONLY, os.O_NOFOLLOW + _O_DIRECTORY = getattr(os, 'O_DIRECTORY', 0) + _O_WRITES = (os.O_WRONLY | os.O_RDWR | os.O_CREAT | os.O_TRUNC + | os.O_APPEND) + # A symlink met with O_NOFOLLOW: ELOOP on Linux and macOS, EMLINK on + # FreeBSD. + _ELOOP, _EMLINK = errno.ELOOP, errno.EMLINK + _ip_address = ipaddress.ip_address + _SOCK_DGRAM = socket.SOCK_DGRAM + _error = CapabilityError + READ, WRITE = _READ, _WRITE + + # the tables, copied so rebinding a module attribute changes nothing + path_events = dict(_PATH_EVENTS) + name_events = frozenset(_NAME_EVENTS) + two_path_events = frozenset(_TWO_PATH_EVENTS) + subprocess_events = frozenset(_SUBPROCESS_EVENTS) + exec_prefixes = tuple(_EXEC_PREFIXES) + ctypes_prefix = _CTYPES_PREFIX + signal_events = frozenset(_SIGNAL_EVENTS) + resolve_events = frozenset(_RESOLVE_EVENTS) + dirs, files, net, lexical = st.dirs, st.files, st.net, st.lexical + max_links = _MAX_SYMLINKS + + # The re-entrancy guard, created here so there is no module attribute to + # assign to. It is set around the walk and nothing else: the walk calls + # only `open`, `close` and `readlink` on its own account, so no user + # code can run while enforcement is off. + busy = threading.local() + + def walk(grant, rel, follow_last): + """Resolve `rel` beneath `grant`, a component at a time. + + Returns `(dirfd, component, owned)`; the caller closes `dirfd` when + `owned`. Raises `CapabilityError` if the path leaves the grant. + """ + dirfd, owned = grant.fd, False + depth = links = 0 + pending = [p for p in rel.split('/') if p not in ('', '.')] + busy.on = True + try: + while pending: + comp = pending.pop(0) + last = not pending + if comp == '..': + if depth == 0: + raise _error('path leaves the grant: %s' % rel) + depth -= 1 + nxt = _open('..', _O_RDONLY | _O_DIRECTORY, dir_fd=dirfd) + if owned: + _close(dirfd) + dirfd, owned = nxt, True + continue + if last and not follow_last: + return dirfd, comp, owned + try: + nxt = _open(comp, _O_RDONLY | _O_NOFOLLOW, dir_fd=dirfd) + except OSError as exc: + if exc.errno not in (_ELOOP, _EMLINK): + if last: + # Not there. That is not a containment answer; + # let the caller's own call raise its own error. + return dirfd, comp, owned + raise + if links >= max_links: + raise _error('too many symlinks: %s' % rel) from None + links += 1 + target = _readlink(comp, dir_fd=dirfd) + if target.startswith('/'): + # Refused rather than reinterpreted: resolving it + # against the grant would silently mean something + # other than what it says. + raise _error( + 'symlink leaves the grant: %s' % rel) from None + pending = [p for p in target.split('/') + if p not in ('', '.')] + pending + continue + if last: + _close(nxt) + return dirfd, comp, owned + if owned: + _close(dirfd) + dirfd, owned = nxt, True + depth += 1 + return dirfd, '.', owned + except BaseException: + if owned: + _close(dirfd) + raise + finally: + busy.on = False + + def contained(grant, path, need, follow): + """Is `path` inside `grant`, with `need` access? + + `path` keeps its `..` deliberately: collapsing them first is what + makes a check disagree with the kernel, because `link/..` is the + directory the link points into and not the one the link sits in. + """ + if need == WRITE and not grant.writable(): + return False + rel = grant.remainder(path) + if rel is None: + return False + if lexical: + depth = 0 + for comp in rel.split('/'): + if comp in ('', '.'): + continue + depth += -1 if comp == '..' else 1 + if depth < 0: + return False + return True + try: + dirfd, _comp, owned = walk(grant, rel, follow_last=follow) + except _error: + return False + except OSError: + # A component that is not there is not a containment answer: the + # path was inside the grant, it simply does not exist. + return True + if owned: + _close(dirfd) + return True + + def check_path(path, need, event, follow=True, opening=False): + # Conversion first and unguarded, because `__fspath__` is user code + # and has to run with the hook live, so its own opens are checked. + if not isinstance(path, (str, bytes)): + if isinstance(path, int): + # A descriptor. Reading through one is already granted, + # since opening it was checked; changing what it names is + # not, because a descriptor cannot be mapped back to a path + # portably enough to check. + if need == WRITE: + raise _error( + '%s: a capability set grants no change through a ' + 'descriptor' % event) + return + if not hasattr(path, '__fspath__'): + return + path = _fspath(path) + if isinstance(path, bytes): + try: + path = _fsdecode(path) + except ValueError: + raise _error('%s: undecodable path' % event) from None + absolute = path if path.startswith('/') \ + else _getcwd().rstrip('/') + '/' + path + if opening and need == READ \ + and files.get(_normpath(_abspath(absolute))) == READ: + return + for grant in dirs: + if contained(grant, absolute, need, follow): + return + raise _error('%s: %s is not granted for %s' % (event, path, need)) + + def writes(mode, flags): + """Does this open ask for anything but reading?""" + if isinstance(mode, str) and mode: + return any(c in mode for c in 'wax+') + if isinstance(flags, int): + return bool(flags & _O_WRITES) + return True + + def check_net(kind, event, args): + sock_obj = args[0] if args else None + address = args[1] if len(args) > 1 else None + if not isinstance(address, tuple) or len(address) < 2: + # A Unix socket names a path, but reaching one is talking to + # whatever is behind it, which is not something a directory + # grant says anything about. A descriptor Erlang passed over is + # unaffected: it is connected or listening already. + raise _error( + '%s: a capability set grants no unix-socket or unknown ' + 'address; a descriptor has to come from Erlang' % event) + if net is None: + raise _error('%s: no network was granted' % event) + host, port = address[0], address[1] + try: + addr = _ip_address(host) + except ValueError: + # A rule names addresses, so an unresolved name matches none. + raise _error('%s: %r is not granted' % (event, address)) from None + mapped = getattr(addr, 'ipv4_mapped', None) + if mapped is not None: + addr = mapped + proto = 'udp' if getattr(sock_obj, 'type', None) == _SOCK_DGRAM \ + else 'tcp' + for rule_proto, rule_net, lo, hi in net[kind]: + if rule_proto == proto and lo <= int(port) <= hi \ + and addr in rule_net: + return + raise _error('%s: %r is not granted' % (event, address)) + + def hook(event, args): + if getattr(busy, 'on', False): + return + if event in subprocess_events or event.startswith(exec_prefixes): + raise _error('%s: a capability set grants no subprocess' % event) + if event.startswith(ctypes_prefix): + raise _error( + '%s: a capability set grants no ctypes, which would reach ' + 'past every other rule' % event) + if event in signal_events: + if event == 'os.killpg' or not args or args[0] != _getpid(): + raise _error( + '%s: a capability set grants no signals to other ' + 'processes' % event) + return + if event in resolve_events: + if net is None or not net['resolve']: + raise _error( + '%s: resolution is its own capability and was not ' + 'granted' % event) + return + if event == 'open': + check_path(args[0], WRITE if writes(args[1], args[2]) else READ, + event, opening=True) + elif event in path_events: + need = path_events[event] + follow = event not in name_events + check_path(args[0], need, event, follow) + if event in two_path_events and len(args) > 1: + check_path(args[1], WRITE, event, follow) + elif event in ('socket.connect', 'socket.sendto'): + check_net('connect', event, args) + elif event == 'socket.bind': + check_net('listen', event, args) + + return _Enforcer(hook=hook, walk=walk, contained=contained, + check_path=check_path, writes=writes) + + +def _parse_net(net): + if not net: + return None + out = {'resolve': bool(net.get('resolve')), 'connect': [], 'listen': []} + for kind in ('connect', 'listen'): + for rule in net.get(kind) or (): + lo, hi = rule['ports'] + out[kind].append((rule['proto'], + ipaddress.ip_network(rule['cidr']), + int(lo), int(hi))) + return out + + +def _disarm_unaudited(): + """Take away the calls CPython does not announce. + + `os.mkfifo` and `os.mknod` create something and raise no audit event, so + a hook cannot refuse them. Removing the names is not a boundary either, + but it is the difference between a documented gap and an open one. + """ + import posix + for name in _UNAUDITED_CREATORS: + for module in (os, posix): + if hasattr(module, name): + try: + delattr(module, name) + except (AttributeError, TypeError): + pass + + +def install(caps): + """Install the capability set. Called once, before any user code.""" + global _summary + if _summary is not None: + return [] + st = _State() + problems = [] + st.lexical = os.open not in os.supports_dir_fd + if st.lexical: + problems.append('this platform has no openat: paths are checked ' + 'lexically and a symlink out of a grant is not seen') + + # Everything the interpreter itself reads. Without these nothing + # imports, which is why WASI preopens its sysroot too. + auto = [(p, _READ) for p in [sys.prefix, sys.base_prefix] + list(sys.path) + if p] + named = [(d['path'], d['access']) for d in caps.get('dirs') or ()] + + seen = set() + for path, access in auto + named: + real = os.path.normpath(os.path.abspath(path)) + if (real, access) in seen: + continue + seen.add((real, access)) + try: + st.dirs.append(_Grant(real, access)) + except OSError as exc: + if access != _READ: + problems.append('cannot open granted directory %s: %s' + % (path, exc)) + for dev in _ALWAYS_READ: + st.files[dev] = _READ + st.net = _parse_net(caps.get('net')) + + _disarm_unaudited() + _summary = { + 'dirs': tuple((g.path, g.access) for g in st.dirs), + 'net': None if st.net is None else { + 'connect': tuple(_rule_text(r) for r in st.net['connect']), + 'listen': tuple(_rule_text(r) for r in st.net['listen']), + 'resolve': st.net['resolve'], + }, + 'strict_paths': not st.lexical, + } + sys.addaudithook(_make_enforcer(st).hook) + return problems + + +def installed(): + return _summary is not None + + +def grants(): + """What was granted, for `child_info` and `erlang.caps()`. + + A fresh copy each time: this is something to look at, never something + the hook consults. + """ + if _summary is None: + return None + net = _summary['net'] + return { + 'dirs': [tuple(d) for d in _summary['dirs']], + 'net': None if net is None else { + 'connect': list(net['connect']), + 'listen': list(net['listen']), + 'resolve': net['resolve'], + }, + 'strict_paths': _summary['strict_paths'], + } + + +def _rule_text(rule): + proto, net, lo, hi = rule + return '%s %s %d-%d' % (proto, net, lo, hi) diff --git a/priv/_erlang_impl/_isolated.py b/priv/_erlang_impl/_isolated.py index 04f6a97..d6b215f 100644 --- a/priv/_erlang_impl/_isolated.py +++ b/priv/_erlang_impl/_isolated.py @@ -751,6 +751,12 @@ def atom(name): def is_isolated(): return True + def caps(): + """What this child was granted, or None when it holds every + authority the user it runs as holds.""" + from . import _caps + return _caps.grants() + def run(main, *, debug=None): loop = rt.get_loop() if debug is not None: @@ -806,7 +812,7 @@ def __getattr__(name): call=call, async_call=async_call, send=send, whereis=whereis, self=self_, atom=atom, Atom=Atom, Pid=Pid, Ref=Ref, Port=Port, ProcessError=ProcessError, SuspensionRequired=SuspensionRequired, - Function=Function, is_isolated=is_isolated, run=run, + Function=Function, is_isolated=is_isolated, caps=caps, run=run, SharedMemory=SharedMemory, SharedBuffer=SharedBuffer, new_event_loop=new_event_loop, get_event_loop_policy=get_event_loop_policy, install=install, spawn_task=spawn_task, sleep=sleep, log=log, @@ -823,7 +829,8 @@ def __getattr__(name): ByteChannel=_not_supported('erlang.ByteChannel'), __all__=['call', 'async_call', 'send', 'whereis', 'self', 'atom', 'Atom', 'Pid', 'Ref', 'ProcessError', 'SuspensionRequired', - 'run', 'sleep', 'spawn_task', 'server', 'is_isolated'], + 'run', 'sleep', 'spawn_task', 'server', 'is_isolated', + 'caps'], ) mod.__dict__.update(ns) sys.modules['erlang'] = mod diff --git a/priv/py_isolated_child.py b/priv/py_isolated_child.py index 006e074..508ccd4 100644 --- a/priv/py_isolated_child.py +++ b/priv/py_isolated_child.py @@ -38,7 +38,7 @@ def _die(reason): def _parse_args(argv): if len(argv) < 2: _die('usage: py_isolated_child.py SOCKET_PATH [options]') - opts = {'socket': argv[1], 'rlimits': {}, 'cgroup': None} + opts = {'socket': argv[1], 'rlimits': {}, 'cgroup': None, 'caps': None} i = 2 while i < len(argv): flag = argv[i] @@ -48,6 +48,13 @@ def _parse_args(argv): elif flag == '--cgroup': opts['cgroup'] = argv[i + 1] i += 2 + elif flag == '--caps-json': + import json + try: + opts['caps'] = json.loads(argv[i + 1]) + except ValueError as exc: + _die('bad --caps-json: %s' % exc) + i += 2 else: _die('unknown option %s' % flag) return opts @@ -167,6 +174,15 @@ def _connect(path): return sock +def _caps_summary(): + """What was granted, so `py_context:child_info/1` can report it.""" + try: + from _erlang_impl import _caps + return _caps.grants() + except Exception: + return None + + def main(argv): opts = _parse_args(argv) _arm_parent_death() @@ -191,10 +207,20 @@ def main(argv): if _AS_VIA_WATCHDOG and 'as' in opts['rlimits']: _start_memory_watchdog(opts['rlimits']['as'], runtime) - if rlimit_errors or cgroup_error: + # Last thing before the parent is told this child is ready, so the + # runtime's own imports are not subject to the grants and everything + # that runs afterwards is: the registered imports, the preload, and + # every request. + caps_errors = [] + if opts['caps'] is not None: + from _erlang_impl import _caps + caps_errors = _caps.install(opts['caps']) + + if rlimit_errors or cgroup_error or caps_errors: problems = [(Atom('rlimit'), Atom(k), msg) for k, msg in rlimit_errors] if cgroup_error: problems.append((Atom('cgroup'), cgroup_error)) + problems += [(Atom('caps'), msg) for msg in caps_errors] try: runtime.event((Atom('startup_error'), problems)) finally: @@ -205,6 +231,7 @@ def main(argv): Atom('python_version'): '%d.%d.%d' % sys.version_info[:3], Atom('executable'): sys.executable, Atom('platform'): sys.platform, + Atom('caps'): _caps_summary(), } runtime.event((Atom('ready'), info)) diff --git a/priv/tests/test_caps.py b/priv/tests/test_caps.py new file mode 100644 index 0000000..31bd851 --- /dev/null +++ b/priv/tests/test_caps.py @@ -0,0 +1,253 @@ +"""Unit tests for the capability resolver and the address matcher. + +These run without a VM, so the containment rules can be read and changed +without a Common Test round trip. `py_isolated_caps_SUITE` covers the same +ground through a real child. + + cd priv && python3 -m unittest tests.test_caps +""" + +import os +import shutil +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from _erlang_impl import _caps # noqa: E402 + + +class PathContainment(unittest.TestCase): + """The tree is the one `wasi_SUITE` uses, so the cases line up.""" + + @classmethod + def setUpClass(cls): + cls.root = tempfile.mkdtemp() + cls.data = os.path.join(cls.root, 'data') + cls.secret = os.path.join(cls.root, 'secret') + os.makedirs(os.path.join(cls.data, 'sub')) + os.makedirs(cls.secret) + _write(os.path.join(cls.data, 'note.txt'), 'inside') + _write(os.path.join(cls.data, 'sub', 'deep.txt'), 'deep') + _write(os.path.join(cls.secret, 'key.txt'), 'secret') + os.symlink(os.path.join(cls.secret, 'key.txt'), + os.path.join(cls.data, 'escape')) + os.symlink(cls.secret, os.path.join(cls.data, 'outdir')) + os.symlink('note.txt', os.path.join(cls.data, 'here')) + os.symlink('loop', os.path.join(cls.data, 'loop')) + cls.grant = _caps._Grant(cls.data, 'write') + # The enforcement lives in the factory's closure, so that is what a + # test drives; there is nothing at module level to call. + state = _caps._State() + state.dirs = [cls.grant] + cls.enf = _caps._make_enforcer(state) + + @classmethod + def tearDownClass(cls): + os.close(cls.grant.fd) + shutil.rmtree(cls.root, ignore_errors=True) + + def reaches(self, rel, follow=True): + """Does `rel` resolve inside the grant?""" + try: + fd, _comp, owned = self.enf.walk(self.grant, rel, follow_last=follow) + except _caps.CapabilityError: + return False + except OSError: + return True # inside the grant, simply not there + if owned: + os.close(fd) + return True + + def test_a_plain_name_resolves(self): + self.assertTrue(self.reaches('note.txt')) + self.assertTrue(self.reaches('./note.txt')) + self.assertTrue(self.reaches('sub/deep.txt')) + + def test_parent_traversal_is_refused(self): + self.assertFalse(self.reaches('../secret/key.txt')) + self.assertFalse(self.reaches('..')) + + def test_partial_traversal_is_refused(self): + self.assertFalse(self.reaches('sub/../../secret/key.txt')) + # And the half of it that is legal really is legal, or the assertion + # above would hold just as well with `..` refused outright. + self.assertTrue(self.reaches('sub/../note.txt')) + + def test_a_symlink_out_is_refused(self): + self.assertFalse(self.reaches('escape')) + + def test_a_symlinked_directory_prefix_is_refused(self): + self.assertFalse(self.reaches('outdir/key.txt')) + + def test_a_symlink_inside_is_followed(self): + self.assertTrue(self.reaches('here')) + + def test_a_cycle_is_refused_rather_than_followed(self): + self.assertFalse(self.reaches('loop')) + + def test_a_name_is_not_followed_when_the_caller_names_it(self): + # Removing a link that leads out of the grant removes something + # inside the grant, so naming it is allowed where following is not. + self.assertFalse(self.reaches('escape', follow=True)) + self.assertTrue(self.reaches('escape', follow=False)) + + def test_a_missing_name_is_not_a_containment_answer(self): + self.assertTrue(self.reaches('missing.txt')) + self.assertFalse(self.reaches('../missing.txt')) + + def test_the_walk_leaks_no_descriptors(self): + before = _lowest_free_fd() + for _ in range(200): + for rel in ('note.txt', 'escape', 'loop', 'sub/../note.txt', + '../secret/key.txt', 'outdir/key.txt'): + self.reaches(rel) + self.assertLessEqual(_lowest_free_fd(), before + 1) + + +class ModuleState(unittest.TestCase): + """The grants must not be reachable through this module. + + An enforcement decision that reads a module attribute is one any Python + in the process can switch off by assigning to it. + """ + + def test_no_lever_is_exported(self): + self.assertFalse(hasattr(_caps, 'allow_path')) + + def test_nothing_on_the_decision_path_lives_at_module_level(self): + # A name the hook resolves when it runs is a name any Python in the + # process can rebind, so none of these may exist here. + for name in ('_walk', '_contained', '_check_path', '_writes', + '_net_allows', '_local', '_state'): + self.assertFalse(hasattr(_caps, name), name) + + def test_the_decision_path_loads_no_module_global(self): + # Not `co_names`, which also lists attribute names: what matters is + # what the code actually loads from the module's namespace, because + # that is what an assignment to this module would change. + import dis + enf = _caps._make_enforcer(_caps._State()) + for part in ('hook', 'walk', 'contained', 'check_path', 'writes'): + code = getattr(enf, part).__code__ + loaded = {i.argval for i in dis.get_instructions(code) + if i.opname == 'LOAD_GLOBAL'} + self.assertEqual(loaded & set(vars(_caps)), set(), part) + + def test_grants_returns_a_copy(self): + # What `grants()` hands back is something to look at, so mutating it + # must not reach anything. + before = _caps.grants() + if before is None: + self.skipTest('no capability set installed in this process') + before['dirs'].append(('/etc', 'write')) + self.assertNotIn(('/etc', 'write'), _caps.grants()['dirs']) + + +class WriteIntent(unittest.TestCase): + """Which opens need a write grant.""" + + def setUp(self): + self.writes = _caps._make_enforcer(_caps._State()).writes + + def test_modes(self): + for mode in ('w', 'a', 'x', 'r+', 'w+b', 'rb+'): + self.assertTrue(self.writes(mode, 0), mode) + for mode in ('r', 'rb', 'rt'): + self.assertFalse(self.writes(mode, 0), mode) + + def test_flags(self): + for flag in (os.O_WRONLY, os.O_RDWR, os.O_CREAT, os.O_TRUNC, + os.O_APPEND, os.O_RDONLY | os.O_CREAT): + self.assertTrue(self.writes(None, flag), flag) + self.assertFalse(self.writes(None, os.O_RDONLY)) + + def test_an_unreadable_intent_is_taken_as_a_write(self): + self.assertTrue(self.writes(None, None)) + + +class AddressMatching(unittest.TestCase): + """The rules erlang_wasm's `wasi_net_SUITE` checks, matched here.""" + + @staticmethod + def grant(connect=(), listen=(), resolve=False): + return _caps._parse_net({'connect': list(connect), + 'listen': list(listen), + 'resolve': resolve}) + + @staticmethod + def rule(cidr, lo, hi, proto='tcp'): + return {'proto': proto, 'cidr': cidr, 'ports': [lo, hi]} + + def allows(self, net, addr, port, kind='connect', dgram=False): + # The matcher lives in the enforcer, so a test builds one over the + # grant it wants rather than poking module state. + state = _caps._State() + state.net = net + enf = _caps._make_enforcer(state) + event = 'socket.bind' if kind == 'listen' else 'socket.connect' + try: + enf.hook(event, (_FakeSocket(dgram), (addr, port))) + return True + except _caps.CapabilityError: + return False + + def test_a_network_and_a_port_range(self): + g = self.grant(connect=[self.rule('10.0.0.0/8', 8000, 8099)]) + self.assertTrue(self.allows(g, '10.1.2.3', 8000)) + self.assertTrue(self.allows(g, '10.255.255.255', 8099)) + self.assertFalse(self.allows(g, '11.0.0.1', 8000)) + self.assertFalse(self.allows(g, '10.1.2.3', 8100)) + self.assertFalse(self.allows(g, '10.1.2.3', 7999)) + + def test_ipv4_mapped_ipv6_is_the_same_address(self): + # A matcher comparing text would let this past a v4 rule. + g = self.grant(connect=[self.rule('127.0.0.0/8', 80, 80)]) + self.assertTrue(self.allows(g, '::ffff:127.0.0.1', 80)) + self.assertFalse(self.allows(g, '::1', 80)) + + def test_udp_and_tcp_are_separate(self): + g = self.grant(connect=[self.rule('127.0.0.1/32', 53, 53, 'udp')]) + self.assertTrue(self.allows(g, '127.0.0.1', 53, dgram=True)) + self.assertFalse(self.allows(g, '127.0.0.1', 53)) + + def test_connect_and_listen_are_separate(self): + g = self.grant(connect=[self.rule('127.0.0.1/32', 80, 80)]) + self.assertTrue(self.allows(g, '127.0.0.1', 80, kind='connect')) + self.assertFalse(self.allows(g, '127.0.0.1', 80, kind='listen')) + + def test_a_wildcard_really_is_a_wildcard(self): + # Nothing is denied implicitly: 0.0.0.0/0 includes the link-local + # and cloud metadata addresses, and this does not second-guess it. + g = self.grant(connect=[self.rule('0.0.0.0/0', 0, 65535)]) + self.assertTrue(self.allows(g, '169.254.169.254', 80)) + + def test_a_name_matches_nothing(self): + # A rule names addresses, so an unresolved name cannot match one. + g = self.grant(connect=[self.rule('0.0.0.0/0', 0, 65535)]) + self.assertFalse(self.allows(g, 'example.com', 80)) + + def test_no_grant_allows_nothing(self): + self.assertFalse(self.allows(None, '127.0.0.1', 80)) + + +class _FakeSocket: + def __init__(self, dgram): + import socket + self.type = socket.SOCK_DGRAM if dgram else socket.SOCK_STREAM + + +def _write(path, text): + with open(path, 'w') as fh: + fh.write(text) + + +def _lowest_free_fd(): + fd = os.dup(0) + os.close(fd) + return fd + + +if __name__ == '__main__': + unittest.main() diff --git a/rebar.config b/rebar.config index ddae020..5b7bc6b 100644 --- a/rebar.config +++ b/rebar.config @@ -71,6 +71,7 @@ <<"docs/asyncio.md">>, <<"docs/workers.md">>, <<"docs/isolated.md">>, + <<"docs/capabilities.md">>, <<"docs/reactor.md">>, <<"docs/process-bound-envs.md">>, <<"docs/security.md">>, @@ -91,6 +92,7 @@ <<"docs/decisions/0006-shared-memory-over-iommap.md">>, <<"docs/decisions/0007-remove-legacy-execution-paths.md">>, <<"docs/decisions/0008-pipe-io-rules.md">>, + <<"docs/decisions/0009-child-capabilities.md">>, <<"docs/preload.md">>, <<"docs/owngil_internals.md">>, <<"docs/event_loop_architecture.md">> @@ -119,6 +121,7 @@ <<"docs/asyncio.md">>, <<"docs/workers.md">>, <<"docs/isolated.md">>, + <<"docs/capabilities.md">>, <<"docs/reactor.md">>, <<"docs/process-bound-envs.md">>, <<"docs/security.md">>, @@ -145,7 +148,8 @@ <<"docs/decisions/0005-py-isolated-gen-statem.md">>, <<"docs/decisions/0006-shared-memory-over-iommap.md">>, <<"docs/decisions/0007-remove-legacy-execution-paths.md">>, - <<"docs/decisions/0008-pipe-io-rules.md">> + <<"docs/decisions/0008-pipe-io-rules.md">>, + <<"docs/decisions/0009-child-capabilities.md">> ]} ]} ]}. diff --git a/src/erlang_python.app.src b/src/erlang_python.app.src index c478e29..57c8879 100644 --- a/src/erlang_python.app.src +++ b/src/erlang_python.app.src @@ -1,6 +1,6 @@ {application, erlang_python, [ {description, "Execute Python applications from Erlang using dirty NIFs"}, - {vsn, "5.0.0"}, + {vsn, "5.1.0"}, {registered, []}, {mod, {erlang_python_app, []}}, {applications, [ diff --git a/src/py_caps.erl b/src/py_caps.erl new file mode 100644 index 0000000..0e8ab2c --- /dev/null +++ b/src/py_caps.erl @@ -0,0 +1,263 @@ +%% Copyright 2026 Benoit Chesneau +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%% @doc The `caps' option: what an isolated child may reach. +%%% +%%% A capability grant names directories, environment variables and network +%%% addresses. Anything not named is not reachable. This module reads the +%%% option, refuses what it cannot make sense of, and renders the result as +%%% the JSON the child parses before it runs any user code. +%%% +%%% ``` +%%% #{dirs => [{"/srv/models", read}, {"/var/data/job42", write}], +%%% env => #{<<"MODEL_DIR">> => <<"/srv/models">>}, +%%% net => #{connect => [{tcp, <<"10.0.0.0/8">>, {5432, 5432}}], +%%% listen => [{tcp, <<"127.0.0.1">>, 8080}], +%%% resolve => deny}} +%%% ''' +%%% +%%% A rule is `{Proto, Addr, Port}': `Proto' is `tcp' or `udp', `Addr' is an +%%% address tuple, a binary address or a binary CIDR, and `Port' is an integer, +%%% `{Lo, Hi}' or `any'. Rules name addresses and never host names: a name +%%% would have to be resolved to be checked and resolved again to be used, and +%%% the two answers can differ. Resolution is its own capability and what it +%%% returns carries no authority. +%%% +%%% Checked here rather than in the child, so a typo is a `{bad_caps, _}' +%%% error from `py_context:new/1' rather than a connection refused much later. +%%% A sandbox that silently refuses everything looks exactly like one that +%%% works. +%%% +%%% The rule shape, the IPv4-mapped folding and the masking are taken from +%%% `wasi_net.erl' in erlang_wasm, so a grant means the same thing in both. +%%% +%%% @private +%%% +%%% Shared memory is not granted here and does not work under a capability +%%% set: a region arrives as a path, and granting the directory holding them +%%% would hand over every region this node owns. The way to make it work is +%%% to pass the region's descriptor rather than its name; see +%%% `docs/capabilities.md'. +%%% +%%% Owns: the meaning of the `caps' option and its wire form. +%%% Talks to: `py_context' (validation at `new/1'), `py_isolated' (argv). +%%% Never: enforces anything; the child does that, in +%%% `priv/_erlang_impl/_caps.py'. +%%% @end +-module(py_caps). + +-export([ + validate/1, + to_json/1 +]). + +-export_type([caps/0, access/0]). + +-type access() :: read | write. +-type rule() :: {tcp | udp, {inet:ip_address(), 0..128}, {0..65535, 0..65535}}. +-type net() :: none | #{connect := [rule()], listen := [rule()], + resolve := boolean()}. +-type caps() :: #{dirs := [{binary(), access()}], + env := #{binary() => binary()}, + net := net()}. + +%%% ============================================================================ +%%% API +%%% ============================================================================ + +%% @doc Read a `caps' option into the form the child is given. +%% +%% `{error, {bad_caps, Detail}}' names the part that could not be read. +-spec validate(term()) -> {ok, caps()} | {error, {bad_caps, term()}}. +validate(Map) when is_map(Map) -> + try + Known = [dirs, env, net], + case maps:keys(maps:without(Known, Map)) of + [] -> ok; + Extra -> throw({unknown_keys, Extra}) + end, + {ok, #{dirs => dirs(maps:get(dirs, Map, [])), + env => env(maps:get(env, Map, #{})), + net => net(maps:get(net, Map, none))}} + catch + throw:Detail -> {error, {bad_caps, Detail}} + end; +validate(Other) -> + {error, {bad_caps, Other}}. + +%% @doc Render a validated grant as the JSON passed to the child in argv. +-spec to_json(caps()) -> binary(). +to_json(#{dirs := Dirs, env := Env, net := Net}) -> + iolist_to_binary(json:encode( + #{<<"dirs">> => [#{<<"path">> => P, <<"access">> => atom_to_binary(A)} + || {P, A} <- Dirs], + <<"env">> => Env, + <<"net">> => net_json(Net)})). + +%%% ============================================================================ +%%% Directories and environment +%%% ============================================================================ + +dirs(L) when is_list(L) -> + [dir(D) || D <- L]; +dirs(Other) -> + throw({dirs, Other}). + +%% An absolute path, so that what a grant covers does not depend on the +%% working directory of whoever wrote it. +dir({Path, Access}) when Access =:= read; Access =:= write -> + case to_bin(Path) of + <<"/", _/binary>> = Bin -> {Bin, Access}; + _ -> throw({dir_not_absolute, Path}) + end; +dir(Other) -> + throw({dir, Other}). + +env(Map) when is_map(Map) -> + maps:from_list([{to_bin(K), to_bin(V)} || {K, V} <- maps:to_list(Map)]); +env(Other) -> + throw({env, Other}). + +%%% ============================================================================ +%%% Network grant +%%% +%%% From wasi_net.erl (erlang_wasm), which parses the same rules for a WASM +%%% guest. Kept in step with it deliberately: a grant should mean one thing. +%%% ============================================================================ + +net(none) -> none; +net(undefined) -> none; +net(Map) when is_map(Map) -> + case maps:keys(maps:without([connect, listen, resolve], Map)) of + [] -> ok; + Extra -> throw({net, {unknown_keys, Extra}}) + end, + #{connect => rules(maps:get(connect, Map, [])), + listen => rules(maps:get(listen, Map, [])), + resolve => resolve(maps:get(resolve, Map, deny))}; +net(Other) -> + throw({net, Other}). + +resolve(allow) -> true; +resolve(deny) -> false; +resolve(Other) -> throw({net, {resolve, Other}}). + +rules(L) when is_list(L) -> [rule(R) || R <- L]; +rules(Other) -> throw({net, Other}). + +rule({Proto, Addr, Port}) when Proto =:= tcp; Proto =:= udp -> + {Proto, cidr(Addr), ports(Port)}; +rule(Other) -> + throw({net, {rule, Other}}). + +%% An address with no prefix length is one host: a full-width prefix. +cidr(Bin) when is_binary(Bin) -> + case binary:split(Bin, <<"/">>) of + [Addr] -> host(parse_or_fail(Addr)); + [Addr, Len] -> network(parse_or_fail(Addr), integer_or_fail(Len, Bin), Bin) + end; +cidr(Tuple) when tuple_size(Tuple) =:= 4; tuple_size(Tuple) =:= 8 -> + host(Tuple); +cidr(Other) -> + throw({net, {address, Other}}). + +host(Addr0) -> + Addr = normalise(Addr0), + {Addr, width(Addr)}. + +%% The prefix length is written in the notation the address was written in, so +%% a mapped base has to have its 96 mapping bits taken off with it. Below 96 +%% the prefix spans addresses inside and outside the mapped block at once, +%% which has no IPv4 meaning; refuse rather than guess which half was meant. +network(Addr, Bits, Written) -> + case normalise(Addr) of + A when tuple_size(A) =:= 4, Bits >= 0, Bits =< 32 -> + {mask(A, Bits), Bits}; + A when tuple_size(A) =:= 8, Bits >= 0, Bits =< 128 -> + {mask(A, Bits), Bits}; + V4 when Bits >= 96, Bits =< 128 -> + {mask(V4, Bits - 96), Bits - 96}; + _ -> + throw({net, {address, Written}}) + end. + +ports(any) -> {0, 65535}; +ports(P) when is_integer(P), P >= 0, P =< 65535 -> {P, P}; +ports({Lo, Hi}) when is_integer(Lo), is_integer(Hi), Lo >= 0, Lo =< Hi, + Hi =< 65535 -> {Lo, Hi}; +ports(Other) -> throw({net, {port, Other}}). + +parse_or_fail(Bin) -> + case inet:parse_address(binary_to_list(Bin)) of + {ok, Addr} -> normalise(Addr); + {error, _} -> throw({net, {address, Bin}}) + end. + +integer_or_fail(Bin, Written) -> + try binary_to_integer(Bin) + catch _:_ -> throw({net, {address, Written}}) + end. + +%% Fold an IPv4-mapped IPv6 address onto the IPv4 address it reaches, so +%% `::ffff:127.0.0.1' cannot walk past a `127.0.0.0/8' rule. The deprecated +%% IPv4-compatible block is left alone: `::0.0.0.1' and `::1' are the same +%% address, so folding it would make loopback ambiguous. +normalise({0, 0, 0, 0, 0, 16#ffff, X, Y}) -> + {X bsr 8, X band 16#ff, Y bsr 8, Y band 16#ff}; +normalise(Addr) -> + Addr. + +width(Addr) when tuple_size(Addr) =:= 4 -> 32; +width(Addr) when tuple_size(Addr) =:= 8 -> 128. + +%% Zeroing the host bits, so a rule written `10.1.2.3/8' means the same +%% network as `10.0.0.0/8' rather than never matching anything. +mask(Addr, Bits) -> + W = width(Addr), + from_int(to_int(Addr) band (((1 bsl Bits) - 1) bsl (W - Bits)), W). + +to_int(Addr) -> + Size = part_size(Addr), + lists:foldl(fun(P, Acc) -> (Acc bsl Size) bor P end, 0, tuple_to_list(Addr)). + +from_int(N, 32) -> + <> = <>, + {A, B, C, D}; +from_int(N, 128) -> + <> = <>, + {A, B, C, D, E, F, G, H}. + +part_size(Addr) when tuple_size(Addr) =:= 4 -> 8; +part_size(Addr) when tuple_size(Addr) =:= 8 -> 16. + +%%% ============================================================================ +%%% Wire form +%%% ============================================================================ + +net_json(none) -> + null; +net_json(#{connect := C, listen := L, resolve := R}) -> + #{<<"connect">> => [rule_json(Rule) || Rule <- C], + <<"listen">> => [rule_json(Rule) || Rule <- L], + <<"resolve">> => R}. + +%% The child matches with Python's `ipaddress', so rules cross as the CIDR +%% text that module reads. The address is already masked and folded here, so +%% both sides agree on what a rule covers without parsing it twice. +rule_json({Proto, {Addr, Bits}, {Lo, Hi}}) -> + #{<<"proto">> => atom_to_binary(Proto), + <<"cidr">> => iolist_to_binary([inet:ntoa(Addr), "/", integer_to_list(Bits)]), + <<"ports">> => [Lo, Hi]}. + +to_bin(B) when is_binary(B) -> B; +to_bin(L) when is_list(L) -> list_to_binary(L); +to_bin(A) when is_atom(A) -> atom_to_binary(A, utf8); +to_bin(Other) -> throw({not_a_string, Other}). diff --git a/src/py_context.erl b/src/py_context.erl index 757df51..210a74f 100644 --- a/src/py_context.erl +++ b/src/py_context.erl @@ -188,7 +188,32 @@ stop(Ctx) when is_pid(Ctx) -> new(Opts) when is_map(Opts) -> Mode = maps:get(mode, Opts, worker), Id = erlang:unique_integer([positive]), - start_link(Id, Mode, Opts). + case check_caps(Mode, Opts) of + ok -> start_link(Id, Mode, Opts); + {error, _} = Err -> Err + end. + +%% @private A capability grant is only meaningful where the interpreter is a +%% child process. Read it here, in the caller, so a malformed rule is the +%% configuration error it is rather than a connection refused much later. +check_caps(Mode, Opts) -> + case maps:get(caps, Opts, undefined) of + undefined -> + ok; + _ when Mode =/= isolated -> + {error, {caps_requires_isolated, Mode}}; + _ when is_map_key(env, Opts) -> + %% The `env' option adds to the child's environment and a grant + %% says what the whole of it is. Taking both would mean one of + %% them silently losing, so say so instead: `caps.env' is the + %% one that names an environment. + {error, {bad_caps, env_option_conflicts_with_caps_env}}; + Caps -> + case py_caps:validate(Caps) of + {ok, _} -> ok; + {error, _} = Err -> Err + end + end. %% @doc Alias for stop/1 for API consistency. -spec destroy(context()) -> ok. diff --git a/src/py_isolated.erl b/src/py_isolated.erl index 566c4f8..3a26721 100644 --- a/src/py_isolated.erl +++ b/src/py_isolated.erl @@ -446,10 +446,38 @@ resolve_exe(Exe) -> start_child(#data{opts = Opts} = St) -> case check_platform_opts(Opts) of - ok -> start_child_1(St); - {error, _} = Err -> Err + ok -> + case caps(Opts) of + {ok, _} -> start_child_1(St); + {error, _} = Err -> Err + end; + {error, _} = Err -> + Err end. +%% A grant is read here as well as in py_context:new/1, so a context started +%% by any other route still fails with the configuration error rather than +%% with a child that refuses everything. +caps(Opts) -> + case maps:get(caps, Opts, undefined) of + undefined -> + {ok, none}; + _ when is_map_key(env, Opts) -> + {error, {bad_caps, env_option_conflicts_with_caps_env}}; + Caps -> + case py_caps:validate(Caps) of + {ok, Valid} -> {ok, with_import_paths(Valid, Opts)}; + {error, _} = Err -> Err + end + end. + +%% A directory named in `paths' is one the child was told to import from, so +%% it is granted for reading. Saying it twice would be a trap, and leaving it +%% ungranted turns `paths' into an import error rather than a grant error. +with_import_paths(#{dirs := Dirs} = Caps, Opts) -> + Extra = [{to_bin(P), read} || P <- maps:get(paths, Opts, [])], + Caps#{dirs => Dirs ++ [D || D <- Extra, not lists:member(D, Dirs)]}. + %% cgroups exist only on Linux; rlimits are POSIX and apply everywhere. %% RLIMIT_AS is enforced by the kernel on Linux and FreeBSD; on macOS the %% child enforces `as' with a watchdog thread on its resident set. @@ -483,7 +511,8 @@ spawn_child(Python, Opts) -> ok = socket:bind(L, #{family => local, path => Path}), ok = socket:listen(L), Script = filename:join(priv_dir(), "py_isolated_child.py"), - Args = [Script, Path | rlimit_args(Opts) ++ cgroup_args(Opts)], + Args = [Script, Path | rlimit_args(Opts) ++ cgroup_args(Opts) + ++ caps_args(Opts)], PortOpts = [exit_status, stderr_to_stdout, binary, use_stdio, {args, Args}, {env, env_opt(Opts)}], Port = open_port({spawn_executable, Python}, PortOpts), @@ -1187,8 +1216,35 @@ cgroup_args(Opts) -> Dir -> ["--cgroup", to_list(Dir)] end. +%% The grant travels in argv rather than in the handshake because argv is +%% read in the child's prologue, before the socket exists and before any +%% user code can run. +caps_args(Opts) -> + case caps(Opts) of + {ok, none} -> []; + {ok, Caps} -> ["--caps-json", binary_to_list(py_caps:to_json(Caps))] + end. + +%% Without a grant the child inherits the VM's environment and the `env' +%% option adds to it. With one, it gets what the grant names and nothing +%% else, except the loader variables, which belong to whoever started the +%% node rather than to the workload and without which an interpreter built +%% against a private libpython does not start at all. env_opt(Opts) -> - [{to_list(K), to_list(V)} || {K, V} <- maps:to_list(maps:get(env, Opts, #{}))]. + User = [{to_list(K), to_list(V)} || {K, V} <- maps:to_list(maps:get(env, Opts, #{}))], + case caps(Opts) of + {ok, none} -> + User; + {ok, #{env := Granted}} -> + %% `User' is empty here: a grant and the `env' option together + %% are refused in caps/1, because the port keeps the last of two + %% settings for the same name and the option would win. + Keep = ["LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH", + "DYLD_FALLBACK_LIBRARY_PATH"], + Clear = [{Name, false} || {Name, _} <- os:env(), + not lists:member(Name, Keep)], + Clear ++ [{to_list(K), to_list(V)} || {K, V} <- maps:to_list(Granted)] + end. to_bin(A) when is_atom(A) -> atom_to_binary(A, utf8); to_bin(L) when is_list(L) -> unicode:characters_to_binary(L); diff --git a/test/coverage_audit.md b/test/coverage_audit.md index bb8a8ca..c0a977a 100644 --- a/test/coverage_audit.md +++ b/test/coverage_audit.md @@ -34,6 +34,7 @@ suite is visible. `scripts/check_code_map.sh` requires a row per module. | `py_channel` | `py_channel_SUITE`, `py_byte_channel_SUITE` | | `py_byte_channel` | `py_channel_SUITE`, `py_byte_channel_SUITE` | | `py_buffer` | `py_buffer_SUITE`, `py_isolated_buffer_SUITE` | +| `py_caps` | `py_isolated_caps_SUITE` | | `py_shm` | `py_isolated_shm_SUITE` | | `py_import` | `py_import_SUITE` | | `py_preload` | `py_preload_SUITE` | diff --git a/test/py_isolated_caps_SUITE.erl b/test/py_isolated_caps_SUITE.erl new file mode 100644 index 0000000..6caea88 --- /dev/null +++ b/test/py_isolated_caps_SUITE.erl @@ -0,0 +1,628 @@ +%% @doc The `caps' option: what an isolated child may reach. +%% +%% The path cases are the ones that matter. A capability set that opens the +%% right files is easy; one that reliably refuses the wrong ones is the whole +%% point. Each escape technique gets its own case, and each is refused as a +%% capability error rather than as a missing file, so the error cannot be used +%% to find out what exists outside a grant. +%% +%% The case names follow `wasi_SUITE' and `wasi_net_SUITE' in erlang_wasm, +%% whose grant model this implements, so the two can be read side by side. +-module(py_isolated_caps_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +-export([all/0, init_per_suite/1, end_per_suite/1, + init_per_testcase/2, end_per_testcase/2]). + +-export([ + reads_inside_a_grant/1, + parent_traversal_is_refused/1, + absolute_path_is_refused/1, + partial_traversal_is_refused/1, + symlink_escape_is_refused/1, + symlink_directory_prefix_is_refused/1, + a_symlink_inside_a_grant_is_followed/1, + a_symlink_cycle_is_refused_rather_than_followed/1, + missing_file_inside_a_grant_is_not_a_refusal/1, + a_read_grant_yields_no_write_whatever_the_flags/1, + a_write_grant_allows_create_and_unlink/1, + listing_is_granted_with_the_directory/1, + imports_still_work/1, + a_network_and_a_port_range/1, + an_ungranted_port_is_refused_even_where_something_listens/1, + binding_is_checked_against_listen_not_connect/1, + resolution_is_its_own_capability/1, + resolution_cannot_widen_a_grant/1, + a_wildcard_grant_really_is_a_wildcard/1, + no_net_key_is_no_network/1, + a_passed_fd_still_serves/1, + env_is_what_was_granted_and_nothing_else/1, + subprocess_is_refused/1, + ctypes_is_refused/1, + the_policy_cannot_be_switched_off_from_python/1, + signalling_another_process_is_refused/1, + unaudited_creators_are_taken_away/1, + every_resolver_is_gated_not_only_getaddrinfo/1, + the_env_option_cannot_widen_a_grant/1, + shared_memory_is_refused_under_a_capability_set/1, + a_user_fspath_runs_enforced/1, + a_unix_socket_is_not_a_file/1, + a_read_grant_cannot_become_a_write/1, + caps_survive_a_child_restart/1, + child_info_reports_the_grants/1, + caps_are_rejected_outside_isolated/1, + a_malformed_rule_is_a_configuration_error/1, + no_caps_changes_nothing/1 +]). + +-define(TEST_MOD, py_test_caps). + +all() -> + [ + %% filesystem + reads_inside_a_grant, + parent_traversal_is_refused, + absolute_path_is_refused, + partial_traversal_is_refused, + symlink_escape_is_refused, + symlink_directory_prefix_is_refused, + a_symlink_inside_a_grant_is_followed, + a_symlink_cycle_is_refused_rather_than_followed, + missing_file_inside_a_grant_is_not_a_refusal, + a_read_grant_yields_no_write_whatever_the_flags, + a_write_grant_allows_create_and_unlink, + listing_is_granted_with_the_directory, + imports_still_work, + %% network + a_network_and_a_port_range, + an_ungranted_port_is_refused_even_where_something_listens, + binding_is_checked_against_listen_not_connect, + resolution_is_its_own_capability, + resolution_cannot_widen_a_grant, + a_wildcard_grant_really_is_a_wildcard, + no_net_key_is_no_network, + a_passed_fd_still_serves, + %% the rest + env_is_what_was_granted_and_nothing_else, + subprocess_is_refused, + ctypes_is_refused, + the_policy_cannot_be_switched_off_from_python, + signalling_another_process_is_refused, + unaudited_creators_are_taken_away, + every_resolver_is_gated_not_only_getaddrinfo, + the_env_option_cannot_widen_a_grant, + shared_memory_is_refused_under_a_capability_set, + a_user_fspath_runs_enforced, + a_unix_socket_is_not_a_file, + a_read_grant_cannot_become_a_write, + caps_survive_a_child_restart, + child_info_reports_the_grants, + caps_are_rejected_outside_isolated, + a_malformed_rule_is_a_configuration_error, + no_caps_changes_nothing + ]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(erlang_python), + [{test_dir, filename:join(code:lib_dir(erlang_python), "test")} | Config]. + +end_per_suite(_Config) -> + ok = application:stop(erlang_python), + ok. + +%% A tree with everything the escape cases need, made fresh for each case so +%% one case cannot leave a symlink behind for the next. +%% +%% root/data/note.txt readable +%% root/data/sub/deep.txt readable, one level down +%% root/data/escape -> root/secret/key.txt +%% root/data/outdir -> root/secret +%% root/data/here -> note.txt (stays inside) +%% root/data/loop -> loop +%% root/secret/key.txt never granted +init_per_testcase(TestCase, Config) -> + Root = filename:join(?config(priv_dir, Config), atom_to_list(TestCase)), + Data = filename:join(Root, "data"), + Secret = filename:join(Root, "secret"), + ok = filelib:ensure_path(filename:join(Data, "sub")), + ok = filelib:ensure_path(Secret), + ok = file:write_file(filename:join(Data, "note.txt"), <<"inside">>), + ok = file:write_file(filename:join([Data, "sub", "deep.txt"]), <<"deep">>), + ok = file:write_file(filename:join(Secret, "key.txt"), <<"secret">>), + ok = file:make_symlink(filename:join(Secret, "key.txt"), + filename:join(Data, "escape")), + ok = file:make_symlink(Secret, filename:join(Data, "outdir")), + ok = file:make_symlink("note.txt", filename:join(Data, "here")), + ok = file:make_symlink("loop", filename:join(Data, "loop")), + [{root, Root}, {data, Data}, {secret, Secret} | Config]. + +end_per_testcase(_TestCase, _Config) -> + flush(), + ok. + +%%% ============================================================================ +%%% Filesystem +%%% ============================================================================ + +reads_inside_a_grant(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + {ok, <<"inside">>} = read(C, path(Config, "note.txt")), + {ok, <<"deep">>} = read(C, path(Config, "sub/deep.txt")), + {ok, <<"inside">>} = read(C, path(Config, "./note.txt")), + alive(C), + stop(C). + +parent_traversal_is_refused(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + refused(read(C, path(Config, "../secret/key.txt"))), + alive(C), + stop(C). + +absolute_path_is_refused(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + refused(read(C, "/etc/hosts")), + refused(read(C, filename:join(?config(secret, Config), "key.txt"))), + alive(C), + stop(C). + +%% Leaves the grant and comes back. Refused because the path leaves at any +%% point, not merely because of where it ends up; and the half of it that is +%% legal really is legal, or this case would pass just as well with `..' +%% refused outright. +partial_traversal_is_refused(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + refused(read(C, path(Config, "sub/../../secret/key.txt"))), + {ok, <<"inside">>} = read(C, path(Config, "sub/../note.txt")), + alive(C), + stop(C). + +symlink_escape_is_refused(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + refused(read(C, path(Config, "escape"))), + alive(C), + stop(C). + +symlink_directory_prefix_is_refused(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + refused(read(C, path(Config, "outdir/key.txt"))), + alive(C), + stop(C). + +a_symlink_inside_a_grant_is_followed(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + {ok, <<"inside">>} = read(C, path(Config, "here")), + alive(C), + stop(C). + +a_symlink_cycle_is_refused_rather_than_followed(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + refused(read(C, path(Config, "loop"))), + alive(C), + stop(C). + +%% A file that is not there is not a capability answer. Distinguishing the two +%% is the whole reason refusals are not `FileNotFoundError'. +missing_file_inside_a_grant_is_not_a_refusal(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + {error, {'FileNotFoundError', _}} = read(C, path(Config, "missing.txt")), + alive(C), + stop(C). + +a_read_grant_yields_no_write_whatever_the_flags(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + refused(call(C, write_file, [path(Config, "new.txt"), <<"x">>])), + refused(call(C, append_file, [path(Config, "note.txt"), <<"x">>])), + refused(call(C, truncate_file, [path(Config, "note.txt")])), + refused(call(C, remove_file, [path(Config, "note.txt")])), + {ok, <<"inside">>} = file:read_file(path(Config, "note.txt")), + alive(C), + stop(C). + +a_write_grant_allows_create_and_unlink(Config) -> + C = ctx(Config, #{dirs => [{data(Config), write}]}), + {ok, <<"ok">>} = call(C, write_file, [path(Config, "new.txt"), <<"written">>]), + {ok, <<"written">>} = file:read_file(path(Config, "new.txt")), + {ok, <<"ok">>} = call(C, remove_file, [path(Config, "new.txt")]), + false = filelib:is_regular(path(Config, "new.txt")), + %% Still only this grant: the neighbouring directory is untouched. + refused(call(C, write_file, + [filename:join(?config(secret, Config), "x"), <<"x">>])), + alive(C), + stop(C). + +listing_is_granted_with_the_directory(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + {ok, Names} = call(C, list_dir, [data(Config)]), + true = lists:member(<<"note.txt">>, Names), + refused(call(C, list_dir, [?config(secret, Config)])), + alive(C), + stop(C). + +%% The interpreter's own path is granted, or nothing would import at all. +imports_still_work(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + {ok, <<"[1, 2]">>} = py_context:eval( + C, <<"__import__('json').dumps([1,2])">>), + {ok, 4} = py_context:eval(C, <<"len(__import__('base64').b64encode(b'ab'))">>), + stop(C). + +%%% ============================================================================ +%%% Network +%%% ============================================================================ + +a_network_and_a_port_range(Config) -> + {LSock, Port} = listener(), + C = ctx(Config, #{net => #{connect => [{tcp, <<"127.0.0.0/8">>, + {Port, Port}}]}}), + {ok, <<"connected">>} = call(C, connect, [<<"127.0.0.1">>, Port]), + {ok, _} = gen_tcp:accept(LSock, 2000), + ok = gen_tcp:close(LSock), + alive(C), + stop(C). + +%% Something is accepting on this port, and the answer is the same one a dead +%% port would get: nothing was attempted. +an_ungranted_port_is_refused_even_where_something_listens(Config) -> + {LSock, Port} = listener(), + Granted = free_port(), + C = ctx(Config, #{net => #{connect => [{tcp, <<"127.0.0.1">>, Granted}]}}), + refused(call(C, connect, [<<"127.0.0.1">>, Port])), + {error, timeout} = gen_tcp:accept(LSock, 300), + ok = gen_tcp:close(LSock), + alive(C), + stop(C). + +%% Binding claims a local address, which is what `listen' grants. A connect +%% grant for the same address does not carry it. +binding_is_checked_against_listen_not_connect(Config) -> + Port = free_port(), + C = ctx(Config, #{net => #{connect => [{tcp, <<"127.0.0.1">>, any}], + listen => [{tcp, <<"127.0.0.1">>, Port}]}}), + {ok, <<"bound">>} = call(C, bind, [<<"127.0.0.1">>, Port]), + refused(call(C, bind, [<<"127.0.0.1">>, free_port()])), + alive(C), + stop(C). + +resolution_is_its_own_capability(Config) -> + C = ctx(Config, #{net => #{connect => [{tcp, <<"127.0.0.1">>, any}]}}), + refused(call(C, resolve, [<<"localhost">>])), + stop(C), + C2 = ctx(Config, #{net => #{connect => [{tcp, <<"127.0.0.1">>, any}], + resolve => allow}}), + {ok, _} = call(C2, resolve, [<<"localhost">>]), + alive(C2), + stop(C2). + +%% An address learned by resolving carries no authority from having been +%% resolved: the connect is still checked, and still refused. +resolution_cannot_widen_a_grant(Config) -> + {LSock, Port} = listener(), + C = ctx(Config, #{net => #{connect => [{tcp, <<"10.0.0.0/8">>, any}], + resolve => allow}}), + {ok, Addrs} = call(C, resolve, [<<"localhost">>]), + true = lists:member(<<"127.0.0.1">>, Addrs), + refused(call(C, connect, [<<"127.0.0.1">>, Port])), + {error, timeout} = gen_tcp:accept(LSock, 300), + ok = gen_tcp:close(LSock), + stop(C). + +%% An inverse case: the documented sharp edge is that nothing is denied +%% implicitly, so adding a hidden deny list has to break the build and force +%% the guide to be corrected. +a_wildcard_grant_really_is_a_wildcard(Config) -> + {LSock, Port} = listener(), + C = ctx(Config, #{net => #{connect => [{tcp, <<"0.0.0.0/0">>, any}]}}), + {ok, <<"connected">>} = call(C, connect, [<<"127.0.0.1">>, Port]), + {ok, _} = gen_tcp:accept(LSock, 2000), + ok = gen_tcp:close(LSock), + stop(C). + +no_net_key_is_no_network(Config) -> + {LSock, Port} = listener(), + C = ctx(Config, #{dirs => [{data(Config), read}]}), + refused(call(C, connect, [<<"127.0.0.1">>, Port])), + C2 = ctx(Config, #{net => #{}}), + refused(call(C2, connect, [<<"127.0.0.1">>, Port])), + {error, timeout} = gen_tcp:accept(LSock, 300), + ok = gen_tcp:close(LSock), + stop(C), + stop(C2). + +%% A socket Erlang opened and handed over needs no grant: the child was given +%% the descriptor, which is the capability. +a_passed_fd_still_serves(Config) -> + {ok, LSock} = gen_tcp:listen(0, [binary, {ip, {127,0,0,1}}, + {active, false}, {backlog, 8}]), + {ok, Port} = inet:port(LSock), + {ok, Fd} = inet:getfd(LSock), + C = ctx(Config, #{dirs => [{data(Config), read}]}), + {ok, ChildFd} = py_context:pass_fd(C, Fd), + ok = py_context:start_loop(C), + {ok, _} = py_context:submit_await(C, ?TEST_MOD, serve, [ChildFd], #{}, 10000), + {ok, Sock} = gen_tcp:connect({127,0,0,1}, Port, [binary, {active, false}], 2000), + ok = gen_tcp:send(Sock, <<"ping">>), + {ok, <<"pong">>} = gen_tcp:recv(Sock, 4, 5000), + ok = gen_tcp:close(Sock), + ok = gen_tcp:close(LSock), + ok = py_context:stop_loop(C), + stop(C). + +%%% ============================================================================ +%%% Environment, processes, shared memory, lifecycle +%%% ============================================================================ + +env_is_what_was_granted_and_nothing_else(Config) -> + true = os:putenv("EP_CAPS_SECRET", "leaked"), + C = ctx(Config, #{env => #{<<"EP_CAPS_GRANTED">> => <<"yes">>}}), + {ok, <<"yes">>} = call(C, getenv, [<<"EP_CAPS_GRANTED">>]), + {ok, none} = call(C, getenv, [<<"EP_CAPS_SECRET">>]), + {ok, none} = call(C, getenv, [<<"HOME">>]), + stop(C), + %% Without a capability set the child inherits as it always did. + C2 = ctx(Config, no_caps), + {ok, <<"leaked">>} = call(C2, getenv, [<<"EP_CAPS_SECRET">>]), + stop(C2), + true = os:unsetenv("EP_CAPS_SECRET"). + +subprocess_is_refused(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + refused(call(C, run_subprocess, [])), + refused(py_context:eval(C, <<"__import__('os').fork()">>)), + alive(C), + stop(C). + +ctypes_is_refused(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + refused(py_context:eval(C, <<"__import__('ctypes').CDLL(None)">>)), + alive(C), + stop(C). + +%% The hook must not read anything the workload can assign to, or the policy +%% is off the moment code says so. +the_policy_cannot_be_switched_off_from_python(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + refused(read(C, "/etc/hosts")), + %% Every name the hook used to resolve when it ran, assigned at once. + ok = py_context:exec(C, <<"import _erlang_impl._caps as c\n" + "c._summary = None\n" + "c._local = type('x', (), {'busy': True})()\n" + "c._writes = lambda *a: False\n" + "c._PATH_EVENTS = {}\n" + "c._RESOLVE_EVENTS = frozenset()\n" + "c._SUBPROCESS_EVENTS = frozenset()\n" + "c._make_enforcer = None\n" + "c.os = None\n">>), + refused(read(C, "/etc/hosts")), + refused(call(C, run_subprocess, [])), + %% And the levers that used to let Python widen a grant are gone. + {ok, false} = py_context:eval( + C, <<"hasattr(__import__('_erlang_impl._caps', fromlist=['x'])," + " 'allow_path')">>), + {ok, false} = py_context:eval( + C, <<"hasattr(__import__('_erlang_impl._caps', fromlist=['x'])," + " '_walk')">>), + alive(C), + stop(C). + +%% The child shares the node's user and its parent is the BEAM, so an +%% unchecked signal is a way to take the node down. +signalling_another_process_is_refused(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + refused(py_context:eval(C, <<"__import__('os').kill(__import__('os')" + ".getppid(), 0)">>)), + refused(py_context:eval(C, <<"__import__('os').killpg(__import__('os')" + ".getpgrp(), 0)">>)), + refused(py_context:eval(C, <<"__import__('os').kill(1, 0)">>)), + %% Signalling itself is its own business. + {ok, none} = py_context:eval(C, <<"__import__('os').kill(__import__('os')" + ".getpid(), 0)">>), + alive(C), + stop(C). + +%% CPython raises no audit event for these, so they cannot be refused and +%% are taken away instead. Their absence is the assertion. +unaudited_creators_are_taken_away(Config) -> + C = ctx(Config, #{dirs => [{data(Config), write}]}), + {ok, false} = py_context:eval(C, <<"hasattr(__import__('os'), 'mkfifo')">>), + {ok, false} = py_context:eval(C, <<"hasattr(__import__('os'), 'mknod')">>), + {ok, false} = py_context:eval(C, <<"hasattr(__import__('posix'), 'mkfifo')">>), + {ok, false} = py_context:eval(C, <<"hasattr(__import__('posix'), 'mknod')">>), + alive(C), + stop(C). + +%% Gating only getaddrinfo would leave every other resolver as a way out, +%% and a name lookup is a message to whoever answers it. +every_resolver_is_gated_not_only_getaddrinfo(Config) -> + C = ctx(Config, #{net => #{connect => [{tcp, <<"127.0.0.1">>, any}]}}), + refused(call(C, resolve, [<<"localhost">>])), + refused(py_context:eval(C, <<"__import__('socket').gethostbyname('localhost')">>)), + refused(py_context:eval(C, <<"__import__('socket').gethostbyname_ex('localhost')">>)), + refused(py_context:eval(C, <<"__import__('socket').gethostbyaddr('127.0.0.1')">>)), + refused(py_context:eval(C, <<"__import__('socket').getnameinfo(('127.0.0.1',80),0)">>)), + refused(py_context:eval(C, <<"__import__('socket').gethostname()">>)), + alive(C), + stop(C). + +%% The `env' option adds to the environment and a grant says what the whole +%% of it is; the port keeps the last setting for a name, so taking both +%% would let the option quietly win. +the_env_option_cannot_widen_a_grant(_Config) -> + {error, {bad_caps, env_option_conflicts_with_caps_env}} = + py_context:new(#{mode => isolated, + caps => #{env => #{<<"A">> => <<"1">>}}, + env => #{<<"SECRET">> => <<"leaked">>}}), + {error, {bad_caps, env_option_conflicts_with_caps_env}} = + py_context:new(#{mode => isolated, caps => #{}, + env => #{<<"SECRET">> => <<"leaked">>}}), + ok. + +%% Shared memory does not combine with a capability set yet: a region +%% arrives as a path, and the only way to grant it would be to hand over the +%% directory holding every region this node owns. Passing the descriptor is +%% the fix, and it is not here yet, so the refusal has to be legible. +shared_memory_is_refused_under_a_capability_set(Config) -> + case py_shm:available() of + false -> + {skip, "iommap not available"}; + true -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + {ok, Shm} = py_shm:new(4096), + refused(py_context:call(C, ?TEST_MOD, shm_write, + [Shm, <<"payload">>], #{}, 10000)), + %% Erlang still has it, unharmed. + ok = py_shm:write(Shm, 0, <<"payload">>), + {ok, <<"payload">>} = py_shm:read(Shm, 0, 7), + alive(C), + ok = py_shm:close(Shm), + stop(C) + end. + +%% Path conversion happens before the re-entrancy guard, so a `__fspath__' +%% method is user code that runs with the hook live rather than a window in +%% which everything is allowed. +a_user_fspath_runs_enforced(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + {ok, false} = py_context:call(C, ?TEST_MOD, read_through_fspath, + [path(Config, "note.txt")], #{}, 10000), + alive(C), + stop(C). + +%% Reaching a Unix socket is talking to whatever is behind it, which a +%% directory grant says nothing about. +a_unix_socket_is_not_a_file(Config) -> + %% Its own short directory: an AF_UNIX path is capped near 104 bytes and + %% `connect' rejects a longer one before the hook ever sees it, which + %% would make this case pass for the wrong reason. + Dir = "/tmp/ep_caps_u" ++ integer_to_list(erlang:unique_integer([positive])), + ok = filelib:ensure_path(Dir), + C = ctx(Config, #{dirs => [{Dir, write}]}), + try + %% The directory is granted for writing, so the path is reachable as + %% a file; talking through it is not what that grant said. + {ok, <<"ok">>} = call(C, write_file, [Dir ++ "/plain", <<"x">>]), + refused(call(C, unix_connect, [Dir ++ "/sock"])), + refused(call(C, unix_bind, [Dir ++ "/mine.sock"])), + alive(C) + after + stop(C), + _ = file:del_dir_r(Dir) + end. + +%% Every route from a read grant to a write, including the ones CPython +%% does not announce by path. +a_read_grant_cannot_become_a_write(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + refused(call(C, write_file, [path(Config, "note.txt"), <<"x">>])), + refused(call(C, truncate_by_descriptor, [path(Config, "note.txt")])), + {ok, <<"inside">>} = file:read_file(path(Config, "note.txt")), + alive(C), + stop(C). + +caps_survive_a_child_restart(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}]}), + refused(read(C, "/etc/hosts")), + ok = py_context:kill(C), + {ok, <<"inside">>} = read(C, path(Config, "note.txt")), + refused(read(C, "/etc/hosts")), + refused(call(C, run_subprocess, [])), + stop(C). + +child_info_reports_the_grants(Config) -> + C = ctx(Config, #{dirs => [{data(Config), read}], + net => #{connect => [{tcp, <<"10.0.0.0/8">>, 443}]}}), + {ok, Info} = py_context:child_info(C), + Caps = maps:get(caps, Info), + Dirs = maps:get(<<"dirs">>, Caps), + Bin = list_to_binary(data(Config)), + true = lists:keymember(Bin, 1, Dirs), + #{<<"connect">> := [<<"tcp 10.0.0.0/8 443-443">>]} = maps:get(<<"net">>, Caps), + stop(C). + +caps_are_rejected_outside_isolated(_Config) -> + {error, {caps_requires_isolated, worker}} = + py_context:new(#{mode => worker, caps => #{}}), + {error, {caps_requires_isolated, owngil}} = + py_context:new(#{mode => owngil, caps => #{}}), + ok. + +%% A malformed rule is a configuration error and is reported as one here, +%% rather than met later as a refused connection: a capability set that +%% silently refuses everything looks exactly like one that works. +a_malformed_rule_is_a_configuration_error(_Config) -> + {error, {bad_caps, {net, {address, <<"nope">>}}}} = + py_context:new(#{mode => isolated, + caps => #{net => #{connect => [{tcp, <<"nope">>, 80}]}}}), + {error, {bad_caps, {dir_not_absolute, "rel"}}} = + py_context:new(#{mode => isolated, caps => #{dirs => [{"rel", read}]}}), + {error, {bad_caps, {net, {port, -1}}}} = + py_context:new(#{mode => isolated, + caps => #{net => #{listen => [{tcp, <<"127.0.0.1">>, -1}]}}}), + {error, {bad_caps, {unknown_keys, [bogus]}}} = + py_context:new(#{mode => isolated, caps => #{bogus => 1}}), + ok. + +no_caps_changes_nothing(Config) -> + C = ctx(Config, no_caps), + {ok, _} = read(C, "/etc/hosts"), + {ok, 4} = py_context:eval(C, <<"2+2">>), + stop(C). + +%%% ============================================================================ +%%% Helpers +%%% ============================================================================ + +ctx(Config, no_caps) -> + new(Config, #{}); +ctx(Config, Caps) -> + new(Config, #{caps => Caps}). + +new(Config, Extra) -> + TestDir = ?config(test_dir, Config), + Opts = maps:merge(#{mode => isolated, paths => [TestDir]}, Extra), + {ok, C} = py_context:new(Opts), + C. + +stop(C) -> + ok = py_context:stop(C). + +%% Every case ends with the context still working: a refusal must not have +%% left the child broken. +alive(C) -> + {ok, 4} = py_context:eval(C, <<"2+2">>). + +data(Config) -> ?config(data, Config). + +path(Config, Rel) -> filename:join(data(Config), Rel). + +read(C, Path) -> + call(C, read_file, [to_bin(Path)]). + +call(C, Fun, Args) -> + py_context:call(C, ?TEST_MOD, Fun, [to_bin(A) || A <- Args], #{}, 10000). + +%% A capability error, and never a missing-file error: the refusal says +%% nothing about whether the path exists. +refused({error, {'CapabilityError', _}}) -> ok; +refused(Other) -> ct:fail({expected_refusal, Other}). + +listener() -> + {ok, LSock} = gen_tcp:listen(0, [binary, {ip, {127,0,0,1}}, + {active, false}, {backlog, 8}]), + {ok, Port} = inet:port(LSock), + {LSock, Port}. + +free_port() -> + {ok, S} = gen_tcp:listen(0, [{ip, {127,0,0,1}}]), + {ok, P} = inet:port(S), + ok = gen_tcp:close(S), + P. + +to_bin(B) when is_binary(B) -> B; +to_bin(L) when is_list(L) -> list_to_binary(L); +to_bin(I) when is_integer(I) -> I; +to_bin(T) when is_tuple(T) -> T. + +flush() -> + receive _ -> flush() after 0 -> ok end. diff --git a/test/py_test_caps.py b/test/py_test_caps.py new file mode 100644 index 0000000..1d32e93 --- /dev/null +++ b/test/py_test_caps.py @@ -0,0 +1,171 @@ +"""Helpers for py_isolated_caps_SUITE. + +Each one is the smallest Python that performs one operation a capability set +either grants or refuses. They return a plain term on success and let the +exception through on refusal, so the suite sees `{error, {'CapabilityError', +Msg}}` and can tell it apart from a missing file. +""" + +import asyncio +import os +import socket +import subprocess + +_servers = {} + + +def _bytes(value): + """Erlang binaries arrive as `str`; `{bytes, B}` is what arrives as bytes.""" + return value.encode() if isinstance(value, str) else value + + +def _text(value): + return value.decode() if isinstance(value, bytes) else value + + +# --- filesystem ------------------------------------------------------------- + +def read_file(path): + with open(path, 'rb') as fh: + return fh.read() + + +def write_file(path, data): + with open(path, 'wb') as fh: + fh.write(_bytes(data)) + return 'ok' + + +def append_file(path, data): + with open(path, 'ab') as fh: + fh.write(_bytes(data)) + return 'ok' + + +def truncate_file(path): + os.truncate(path, 0) + return 'ok' + + +def remove_file(path): + os.remove(path) + return 'ok' + + +def list_dir(path): + return sorted(os.listdir(path)) + + +# --- network ---------------------------------------------------------------- + +def connect(host, port): + sock = socket.socket() + try: + sock.settimeout(5) + sock.connect((_text(host), port)) + return 'connected' + finally: + sock.close() + + +def bind(host, port): + sock = socket.socket() + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((_text(host), port)) + return 'bound' + finally: + sock.close() + + +def resolve(name): + name = _text(name) + return sorted({info[4][0] for info in socket.getaddrinfo(name, 80)}) + + +class _Echo(asyncio.Protocol): + def connection_made(self, transport): + self.transport = transport + + def data_received(self, data): + self.transport.write(b'pong') + + +async def serve(fd): + """Accept on a descriptor Erlang passed over and answer one request.""" + import erlang + _servers[fd] = await erlang.server.serve(fd, _Echo) + return 'serving' + + +# --- the rest --------------------------------------------------------------- + +def getenv(name): + name = _text(name) + value = os.environ.get(name) + return None if value is None else value + + +def run_subprocess(): + subprocess.run(['true'], check=False) + return 'ran' + + +def shm_write(region, data): + data = _bytes(data) + region[0:len(data)] = data + return 'ok' + + +class _Fspath: + """A path object whose conversion tries to read outside every grant.""" + + def __init__(self, path): + self.path = path + self.leaked = None + + def __fspath__(self): + try: + with open('/etc/hosts'): + self.leaked = True + except Exception: + self.leaked = False + return self.path + + +def read_through_fspath(path): + """Did the conversion get an unchecked read? It must not.""" + obj = _Fspath(_text(path)) + try: + with open(obj): + pass + except Exception: + pass + return obj.leaked + + +def truncate_by_descriptor(path): + fd = os.open(_text(path), os.O_RDONLY) + try: + os.truncate(fd, 0) + return 'truncated' + finally: + os.close(fd) + + +def unix_connect(path): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + sock.connect(_text(path)) + return 'connected' + finally: + sock.close() + + +def unix_bind(path): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + sock.bind(_text(path)) + return 'bound' + finally: + sock.close() From 581f3fadb1df8c6bac376099512fdbcb16a9a1d5 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Tue, 1 Sep 2026 11:49:50 +0200 Subject: [PATCH 15/15] Revert the capability set, and say what isolated mode bounds (#85) Enforcing what a child may reach from inside CPython does not work: an audit hook never sees a C extension, and the open event does not say which directory a relative path resolves against, so the containment could be walked around with documented calls. The tree returns to 5.0.0 and the guides say plainly that the child holds every authority the node's user holds, with the operating system mechanisms that do bound it. --- CHANGELOG.md | 25 - README.md | 11 +- docs/capabilities.md | 238 -------- docs/code-map.md | 2 - docs/decisions/0009-child-capabilities.md | 92 ---- docs/decisions/overview.md | 1 - docs/isolated.md | 25 +- docs/security.md | 36 +- priv/_erlang_impl/_caps.py | 571 -------------------- priv/_erlang_impl/_isolated.py | 11 +- priv/py_isolated_child.py | 31 +- priv/tests/test_caps.py | 253 --------- rebar.config | 6 +- src/erlang_python.app.src | 2 +- src/py_caps.erl | 263 --------- src/py_context.erl | 27 +- src/py_isolated.erl | 64 +-- test/coverage_audit.md | 1 - test/py_isolated_caps_SUITE.erl | 628 ---------------------- test/py_test_caps.py | 171 ------ 20 files changed, 49 insertions(+), 2409 deletions(-) delete mode 100644 docs/capabilities.md delete mode 100644 docs/decisions/0009-child-capabilities.md delete mode 100644 priv/_erlang_impl/_caps.py delete mode 100644 priv/tests/test_caps.py delete mode 100644 src/py_caps.erl delete mode 100644 test/py_isolated_caps_SUITE.erl delete mode 100644 test/py_test_caps.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d70309d..1514ebf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,30 +1,5 @@ # Changelog -## 5.1.0 (unreleased) - -### Added - -- **Capabilities for isolated children** - `py_context:new(#{mode => isolated, - caps => ...})` names the directories, environment variables and network - addresses a child may reach; anything not named is refused. Leaving a key - out grants none of it, and omitting `caps` leaves existing behaviour - unchanged. Paths are resolved a component at a time with `openat` and - `O_NOFOLLOW` from the descriptor of the grant, so `..`, absolute paths, - symlinks out of a grant and symlinked directory prefixes are all refused, - and refusals are `PermissionError` rather than `FileNotFoundError` so they - disclose nothing about what exists outside. Network rules name addresses - and never host names, resolution is its own capability covering every - resolver, and binding is checked against `listen` rather than `connect`. - Process creation, `ctypes`, signals to another process and Unix-socket - addresses are refused outright. The model and its vocabulary come from - erlang_wasm's WASI implementation. - - This is a cooperative policy over Python and not a boundary: it is built - on a CPython audit hook, so it binds Python and not a C extension, and it - covers only what CPython announces. `docs/capabilities.md` says what holds - and what does not. Shared memory and capability sets do not combine yet, - because a region reaches the child as a path. - ## 5.0.0 (2026-08-29) ### Added diff --git a/README.md b/README.md index 277e715..ca06e00 100644 --- a/README.md +++ b/README.md @@ -622,21 +622,14 @@ When creating Python contexts, you can choose the execution mode: %% segfault only takes the child down, rlimits bound memory and CPU. {ok, Ctx} = py_context:new(#{mode => isolated, kill_after => 1000, rlimits => #{as => 512 * 1024 * 1024}}). - -%% Name what the child may reach, and it reaches nothing else. -{ok, Ctx} = py_context:new(#{mode => isolated, - caps => #{dirs => [{"/srv/models", read}], - net => #{connect => [{tcp, <<"10.0.0.0/8">>, 5432}]}}}). ``` **Isolated mode** is the only mode with a hard bound: `py_context:interrupt/1` stops a blocking C call, and `SIGKILL` is the backstop. It costs a process per context (about 16 MB and 40 ms to start) and roughly twice the call latency. Bulk data crosses through shared memory (`py_shm`, with the optional -[iommap](https://hex.pm/packages/iommap) dependency), and the `caps` option -names the files, addresses and environment the child may reach, as a -cooperative policy over Python rather than a kernel boundary. See -[Isolated Contexts](docs/isolated.md) and [Capabilities](docs/capabilities.md). +[iommap](https://hex.pm/packages/iommap) dependency). See +[Isolated Contexts](docs/isolated.md). **Worker mode is recommended** because it works with any Python version and automatically benefits from free-threaded Python (3.13t+) when available. Each context owns a dedicated pthread, providing stable thread affinity for libraries with thread-local state (numpy, torch, tensorflow). diff --git a/docs/capabilities.md b/docs/capabilities.md deleted file mode 100644 index c1bb0a9..0000000 --- a/docs/capabilities.md +++ /dev/null @@ -1,238 +0,0 @@ -# Capabilities - -This guide covers `caps`, the option that says what an isolated child may -reach: which directories, which environment variables, which addresses. -Python that asks for anything else is refused. It is the WASI model, and -the vocabulary is the same as -[erlang_wasm](https://github.com/benoitc/erlang_wasm)'s, so a grant means -the same thing in both. - -**Read this before you rely on it.** `caps` is a cooperative policy over -Python, not a boundary. It stops code that is not trying to get out, and it -makes what a job may touch explicit and reviewable. It does not stop code -that is trying to get out: a C extension calling `open(2)` never reaches -the audit hook it is built on. Use it for code you partly trust; use the -process boundary in [Isolated Contexts](isolated.md) for the rest, and see -[what holds and what does not](#what-holds-and-what-does-not) for the -detail. - -Read [Isolated Contexts](isolated.md) first: `caps` only applies there. - -## Grant what it needs - -```erlang -{ok, Ctx} = py_context:new(#{ - mode => isolated, - caps => #{ - dirs => [{"/srv/models", read}, - {"/var/data/job42", write}], - env => #{<<"MODEL_DIR">> => <<"/srv/models">>}, - net => #{connect => [{tcp, <<"10.0.0.0/8">>, {5432, 5432}}], - resolve => deny} - }}), -{ok, _} = py_context:call(Ctx, scorer, run, [<<"job42">>]). -``` - -Inside the child, ordinary Python works inside the grants and fails outside -them: - -```python -open('/srv/models/weights.bin', 'rb') # granted -open('/var/data/job42/out.csv', 'w') # granted -open('/etc/passwd') # PermissionError -open('/srv/models/w', 'w') # PermissionError: read grant -``` - -Leave a key out and there is none of it. `caps => #{}` grants nothing but the -interpreter's own files, and no `caps` key at all is the behaviour you have -today: the child holds every authority the user running the node holds. - -## What each key grants - -| key | grants | leaving it out means | -| --- | --- | --- | -| `dirs` | directories, `read` or `write` | no filesystem beyond the interpreter's own | -| `env` | environment variables | zero variables, **not** the node's | -| `net` | sockets, to the addresses you name | no network at all | - -`read` covers opening, reading and listing. `write` adds creating, renaming, -unlinking and truncating. Rights apply to everything below the directory, so -a `read` grant yields no writable file however the code opens it. - -Always granted, because nothing works otherwise: the interpreter's own -`sys.path`, `sys.prefix` and `sys.base_prefix` for reading, so imports work, -and `/dev/null`, `/dev/zero`, `/dev/random`, `/dev/urandom`. WASI preopens -its sysroot for the same reason. Directories you list in the `paths` option -are granted for reading too, since that option tells the child to import -from them. - -The `env` option and `caps` cannot be used together. The option adds -variables and a grant says what the whole environment is, so taking both -would let the option quietly win; `caps.env` is the one that names an -environment, and using both is `{error, {bad_caps, -env_option_conflicts_with_caps_env}}`. - -## Name a network - -```erlang -net => #{connect => [{tcp, <<"10.0.0.0/8">>, {8000, 8099}}], - listen => [{tcp, <<"127.0.0.1">>, 8080}], - resolve => allow} -``` - -A rule is `{Proto, Addr, Port}`. `Proto` is `tcp` or `udp`, `Addr` is an -address tuple, a binary address or a binary CIDR, and `Port` is an integer, a -`{Lo, Hi}` range, or `any`. - -Four things to know before writing your first grant: - -- **`connect` and `listen` are separate**, and neither implies the other. - Binding claims a local address, which is what `listen` grants, so code - wanting a particular source port needs a `listen` rule for it. -- **You name addresses, never names.** There is no rule that says - `example.com`: a name would have to be resolved to be checked and resolved - again to be used, and the two answers can differ. `resolve` is its own - capability, off unless granted, and what it returns carries no authority. - Code may learn an address it cannot reach, and the connect is refused then. -- **`::ffff:127.0.0.1` is `127.0.0.1`.** IPv4-mapped addresses are folded - before matching, so the mapped notation cannot walk past an IPv4 rule. -- **Nothing is denied implicitly.** `<<"0.0.0.0/0">>` really does include - link-local and cloud metadata addresses. Name what you mean. - -A socket Erlang opened and handed over with `py_context:pass_fd/2` needs no -rule: the child was given the descriptor, and the descriptor is the -capability. That is how you serve on a port under a capability set. - -```erlang -{ok, LSock} = gen_tcp:listen(8080, [binary, {active, false}]), -{ok, Fd} = inet:getfd(LSock), -{ok, ChildFd} = py_context:pass_fd(Ctx, Fd), -ok = py_context:start_loop(Ctx), -{ok, _} = py_context:submit_await(Ctx, myapp, serve, [ChildFd]). -``` - -## Shared memory does not combine with this yet - -A `py_shm` region reaches the child as a path, so under a capability set it -is refused like any other ungranted path. Granting it would mean granting -the directory the node keeps every region in, which hands over every -region, and an open grant cannot prevent truncation anyway because -`file.truncate()` announces nothing. A truncated region is a `SIGBUS` in -the VM that mapped it, so the half-measure is worse than the refusal. - -The fix is to pass the region's descriptor rather than its name, with -`memfd_create` and `F_SEAL_SHRINK | F_SEAL_GROW | F_SEAL_SEAL` on Linux, -which keeps a writable mapping while making the object unresizable, and -cooperative-only handling where sealing does not exist. Until that lands, -use shared memory or a capability set, not both. - -## What is refused - -Every route below has its own test case in `py_isolated_caps_SUITE`. All are -refused with a `PermissionError` and never a `FileNotFoundError`, so the -error cannot be used to find out what exists outside a grant. - -| the code asks for | it gets | -| --- | --- | -| `note.txt`, `./note.txt`, `sub/deep.txt` | opened | -| `sub/../note.txt` | opened: it never leaves the grant | -| `../secret/key.txt` | refused | -| `/etc/passwd` | refused | -| `escape.txt`, a symlink out of the grant | refused | -| `outdir/key.txt`, through a symlinked directory | refused | -| `sub/../../secret/key.txt` | refused | -| a symlink cycle | refused | -| `missing.txt` inside a grant | `FileNotFoundError` | -| `subprocess.run`, `os.fork`, `os.exec*` | refused | -| `ctypes.CDLL` | refused | -| `os.kill` at another process, `os.killpg` | refused | -| `socket.gethostbyname` and every other resolver | refused unless `resolve` | -| `os.mkfifo`, `os.mknod` | not there: see below | -| a Unix-socket address, connect or bind | refused | - -Signalling is refused because the child usually shares the node's user and -its parent is the BEAM, so an unchecked `os.kill` is a way to take the node -down. Signalling itself is allowed. Every resolver is gated, not only -`getaddrinfo`: a name lookup is a message to whoever answers it, so gating -one of them would leave the rest as a way out. A Unix-socket address is -refused rather than checked as a file, because reaching one is talking to -whatever is behind it, which a directory grant says nothing about; a -descriptor Erlang passed over with `py_context:pass_fd/2` is unaffected. - -Subprocesses are refused outright: a capability set names what may be -reached, and another process is not something you granted. `ctypes` is -refused because it reaches libc directly, which would make every rule above -advisory. A library that needs `ctypes` cannot run under a capability set. - -## What holds and what does not - -Enforcement is a CPython audit hook, so the whole of what it can see is -what CPython announces, and everything below follows from that. - -**What holds.** Python code that asks for a path, an address or a process -outside the grants is refused, whether it asks through `open`, `os.open`, -`pathlib`, numpy or any other library, because the event is raised by the -interpreter rather than by the caller. Path containment is resolved by the -kernel one component at a time, so `..`, absolute paths, symlinks out of a -grant and symlinked directory prefixes are all refused rather than -lexically guessed at. Nothing on the decision path is reachable by name: -the grants, the tables and even the `os` functions the check uses are bound -into the hook's closure when it is installed, so assigning to this module -changes nothing. - -**What does not hold.** - -- A C extension calling `open(2)` or `connect(2)` never reaches an audit - hook. Neither does `file.truncate()` or `mmap.resize()`, which CPython - does not announce, so a writable descriptor can always shorten its own - file. That is part of what a `write` grant grants. -- A thread that replaces a path between the check and the kernel's own - resolution is not stopped. Do not point a `write` grant at a directory - another party writes to concurrently. -- `os.stat`, `os.access` and the rest of the calls that observe without - reaching are left alone, so what exists outside a grant stays visible - even though reading it does not. -- Closure state is a bar, not a wall. Python exposes its own object graph, - and code that goes looking can reach a hook's cells. - -**What would make it hold.** A kernel. Landlock on Linux takes the same -grant table and enforces it below the interpreter, which is the point at -which a C extension stops being an exception; moving the state and the hook -into the NIF would take the rest. Neither is here yet. Until then, the -boundary you have is the process: `rlimits`, `kill_after`, and the -supervision in [Isolated Contexts](isolated.md). - -## Cost - -The check is an audit hook, so it runs on every open, and paths are resolved -one component at a time against the descriptor of the grant. Measured on -macOS with Python 3.14, an open inside a grant costs about 11 microseconds -more than an unguarded one, and the cost grows with the depth of the path -below its grant. Grant close to what the code reads: `/srv/models` rather -than `/`. - -Nothing else changes. Calls, results, shared memory and interrupts are what -they were. - -## Check what a child got - -```erlang -{ok, Info} = py_context:child_info(Ctx), -maps:get(caps, Info). -``` - -```python -import erlang -erlang.caps() # None when no capability set was given -``` - -Both report the grants as the child holds them, including the automatic -ones, and `strict_paths` tells you whether the platform resolved paths -component by component or fell back to a lexical check. - -## See also - -- [Isolated Contexts](isolated.md) for the process boundary itself -- [Security](security.md) for what the embedded modes do instead -- [decision 0009](decisions/0009-child-capabilities.md) for why it is shaped - this way diff --git a/docs/code-map.md b/docs/code-map.md index fefea58..9da9c9b 100644 --- a/docs/code-map.md +++ b/docs/code-map.md @@ -29,7 +29,6 @@ exercised by suites). Guides are in `docs/`, suites in `test/`. Start with | `py_channel`, `py_byte_channel` | Term and byte queues between Erlang and Python coroutines (NIF resources) | live | channel | `py_channel_SUITE`, `py_byte_channel_SUITE` | | `py_buffer` | Native streaming input buffer; shared variant delegates to `py_shm` | live | buffer, isolated | `py_buffer_SUITE`, `py_isolated_buffer_SUITE` | | `py_shm` | Shared memory regions over iommap and the ring behind shared buffers | live | isolated | `py_isolated_shm_SUITE` | -| `py_caps` | The `caps' option: what an isolated child may reach, and its wire form | live | capabilities | `py_isolated_caps_SUITE` | | `py_import` | Registry of imports and `sys.path` entries applied to every interpreter | live | imports | `py_import_SUITE` | | `py_preload` | Code run once per interpreter at start | live | preload | `py_preload_SUITE` | | `py_state` | Shared key/value store visible from Python as `erlang.state_get/set/delete/keys` | live | README (shared state) | `py_state_SUITE` | @@ -81,7 +80,6 @@ loop, channels and servers. | `_erlang_impl/_etf.py` | Pure-Python ETF codec with the `py_convert.c` mapping | isolated child | | `_erlang_impl/_isolated.py` | Child runtime: socket frames, reader thread, re-entrant main loop, interrupt signal, asyncio loop, the `erlang` shim | isolated child | | `_erlang_impl/_shm.py` | `SharedMemory` and `SharedBuffer` wrappers over mmap | all | -| `_erlang_impl/_caps.py` | Capability enforcement in the child: path containment, address matching, the audit hook | isolated child | | `py_isolated_child.py` | Child launcher: rlimits, parent-death signal, cgroup join, connect | isolated child | | `test_erlang_loop.py`, `test_async_task.py`, `test_channel_ref.py`, `tests/` | Python-side tests of the loop, tasks and channels | test | diff --git a/docs/decisions/0009-child-capabilities.md b/docs/decisions/0009-child-capabilities.md deleted file mode 100644 index 682e024..0000000 --- a/docs/decisions/0009-child-capabilities.md +++ /dev/null @@ -1,92 +0,0 @@ -# 0009: An isolated child reaches only what it was granted - -Since 5.1.0. Code: `src/py_caps.erl`, `priv/_erlang_impl/_caps.py`, the -prologue of `priv/py_isolated_child.py`. - -## Situation - -Isolated mode bounded what Python could *consume*: memory, CPU, time, and a -crash. It bounded nothing it could *reach*. The child ran as the node's user -with the node's environment, could read and write every file that user -could, dial anywhere, spawn processes, and truncate the shared memory -regions it was handed, which turns the mapping the VM holds into a `SIGBUS`. -The audit hook the embedded modes install was never installed there. - -The first sketch was a deny list of dangerous operations. That is the wrong -shape: it enumerates what to stop, so it is wrong the moment something new -appears, and it says nothing about what a job is supposed to touch. - -## Decision - -Grant capabilities instead. `py_context:new(#{mode => isolated, caps => ...})` -names directories with an access level, environment variables, and network -rules; nothing else is reachable. No `caps` key leaves existing behaviour -alone, so this is additive. - -The model, the option shape, the refusal semantics and the test list are -taken from `erlang_wasm`'s WASI preview 1 implementation rather than -invented: `{Proto, Addr, Port}` rules, addresses and never host names, -resolution as its own capability, binding checked against `listen` and not -`connect`, IPv4-mapped folding, a malformed rule raising where the grant is -written. A grant means the same thing in both projects. - -Enforcement is an audit hook in the child, installed last in the prologue so -the runtime's own imports are outside the grants and everything after them -is inside. Paths are resolved a component at a time with `openat` and -`O_NOFOLLOW` from the descriptor of the grant, following symlinks by hand: -that is erlang_wasm's `native` backend, which needed a C NIF there because -Erlang has no `openat`, and needs none here because Python has one. - -Not chosen: routing every open through Erlang so `wasi_fs` could enforce it -directly. It would share one implementation and close the check-to-use -window, but it makes an `open` a socket round trip from arbitrary code, -including the child's own reader thread, and the deadlock surface is not -worth it at 25 microseconds a call. - -## What this is not - -Three review rounds all found the same shape of defect: a way for Python to -step around a check written in Python. Each was real and each was closed, -but the pattern is the point. An audit hook is a cooperative policy, and -the honest split is: - -* `caps` is for code you partly trust. It stops mistakes and casual misuse, - and it makes what a job may touch reviewable. -* The process, its rlimits and `kill_after` are what hold against code that - is trying to get out. -* Kernel enforcement (Landlock, seccomp, or the state and hook moved into - the NIF) is the point at which `caps` may be described as protection - against adversarial code. It is not there yet, and the guide says so - rather than implying otherwise. - -## Consequences - -- It is a policy over Python, not a boundary. A C extension calling - `open(2)`, or a thread swapping a path between the check and the kernel's - resolution, is not stopped. The guide says so in those words. -- The grants have to live in the hook's closure rather than in module - state, and nothing inside the interpreter may widen them. An earlier - version kept them in a module attribute and let `_shm` add region paths: - both were levers any Python could pull. Regions are now granted from - Erlang, as the directory holding them, opened and nothing more. -- Enforcement can only cover what CPython announces. Calls that create - something and raise no audit event are removed from `os` and `posix` - instead, and the ones that only observe are documented as visible. -- Nothing on the decision path may be reachable by name, including the - re-entrancy guard, the event tables and the `os` functions the check - itself calls. The guard is set around the containment walk alone, since a - wider one would leave a user `__fspath__` running with enforcement off. -- Shared memory does not combine with a capability set. A region arrives as - a path, granting the directory would hand over every region the node - owns, and an open-only grant cannot stop truncation because - `file.truncate()` announces nothing. Passing the descriptor, sealed on - Linux, is the way to make it work. -- `ctypes` must be refused, or every rule is advisory. Libraries that need - it cannot run under a capability set. -- The interpreter's own `sys.path` is granted automatically, or nothing - imports. A capability set therefore always grants reading the standard - library, as WASI's preopened sysroot does. -- An open inside a grant costs about 11 microseconds more, growing with path - depth, so grants should sit close to what is read. -- Landlock on Linux consumes the same table and would make it a boundary. - The table is shaped for that. diff --git a/docs/decisions/overview.md b/docs/decisions/overview.md index 9cc5e18..0d1f7eb 100644 --- a/docs/decisions/overview.md +++ b/docs/decisions/overview.md @@ -16,4 +16,3 @@ what was decided, what it costs, and where the code is. | [0006](0006-shared-memory-over-iommap.md) | Bulk data through iommap regions, handles as plain tuples | 5.0.0 | | [0007](0007-remove-legacy-execution-paths.md) | One execution path per mode; the legacy API is removed | 5.0.0 | | [0008](0008-pipe-io-rules.md) | Pipe I/O is non-blocking, deadlined and waited with poll | 3.1.0, 5.0.0 | -| [0009](0009-child-capabilities.md) | An isolated child reaches only what it was granted | 5.1.0 | diff --git a/docs/isolated.md b/docs/isolated.md index 87ac086..9705a70 100644 --- a/docs/isolated.md +++ b/docs/isolated.md @@ -312,19 +312,24 @@ file on purpose; sealing and syscall filtering are separate hardening work. event worker to step an idle loop). - `erlang.call` from inside a coroutine blocks the loop, as in the embedded modes; use `erlang.async_call`. -- Without `caps` the child holds every authority the user running the node - holds: it reads and writes what that user can, dials anywhere and can - spawn processes. - The child decodes terms with the same rules as the NIF, so atoms sent from Python are created in the VM's atom table. Do not let untrusted code mint unbounded distinct atoms. -- No syscall filtering: process isolation plus rlimits is the boundary. What - the child may *reach* (files, addresses, environment) is named with the - `caps` option, which is a cooperative policy over Python rather than a - kernel boundary: see [capabilities](capabilities.md). -- Shared memory and `caps` do not combine: a region reaches the child as a - path, and granting it would grant every region the node owns. A seccomp (Linux) or Capsicum (FreeBSD) - sandbox is a separate hardening step. +- **The child holds every authority the user running the node holds.** It + reads and writes every file that user can, opens any network connection, + spawns processes, and reads the environment the node was started with, + including any credentials in it. Isolated mode bounds what Python may + *consume*, not what it may *reach*. +- No syscall filtering. Process isolation plus rlimits is the boundary, and + it is a resilience boundary: a crash, a runaway loop or a memory blowup is + contained, a deliberate reach for something is not. Confining what the + child may reach needs a kernel sandbox (Landlock on Linux, Seatbelt on + macOS), which is not here; enforcing it inside CPython was tried and does + not work, because an audit hook cannot see a C extension calling `open(2)` + and cannot tell which directory a relative path resolves against. If you + need that confinement today, use what the operating system already gives + you around the whole node: a container, a jail, or a separate user with + file permissions to match. - Each call copies its arguments and result through the socket: a 1 MB binary round-trips in about 1.3 ms, 16 MB in about 27 ms (worker mode: 0.2 ms and 3 ms). For bulk data use shared memory (below). diff --git a/docs/security.md b/docs/security.md index 640c77f..858b718 100644 --- a/docs/security.md +++ b/docs/security.md @@ -160,21 +160,27 @@ child process: A crash kills only the child, `py_context:kill/1` is total, and rlimits or cgroups bound resources. See [Isolated Contexts](isolated.md). -That bounds what Python may consume. What it may *reach* is named with the -`caps` option: - -```erlang -{ok, Ctx} = py_context:new(#{mode => isolated, - caps => #{dirs => [{"/srv/models", read}], - net => #{connect => [{tcp, <<"10.0.0.0/8">>, 5432}]}}}). -``` - -Anything not named is refused. That is a cooperative policy over Python: it -binds Python code, not a C extension, because it is built on an audit hook. -[Capabilities](capabilities.md) says what holds and what does not. The child -is not sandboxed at the syscall level; that is a separate hardening step, -and the point at which capability sets would bind an adversary rather than -a mistake. +That is a boundary against Python *failing*, not against Python *reaching*. +The child runs as the same user as the node, so it can read and write +whatever that user can, open any connection, spawn processes, and read the +environment the node was started with. Nothing in this library confines it, +and confining it from inside the interpreter does not work: an audit hook +never sees a C extension calling `open(2)`, and the audit event for `open` +does not say which directory a relative path is resolved against, so a +check written in Python can be walked around with documented calls. + +If you need to bound what Python may reach, use the mechanisms the +operating system already has, around the node rather than inside it: + +- a container or jail, with the filesystem and network the job should see; +- a separate user, with file permissions to match, and `py_context:new/1` + started from a node running as that user; +- on Linux, a systemd unit with `ProtectSystem`, `ReadWritePaths` and + `IPAddressDeny`, which express the same intent and are enforced by the + kernel. + +Kernel sandboxing per context (Landlock, Seatbelt) would let this library +express it directly; it is not implemented. ## Signal Handling Note diff --git a/priv/_erlang_impl/_caps.py b/priv/_erlang_impl/_caps.py deleted file mode 100644 index 9032fe2..0000000 --- a/priv/_erlang_impl/_caps.py +++ /dev/null @@ -1,571 +0,0 @@ -# Copyright 2026 Benoit Chesneau -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""What an isolated child may reach, enforced over Python. - -Erlang names the directories, environment variables and addresses this -process may reach (`py_caps.erl`); this module refuses the rest. It is -installed once, in the child's prologue, before any user code runs. - -**This is a cooperative policy, not a boundary.** The difference decides -what you may use it for: - -* It stops code that is not trying to get out. Reading the wrong dataset, - writing outside a job's directory, calling home: those stop, and what a - job may touch becomes something you can read off the context options. -* It does not stop code that is trying to get out. A C extension calling - `open(2)` never reaches an audit hook. Neither does `file.truncate()`, - which CPython does not announce. And the grants live in a closure rather - than in a module attribute, which removes the one-line way to switch them - off but not the last one: Python exposes its own object graph, so code - that goes looking can reach a hook's cells. - -Hard isolation is a kernel's job and is not here yet. Landlock on Linux -takes the same grant table; moving the state and the hook into the NIF -would take the rest. Until then, treat `caps` as a policy for partially -trusted code, and the process boundary in `docs/isolated.md` as the thing -that holds against the rest. - -The audit surface, which is the whole of what is enforced: - -* **Refused by rule**: `open`, `os.listdir`, `os.scandir`, `os.mkdir`, - `os.rmdir`, `os.remove`, `os.rename`, `os.link`, `os.symlink`, - `os.truncate`, `os.chmod`, `os.chown`, `os.utime`, the `shutil.*` events, - `socket.connect`, `socket.bind`, `socket.sendto`, and every resolver. -* **Refused outright**: process creation, `ctypes`, signals to anything but - this process, and Unix-socket addresses. -* **Ignored, deliberately**: `os.stat`, `os.access`, `os.statvfs`, - `os.chdir` and the other calls that observe without reaching. What exists - outside a grant stays visible; reading it does not. -* **Removed, because CPython announces nothing**: `os.mkfifo` and - `os.mknod`, which create. `file.truncate()` and `mmap.resize()` announce - nothing either and cannot be removed, so a writable descriptor can always - shorten its own file. That is part of what a `write` grant grants, and it - is why there is no grant that means "open but do not resize". - -Path containment is erlang_wasm's native backend (`c_src/wasi_file_nif.c`), -which needs no C here because Python has `openat`: walk a component at a -time with ``O_NOFOLLOW`` from the descriptor of the grant, follow a symlink -by hand so it is worth what the same text written out is worth, and count -depth so ``..`` moves inside a grant but not out of it. - -Refusals are `PermissionError`, never `FileNotFoundError`, so a refusal -says nothing about what exists outside a grant. -""" - -import errno -import ipaddress -import os -import socket -import sys -import threading - -__all__ = ['install', 'installed', 'grants', 'CapabilityError'] - -# Eight, as Linux allows per path. It has to be a constant a cycle cannot -# outrun; a self-referential link is otherwise not an error but a hang. -_MAX_SYMLINKS = 8 - -_READ, _WRITE = 'read', 'write' - -# Devices that carry nothing about the host and whose absence breaks code in -# ways that are hard to read. Granted for reading with any capability set. -_ALWAYS_READ = ('/dev/null', '/dev/urandom', '/dev/random', '/dev/zero') - -# Process creation, always refused: a capability set names what may be -# reached, and another process is not something it was granted. -_SUBPROCESS_EVENTS = frozenset({ - 'subprocess.Popen', 'os.system', 'os.popen', 'os.fork', 'os.forkpty', - 'os.posix_spawn', 'os.posix_spawnp', -}) -_EXEC_PREFIXES = ('os.exec', 'os.spawn') - -# Signalling, refused except towards this process. The child usually shares -# the node's user and its parent is the BEAM, so an unchecked os.kill is a -# way to take the node down. -_SIGNAL_EVENTS = frozenset({'os.kill', 'os.killpg'}) - -# Resolution, granted by `resolve`. Every one of these reaches a resolver, -# so gating only getaddrinfo would leave the rest as a way out. -_RESOLVE_EVENTS = frozenset({ - 'socket.getaddrinfo', 'socket.gethostbyname', 'socket.gethostbyaddr', - 'socket.getnameinfo', 'socket.getservbyname', 'socket.gethostname', -}) - -# ctypes reaches libc directly, so leaving it open would make every rule -# here advisory. A library that needs it cannot run under a capability set. -_CTYPES_PREFIX = 'ctypes.' - -# Calls that create something and announce nothing, so they are taken away -# rather than refused. -_UNAUDITED_CREATORS = ('mkfifo', 'mknod') - -# Audit events that name a path, and what they need for it. -_PATH_EVENTS = { - 'os.listdir': _READ, - 'os.scandir': _READ, - 'os.mkdir': _WRITE, - 'os.rmdir': _WRITE, - 'os.remove': _WRITE, - 'os.rename': _WRITE, - 'os.link': _WRITE, - 'os.symlink': _WRITE, - 'os.truncate': _WRITE, - 'os.chmod': _WRITE, - 'os.chown': _WRITE, - 'os.utime': _WRITE, - 'shutil.copyfile': _WRITE, - 'shutil.copymode': _WRITE, - 'shutil.copystat': _WRITE, - 'shutil.copytree': _WRITE, - 'shutil.move': _WRITE, - 'shutil.rmtree': _WRITE, - 'shutil.unpack_archive': _WRITE, -} - -# Operations that act on a name and not on what it points at, so the last -# component is not followed: removing a symlink that leads out of a grant -# removes something inside the grant. -_NAME_EVENTS = frozenset({ - 'os.remove', 'os.rename', 'os.symlink', 'os.link', 'os.rmdir', 'os.mkdir', -}) - -# Events that name two paths; both ends are checked. -_TWO_PATH_EVENTS = frozenset({ - 'os.rename', 'os.link', 'os.symlink', 'shutil.copyfile', 'shutil.copymode', - 'shutil.copystat', 'shutil.copytree', 'shutil.move', -}) - -# A summary of what was granted, for `grants()`. It gates nothing: the -# grants themselves are reachable only from the hook's closure. -_summary = None - - -class CapabilityError(PermissionError): - """Raised for anything a capability set does not grant. - - A `PermissionError`, so code that already handles one keeps working, and - never a `FileNotFoundError`: whether a path outside a grant exists is - not something a refusal should disclose. - """ - - -class _Grant: - """One granted directory, held open. - - Opened once and kept: naming the directory by path on every check would - leave it to be resolved again each time, so replacing it would move the - grant. Anchored to the descriptor, a swapped *child* is what gets - refused. - - Both the path as granted and its resolved form are prefixes, because a - grant is often reached through a symlink (`/tmp` is `/private/tmp` on - macOS) and code inside the child will name it either way. - """ - - __slots__ = ('path', 'access', 'fd', 'prefixes') - - def __init__(self, path, access): - self.path = path - self.access = access - self.fd = os.open(path, os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0)) - real = os.path.realpath(path) - self.prefixes = (path,) if real == path else (path, real) - - def writable(self): - return self.access == _WRITE - - def remainder(self, path): - """The part of `path` below this grant, or None if it is not under it.""" - for prefix in self.prefixes: - if path == prefix: - return '' - if path.startswith(prefix.rstrip('/') + '/'): - return path[len(prefix.rstrip('/')) + 1:] - return None - - -class _State: - __slots__ = ('dirs', 'files', 'net', 'lexical') - - def __init__(self): - self.dirs = [] - # Exact paths that may be opened for reading: the devices above. - self.files = {} - self.net = None - self.lexical = False - - -class _Enforcer: - """What `_make_enforcer` returns: the hook, and its parts for tests.""" - - __slots__ = ('hook', 'walk', 'contained', 'check_path', 'writes') - - def __init__(self, **parts): - for name, part in parts.items(): - setattr(self, name, part) - - -def _make_enforcer(st): - """Build the audit hook over `st`. - - Everything on the decision path is bound here rather than looked up when - the hook runs, because a name resolved at call time is a name any Python - in this process can rebind: `_caps._writes = lambda *_: False` would - otherwise turn every open into a read. That includes `os` itself, so the - primitives are bound one by one, and the walk lives here rather than at - module level so no shared function object is left behind whose defaults - could be rewritten. - """ - # syscalls and constants, bound once - _open, _close, _readlink = os.open, os.close, os.readlink - _getcwd, _getpid = os.getcwd, os.getpid - _fspath, _fsdecode = os.fspath, os.fsdecode - _normpath, _abspath = os.path.normpath, os.path.abspath - _O_RDONLY, _O_NOFOLLOW = os.O_RDONLY, os.O_NOFOLLOW - _O_DIRECTORY = getattr(os, 'O_DIRECTORY', 0) - _O_WRITES = (os.O_WRONLY | os.O_RDWR | os.O_CREAT | os.O_TRUNC - | os.O_APPEND) - # A symlink met with O_NOFOLLOW: ELOOP on Linux and macOS, EMLINK on - # FreeBSD. - _ELOOP, _EMLINK = errno.ELOOP, errno.EMLINK - _ip_address = ipaddress.ip_address - _SOCK_DGRAM = socket.SOCK_DGRAM - _error = CapabilityError - READ, WRITE = _READ, _WRITE - - # the tables, copied so rebinding a module attribute changes nothing - path_events = dict(_PATH_EVENTS) - name_events = frozenset(_NAME_EVENTS) - two_path_events = frozenset(_TWO_PATH_EVENTS) - subprocess_events = frozenset(_SUBPROCESS_EVENTS) - exec_prefixes = tuple(_EXEC_PREFIXES) - ctypes_prefix = _CTYPES_PREFIX - signal_events = frozenset(_SIGNAL_EVENTS) - resolve_events = frozenset(_RESOLVE_EVENTS) - dirs, files, net, lexical = st.dirs, st.files, st.net, st.lexical - max_links = _MAX_SYMLINKS - - # The re-entrancy guard, created here so there is no module attribute to - # assign to. It is set around the walk and nothing else: the walk calls - # only `open`, `close` and `readlink` on its own account, so no user - # code can run while enforcement is off. - busy = threading.local() - - def walk(grant, rel, follow_last): - """Resolve `rel` beneath `grant`, a component at a time. - - Returns `(dirfd, component, owned)`; the caller closes `dirfd` when - `owned`. Raises `CapabilityError` if the path leaves the grant. - """ - dirfd, owned = grant.fd, False - depth = links = 0 - pending = [p for p in rel.split('/') if p not in ('', '.')] - busy.on = True - try: - while pending: - comp = pending.pop(0) - last = not pending - if comp == '..': - if depth == 0: - raise _error('path leaves the grant: %s' % rel) - depth -= 1 - nxt = _open('..', _O_RDONLY | _O_DIRECTORY, dir_fd=dirfd) - if owned: - _close(dirfd) - dirfd, owned = nxt, True - continue - if last and not follow_last: - return dirfd, comp, owned - try: - nxt = _open(comp, _O_RDONLY | _O_NOFOLLOW, dir_fd=dirfd) - except OSError as exc: - if exc.errno not in (_ELOOP, _EMLINK): - if last: - # Not there. That is not a containment answer; - # let the caller's own call raise its own error. - return dirfd, comp, owned - raise - if links >= max_links: - raise _error('too many symlinks: %s' % rel) from None - links += 1 - target = _readlink(comp, dir_fd=dirfd) - if target.startswith('/'): - # Refused rather than reinterpreted: resolving it - # against the grant would silently mean something - # other than what it says. - raise _error( - 'symlink leaves the grant: %s' % rel) from None - pending = [p for p in target.split('/') - if p not in ('', '.')] + pending - continue - if last: - _close(nxt) - return dirfd, comp, owned - if owned: - _close(dirfd) - dirfd, owned = nxt, True - depth += 1 - return dirfd, '.', owned - except BaseException: - if owned: - _close(dirfd) - raise - finally: - busy.on = False - - def contained(grant, path, need, follow): - """Is `path` inside `grant`, with `need` access? - - `path` keeps its `..` deliberately: collapsing them first is what - makes a check disagree with the kernel, because `link/..` is the - directory the link points into and not the one the link sits in. - """ - if need == WRITE and not grant.writable(): - return False - rel = grant.remainder(path) - if rel is None: - return False - if lexical: - depth = 0 - for comp in rel.split('/'): - if comp in ('', '.'): - continue - depth += -1 if comp == '..' else 1 - if depth < 0: - return False - return True - try: - dirfd, _comp, owned = walk(grant, rel, follow_last=follow) - except _error: - return False - except OSError: - # A component that is not there is not a containment answer: the - # path was inside the grant, it simply does not exist. - return True - if owned: - _close(dirfd) - return True - - def check_path(path, need, event, follow=True, opening=False): - # Conversion first and unguarded, because `__fspath__` is user code - # and has to run with the hook live, so its own opens are checked. - if not isinstance(path, (str, bytes)): - if isinstance(path, int): - # A descriptor. Reading through one is already granted, - # since opening it was checked; changing what it names is - # not, because a descriptor cannot be mapped back to a path - # portably enough to check. - if need == WRITE: - raise _error( - '%s: a capability set grants no change through a ' - 'descriptor' % event) - return - if not hasattr(path, '__fspath__'): - return - path = _fspath(path) - if isinstance(path, bytes): - try: - path = _fsdecode(path) - except ValueError: - raise _error('%s: undecodable path' % event) from None - absolute = path if path.startswith('/') \ - else _getcwd().rstrip('/') + '/' + path - if opening and need == READ \ - and files.get(_normpath(_abspath(absolute))) == READ: - return - for grant in dirs: - if contained(grant, absolute, need, follow): - return - raise _error('%s: %s is not granted for %s' % (event, path, need)) - - def writes(mode, flags): - """Does this open ask for anything but reading?""" - if isinstance(mode, str) and mode: - return any(c in mode for c in 'wax+') - if isinstance(flags, int): - return bool(flags & _O_WRITES) - return True - - def check_net(kind, event, args): - sock_obj = args[0] if args else None - address = args[1] if len(args) > 1 else None - if not isinstance(address, tuple) or len(address) < 2: - # A Unix socket names a path, but reaching one is talking to - # whatever is behind it, which is not something a directory - # grant says anything about. A descriptor Erlang passed over is - # unaffected: it is connected or listening already. - raise _error( - '%s: a capability set grants no unix-socket or unknown ' - 'address; a descriptor has to come from Erlang' % event) - if net is None: - raise _error('%s: no network was granted' % event) - host, port = address[0], address[1] - try: - addr = _ip_address(host) - except ValueError: - # A rule names addresses, so an unresolved name matches none. - raise _error('%s: %r is not granted' % (event, address)) from None - mapped = getattr(addr, 'ipv4_mapped', None) - if mapped is not None: - addr = mapped - proto = 'udp' if getattr(sock_obj, 'type', None) == _SOCK_DGRAM \ - else 'tcp' - for rule_proto, rule_net, lo, hi in net[kind]: - if rule_proto == proto and lo <= int(port) <= hi \ - and addr in rule_net: - return - raise _error('%s: %r is not granted' % (event, address)) - - def hook(event, args): - if getattr(busy, 'on', False): - return - if event in subprocess_events or event.startswith(exec_prefixes): - raise _error('%s: a capability set grants no subprocess' % event) - if event.startswith(ctypes_prefix): - raise _error( - '%s: a capability set grants no ctypes, which would reach ' - 'past every other rule' % event) - if event in signal_events: - if event == 'os.killpg' or not args or args[0] != _getpid(): - raise _error( - '%s: a capability set grants no signals to other ' - 'processes' % event) - return - if event in resolve_events: - if net is None or not net['resolve']: - raise _error( - '%s: resolution is its own capability and was not ' - 'granted' % event) - return - if event == 'open': - check_path(args[0], WRITE if writes(args[1], args[2]) else READ, - event, opening=True) - elif event in path_events: - need = path_events[event] - follow = event not in name_events - check_path(args[0], need, event, follow) - if event in two_path_events and len(args) > 1: - check_path(args[1], WRITE, event, follow) - elif event in ('socket.connect', 'socket.sendto'): - check_net('connect', event, args) - elif event == 'socket.bind': - check_net('listen', event, args) - - return _Enforcer(hook=hook, walk=walk, contained=contained, - check_path=check_path, writes=writes) - - -def _parse_net(net): - if not net: - return None - out = {'resolve': bool(net.get('resolve')), 'connect': [], 'listen': []} - for kind in ('connect', 'listen'): - for rule in net.get(kind) or (): - lo, hi = rule['ports'] - out[kind].append((rule['proto'], - ipaddress.ip_network(rule['cidr']), - int(lo), int(hi))) - return out - - -def _disarm_unaudited(): - """Take away the calls CPython does not announce. - - `os.mkfifo` and `os.mknod` create something and raise no audit event, so - a hook cannot refuse them. Removing the names is not a boundary either, - but it is the difference between a documented gap and an open one. - """ - import posix - for name in _UNAUDITED_CREATORS: - for module in (os, posix): - if hasattr(module, name): - try: - delattr(module, name) - except (AttributeError, TypeError): - pass - - -def install(caps): - """Install the capability set. Called once, before any user code.""" - global _summary - if _summary is not None: - return [] - st = _State() - problems = [] - st.lexical = os.open not in os.supports_dir_fd - if st.lexical: - problems.append('this platform has no openat: paths are checked ' - 'lexically and a symlink out of a grant is not seen') - - # Everything the interpreter itself reads. Without these nothing - # imports, which is why WASI preopens its sysroot too. - auto = [(p, _READ) for p in [sys.prefix, sys.base_prefix] + list(sys.path) - if p] - named = [(d['path'], d['access']) for d in caps.get('dirs') or ()] - - seen = set() - for path, access in auto + named: - real = os.path.normpath(os.path.abspath(path)) - if (real, access) in seen: - continue - seen.add((real, access)) - try: - st.dirs.append(_Grant(real, access)) - except OSError as exc: - if access != _READ: - problems.append('cannot open granted directory %s: %s' - % (path, exc)) - for dev in _ALWAYS_READ: - st.files[dev] = _READ - st.net = _parse_net(caps.get('net')) - - _disarm_unaudited() - _summary = { - 'dirs': tuple((g.path, g.access) for g in st.dirs), - 'net': None if st.net is None else { - 'connect': tuple(_rule_text(r) for r in st.net['connect']), - 'listen': tuple(_rule_text(r) for r in st.net['listen']), - 'resolve': st.net['resolve'], - }, - 'strict_paths': not st.lexical, - } - sys.addaudithook(_make_enforcer(st).hook) - return problems - - -def installed(): - return _summary is not None - - -def grants(): - """What was granted, for `child_info` and `erlang.caps()`. - - A fresh copy each time: this is something to look at, never something - the hook consults. - """ - if _summary is None: - return None - net = _summary['net'] - return { - 'dirs': [tuple(d) for d in _summary['dirs']], - 'net': None if net is None else { - 'connect': list(net['connect']), - 'listen': list(net['listen']), - 'resolve': net['resolve'], - }, - 'strict_paths': _summary['strict_paths'], - } - - -def _rule_text(rule): - proto, net, lo, hi = rule - return '%s %s %d-%d' % (proto, net, lo, hi) diff --git a/priv/_erlang_impl/_isolated.py b/priv/_erlang_impl/_isolated.py index d6b215f..04f6a97 100644 --- a/priv/_erlang_impl/_isolated.py +++ b/priv/_erlang_impl/_isolated.py @@ -751,12 +751,6 @@ def atom(name): def is_isolated(): return True - def caps(): - """What this child was granted, or None when it holds every - authority the user it runs as holds.""" - from . import _caps - return _caps.grants() - def run(main, *, debug=None): loop = rt.get_loop() if debug is not None: @@ -812,7 +806,7 @@ def __getattr__(name): call=call, async_call=async_call, send=send, whereis=whereis, self=self_, atom=atom, Atom=Atom, Pid=Pid, Ref=Ref, Port=Port, ProcessError=ProcessError, SuspensionRequired=SuspensionRequired, - Function=Function, is_isolated=is_isolated, caps=caps, run=run, + Function=Function, is_isolated=is_isolated, run=run, SharedMemory=SharedMemory, SharedBuffer=SharedBuffer, new_event_loop=new_event_loop, get_event_loop_policy=get_event_loop_policy, install=install, spawn_task=spawn_task, sleep=sleep, log=log, @@ -829,8 +823,7 @@ def __getattr__(name): ByteChannel=_not_supported('erlang.ByteChannel'), __all__=['call', 'async_call', 'send', 'whereis', 'self', 'atom', 'Atom', 'Pid', 'Ref', 'ProcessError', 'SuspensionRequired', - 'run', 'sleep', 'spawn_task', 'server', 'is_isolated', - 'caps'], + 'run', 'sleep', 'spawn_task', 'server', 'is_isolated'], ) mod.__dict__.update(ns) sys.modules['erlang'] = mod diff --git a/priv/py_isolated_child.py b/priv/py_isolated_child.py index 508ccd4..006e074 100644 --- a/priv/py_isolated_child.py +++ b/priv/py_isolated_child.py @@ -38,7 +38,7 @@ def _die(reason): def _parse_args(argv): if len(argv) < 2: _die('usage: py_isolated_child.py SOCKET_PATH [options]') - opts = {'socket': argv[1], 'rlimits': {}, 'cgroup': None, 'caps': None} + opts = {'socket': argv[1], 'rlimits': {}, 'cgroup': None} i = 2 while i < len(argv): flag = argv[i] @@ -48,13 +48,6 @@ def _parse_args(argv): elif flag == '--cgroup': opts['cgroup'] = argv[i + 1] i += 2 - elif flag == '--caps-json': - import json - try: - opts['caps'] = json.loads(argv[i + 1]) - except ValueError as exc: - _die('bad --caps-json: %s' % exc) - i += 2 else: _die('unknown option %s' % flag) return opts @@ -174,15 +167,6 @@ def _connect(path): return sock -def _caps_summary(): - """What was granted, so `py_context:child_info/1` can report it.""" - try: - from _erlang_impl import _caps - return _caps.grants() - except Exception: - return None - - def main(argv): opts = _parse_args(argv) _arm_parent_death() @@ -207,20 +191,10 @@ def main(argv): if _AS_VIA_WATCHDOG and 'as' in opts['rlimits']: _start_memory_watchdog(opts['rlimits']['as'], runtime) - # Last thing before the parent is told this child is ready, so the - # runtime's own imports are not subject to the grants and everything - # that runs afterwards is: the registered imports, the preload, and - # every request. - caps_errors = [] - if opts['caps'] is not None: - from _erlang_impl import _caps - caps_errors = _caps.install(opts['caps']) - - if rlimit_errors or cgroup_error or caps_errors: + if rlimit_errors or cgroup_error: problems = [(Atom('rlimit'), Atom(k), msg) for k, msg in rlimit_errors] if cgroup_error: problems.append((Atom('cgroup'), cgroup_error)) - problems += [(Atom('caps'), msg) for msg in caps_errors] try: runtime.event((Atom('startup_error'), problems)) finally: @@ -231,7 +205,6 @@ def main(argv): Atom('python_version'): '%d.%d.%d' % sys.version_info[:3], Atom('executable'): sys.executable, Atom('platform'): sys.platform, - Atom('caps'): _caps_summary(), } runtime.event((Atom('ready'), info)) diff --git a/priv/tests/test_caps.py b/priv/tests/test_caps.py deleted file mode 100644 index 31bd851..0000000 --- a/priv/tests/test_caps.py +++ /dev/null @@ -1,253 +0,0 @@ -"""Unit tests for the capability resolver and the address matcher. - -These run without a VM, so the containment rules can be read and changed -without a Common Test round trip. `py_isolated_caps_SUITE` covers the same -ground through a real child. - - cd priv && python3 -m unittest tests.test_caps -""" - -import os -import shutil -import sys -import tempfile -import unittest - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from _erlang_impl import _caps # noqa: E402 - - -class PathContainment(unittest.TestCase): - """The tree is the one `wasi_SUITE` uses, so the cases line up.""" - - @classmethod - def setUpClass(cls): - cls.root = tempfile.mkdtemp() - cls.data = os.path.join(cls.root, 'data') - cls.secret = os.path.join(cls.root, 'secret') - os.makedirs(os.path.join(cls.data, 'sub')) - os.makedirs(cls.secret) - _write(os.path.join(cls.data, 'note.txt'), 'inside') - _write(os.path.join(cls.data, 'sub', 'deep.txt'), 'deep') - _write(os.path.join(cls.secret, 'key.txt'), 'secret') - os.symlink(os.path.join(cls.secret, 'key.txt'), - os.path.join(cls.data, 'escape')) - os.symlink(cls.secret, os.path.join(cls.data, 'outdir')) - os.symlink('note.txt', os.path.join(cls.data, 'here')) - os.symlink('loop', os.path.join(cls.data, 'loop')) - cls.grant = _caps._Grant(cls.data, 'write') - # The enforcement lives in the factory's closure, so that is what a - # test drives; there is nothing at module level to call. - state = _caps._State() - state.dirs = [cls.grant] - cls.enf = _caps._make_enforcer(state) - - @classmethod - def tearDownClass(cls): - os.close(cls.grant.fd) - shutil.rmtree(cls.root, ignore_errors=True) - - def reaches(self, rel, follow=True): - """Does `rel` resolve inside the grant?""" - try: - fd, _comp, owned = self.enf.walk(self.grant, rel, follow_last=follow) - except _caps.CapabilityError: - return False - except OSError: - return True # inside the grant, simply not there - if owned: - os.close(fd) - return True - - def test_a_plain_name_resolves(self): - self.assertTrue(self.reaches('note.txt')) - self.assertTrue(self.reaches('./note.txt')) - self.assertTrue(self.reaches('sub/deep.txt')) - - def test_parent_traversal_is_refused(self): - self.assertFalse(self.reaches('../secret/key.txt')) - self.assertFalse(self.reaches('..')) - - def test_partial_traversal_is_refused(self): - self.assertFalse(self.reaches('sub/../../secret/key.txt')) - # And the half of it that is legal really is legal, or the assertion - # above would hold just as well with `..` refused outright. - self.assertTrue(self.reaches('sub/../note.txt')) - - def test_a_symlink_out_is_refused(self): - self.assertFalse(self.reaches('escape')) - - def test_a_symlinked_directory_prefix_is_refused(self): - self.assertFalse(self.reaches('outdir/key.txt')) - - def test_a_symlink_inside_is_followed(self): - self.assertTrue(self.reaches('here')) - - def test_a_cycle_is_refused_rather_than_followed(self): - self.assertFalse(self.reaches('loop')) - - def test_a_name_is_not_followed_when_the_caller_names_it(self): - # Removing a link that leads out of the grant removes something - # inside the grant, so naming it is allowed where following is not. - self.assertFalse(self.reaches('escape', follow=True)) - self.assertTrue(self.reaches('escape', follow=False)) - - def test_a_missing_name_is_not_a_containment_answer(self): - self.assertTrue(self.reaches('missing.txt')) - self.assertFalse(self.reaches('../missing.txt')) - - def test_the_walk_leaks_no_descriptors(self): - before = _lowest_free_fd() - for _ in range(200): - for rel in ('note.txt', 'escape', 'loop', 'sub/../note.txt', - '../secret/key.txt', 'outdir/key.txt'): - self.reaches(rel) - self.assertLessEqual(_lowest_free_fd(), before + 1) - - -class ModuleState(unittest.TestCase): - """The grants must not be reachable through this module. - - An enforcement decision that reads a module attribute is one any Python - in the process can switch off by assigning to it. - """ - - def test_no_lever_is_exported(self): - self.assertFalse(hasattr(_caps, 'allow_path')) - - def test_nothing_on_the_decision_path_lives_at_module_level(self): - # A name the hook resolves when it runs is a name any Python in the - # process can rebind, so none of these may exist here. - for name in ('_walk', '_contained', '_check_path', '_writes', - '_net_allows', '_local', '_state'): - self.assertFalse(hasattr(_caps, name), name) - - def test_the_decision_path_loads_no_module_global(self): - # Not `co_names`, which also lists attribute names: what matters is - # what the code actually loads from the module's namespace, because - # that is what an assignment to this module would change. - import dis - enf = _caps._make_enforcer(_caps._State()) - for part in ('hook', 'walk', 'contained', 'check_path', 'writes'): - code = getattr(enf, part).__code__ - loaded = {i.argval for i in dis.get_instructions(code) - if i.opname == 'LOAD_GLOBAL'} - self.assertEqual(loaded & set(vars(_caps)), set(), part) - - def test_grants_returns_a_copy(self): - # What `grants()` hands back is something to look at, so mutating it - # must not reach anything. - before = _caps.grants() - if before is None: - self.skipTest('no capability set installed in this process') - before['dirs'].append(('/etc', 'write')) - self.assertNotIn(('/etc', 'write'), _caps.grants()['dirs']) - - -class WriteIntent(unittest.TestCase): - """Which opens need a write grant.""" - - def setUp(self): - self.writes = _caps._make_enforcer(_caps._State()).writes - - def test_modes(self): - for mode in ('w', 'a', 'x', 'r+', 'w+b', 'rb+'): - self.assertTrue(self.writes(mode, 0), mode) - for mode in ('r', 'rb', 'rt'): - self.assertFalse(self.writes(mode, 0), mode) - - def test_flags(self): - for flag in (os.O_WRONLY, os.O_RDWR, os.O_CREAT, os.O_TRUNC, - os.O_APPEND, os.O_RDONLY | os.O_CREAT): - self.assertTrue(self.writes(None, flag), flag) - self.assertFalse(self.writes(None, os.O_RDONLY)) - - def test_an_unreadable_intent_is_taken_as_a_write(self): - self.assertTrue(self.writes(None, None)) - - -class AddressMatching(unittest.TestCase): - """The rules erlang_wasm's `wasi_net_SUITE` checks, matched here.""" - - @staticmethod - def grant(connect=(), listen=(), resolve=False): - return _caps._parse_net({'connect': list(connect), - 'listen': list(listen), - 'resolve': resolve}) - - @staticmethod - def rule(cidr, lo, hi, proto='tcp'): - return {'proto': proto, 'cidr': cidr, 'ports': [lo, hi]} - - def allows(self, net, addr, port, kind='connect', dgram=False): - # The matcher lives in the enforcer, so a test builds one over the - # grant it wants rather than poking module state. - state = _caps._State() - state.net = net - enf = _caps._make_enforcer(state) - event = 'socket.bind' if kind == 'listen' else 'socket.connect' - try: - enf.hook(event, (_FakeSocket(dgram), (addr, port))) - return True - except _caps.CapabilityError: - return False - - def test_a_network_and_a_port_range(self): - g = self.grant(connect=[self.rule('10.0.0.0/8', 8000, 8099)]) - self.assertTrue(self.allows(g, '10.1.2.3', 8000)) - self.assertTrue(self.allows(g, '10.255.255.255', 8099)) - self.assertFalse(self.allows(g, '11.0.0.1', 8000)) - self.assertFalse(self.allows(g, '10.1.2.3', 8100)) - self.assertFalse(self.allows(g, '10.1.2.3', 7999)) - - def test_ipv4_mapped_ipv6_is_the_same_address(self): - # A matcher comparing text would let this past a v4 rule. - g = self.grant(connect=[self.rule('127.0.0.0/8', 80, 80)]) - self.assertTrue(self.allows(g, '::ffff:127.0.0.1', 80)) - self.assertFalse(self.allows(g, '::1', 80)) - - def test_udp_and_tcp_are_separate(self): - g = self.grant(connect=[self.rule('127.0.0.1/32', 53, 53, 'udp')]) - self.assertTrue(self.allows(g, '127.0.0.1', 53, dgram=True)) - self.assertFalse(self.allows(g, '127.0.0.1', 53)) - - def test_connect_and_listen_are_separate(self): - g = self.grant(connect=[self.rule('127.0.0.1/32', 80, 80)]) - self.assertTrue(self.allows(g, '127.0.0.1', 80, kind='connect')) - self.assertFalse(self.allows(g, '127.0.0.1', 80, kind='listen')) - - def test_a_wildcard_really_is_a_wildcard(self): - # Nothing is denied implicitly: 0.0.0.0/0 includes the link-local - # and cloud metadata addresses, and this does not second-guess it. - g = self.grant(connect=[self.rule('0.0.0.0/0', 0, 65535)]) - self.assertTrue(self.allows(g, '169.254.169.254', 80)) - - def test_a_name_matches_nothing(self): - # A rule names addresses, so an unresolved name cannot match one. - g = self.grant(connect=[self.rule('0.0.0.0/0', 0, 65535)]) - self.assertFalse(self.allows(g, 'example.com', 80)) - - def test_no_grant_allows_nothing(self): - self.assertFalse(self.allows(None, '127.0.0.1', 80)) - - -class _FakeSocket: - def __init__(self, dgram): - import socket - self.type = socket.SOCK_DGRAM if dgram else socket.SOCK_STREAM - - -def _write(path, text): - with open(path, 'w') as fh: - fh.write(text) - - -def _lowest_free_fd(): - fd = os.dup(0) - os.close(fd) - return fd - - -if __name__ == '__main__': - unittest.main() diff --git a/rebar.config b/rebar.config index 5b7bc6b..ddae020 100644 --- a/rebar.config +++ b/rebar.config @@ -71,7 +71,6 @@ <<"docs/asyncio.md">>, <<"docs/workers.md">>, <<"docs/isolated.md">>, - <<"docs/capabilities.md">>, <<"docs/reactor.md">>, <<"docs/process-bound-envs.md">>, <<"docs/security.md">>, @@ -92,7 +91,6 @@ <<"docs/decisions/0006-shared-memory-over-iommap.md">>, <<"docs/decisions/0007-remove-legacy-execution-paths.md">>, <<"docs/decisions/0008-pipe-io-rules.md">>, - <<"docs/decisions/0009-child-capabilities.md">>, <<"docs/preload.md">>, <<"docs/owngil_internals.md">>, <<"docs/event_loop_architecture.md">> @@ -121,7 +119,6 @@ <<"docs/asyncio.md">>, <<"docs/workers.md">>, <<"docs/isolated.md">>, - <<"docs/capabilities.md">>, <<"docs/reactor.md">>, <<"docs/process-bound-envs.md">>, <<"docs/security.md">>, @@ -148,8 +145,7 @@ <<"docs/decisions/0005-py-isolated-gen-statem.md">>, <<"docs/decisions/0006-shared-memory-over-iommap.md">>, <<"docs/decisions/0007-remove-legacy-execution-paths.md">>, - <<"docs/decisions/0008-pipe-io-rules.md">>, - <<"docs/decisions/0009-child-capabilities.md">> + <<"docs/decisions/0008-pipe-io-rules.md">> ]} ]} ]}. diff --git a/src/erlang_python.app.src b/src/erlang_python.app.src index 57c8879..c478e29 100644 --- a/src/erlang_python.app.src +++ b/src/erlang_python.app.src @@ -1,6 +1,6 @@ {application, erlang_python, [ {description, "Execute Python applications from Erlang using dirty NIFs"}, - {vsn, "5.1.0"}, + {vsn, "5.0.0"}, {registered, []}, {mod, {erlang_python_app, []}}, {applications, [ diff --git a/src/py_caps.erl b/src/py_caps.erl deleted file mode 100644 index 0e8ab2c..0000000 --- a/src/py_caps.erl +++ /dev/null @@ -1,263 +0,0 @@ -%% Copyright 2026 Benoit Chesneau -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% http://www.apache.org/licenses/LICENSE-2.0 -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. - -%%% @doc The `caps' option: what an isolated child may reach. -%%% -%%% A capability grant names directories, environment variables and network -%%% addresses. Anything not named is not reachable. This module reads the -%%% option, refuses what it cannot make sense of, and renders the result as -%%% the JSON the child parses before it runs any user code. -%%% -%%% ``` -%%% #{dirs => [{"/srv/models", read}, {"/var/data/job42", write}], -%%% env => #{<<"MODEL_DIR">> => <<"/srv/models">>}, -%%% net => #{connect => [{tcp, <<"10.0.0.0/8">>, {5432, 5432}}], -%%% listen => [{tcp, <<"127.0.0.1">>, 8080}], -%%% resolve => deny}} -%%% ''' -%%% -%%% A rule is `{Proto, Addr, Port}': `Proto' is `tcp' or `udp', `Addr' is an -%%% address tuple, a binary address or a binary CIDR, and `Port' is an integer, -%%% `{Lo, Hi}' or `any'. Rules name addresses and never host names: a name -%%% would have to be resolved to be checked and resolved again to be used, and -%%% the two answers can differ. Resolution is its own capability and what it -%%% returns carries no authority. -%%% -%%% Checked here rather than in the child, so a typo is a `{bad_caps, _}' -%%% error from `py_context:new/1' rather than a connection refused much later. -%%% A sandbox that silently refuses everything looks exactly like one that -%%% works. -%%% -%%% The rule shape, the IPv4-mapped folding and the masking are taken from -%%% `wasi_net.erl' in erlang_wasm, so a grant means the same thing in both. -%%% -%%% @private -%%% -%%% Shared memory is not granted here and does not work under a capability -%%% set: a region arrives as a path, and granting the directory holding them -%%% would hand over every region this node owns. The way to make it work is -%%% to pass the region's descriptor rather than its name; see -%%% `docs/capabilities.md'. -%%% -%%% Owns: the meaning of the `caps' option and its wire form. -%%% Talks to: `py_context' (validation at `new/1'), `py_isolated' (argv). -%%% Never: enforces anything; the child does that, in -%%% `priv/_erlang_impl/_caps.py'. -%%% @end --module(py_caps). - --export([ - validate/1, - to_json/1 -]). - --export_type([caps/0, access/0]). - --type access() :: read | write. --type rule() :: {tcp | udp, {inet:ip_address(), 0..128}, {0..65535, 0..65535}}. --type net() :: none | #{connect := [rule()], listen := [rule()], - resolve := boolean()}. --type caps() :: #{dirs := [{binary(), access()}], - env := #{binary() => binary()}, - net := net()}. - -%%% ============================================================================ -%%% API -%%% ============================================================================ - -%% @doc Read a `caps' option into the form the child is given. -%% -%% `{error, {bad_caps, Detail}}' names the part that could not be read. --spec validate(term()) -> {ok, caps()} | {error, {bad_caps, term()}}. -validate(Map) when is_map(Map) -> - try - Known = [dirs, env, net], - case maps:keys(maps:without(Known, Map)) of - [] -> ok; - Extra -> throw({unknown_keys, Extra}) - end, - {ok, #{dirs => dirs(maps:get(dirs, Map, [])), - env => env(maps:get(env, Map, #{})), - net => net(maps:get(net, Map, none))}} - catch - throw:Detail -> {error, {bad_caps, Detail}} - end; -validate(Other) -> - {error, {bad_caps, Other}}. - -%% @doc Render a validated grant as the JSON passed to the child in argv. --spec to_json(caps()) -> binary(). -to_json(#{dirs := Dirs, env := Env, net := Net}) -> - iolist_to_binary(json:encode( - #{<<"dirs">> => [#{<<"path">> => P, <<"access">> => atom_to_binary(A)} - || {P, A} <- Dirs], - <<"env">> => Env, - <<"net">> => net_json(Net)})). - -%%% ============================================================================ -%%% Directories and environment -%%% ============================================================================ - -dirs(L) when is_list(L) -> - [dir(D) || D <- L]; -dirs(Other) -> - throw({dirs, Other}). - -%% An absolute path, so that what a grant covers does not depend on the -%% working directory of whoever wrote it. -dir({Path, Access}) when Access =:= read; Access =:= write -> - case to_bin(Path) of - <<"/", _/binary>> = Bin -> {Bin, Access}; - _ -> throw({dir_not_absolute, Path}) - end; -dir(Other) -> - throw({dir, Other}). - -env(Map) when is_map(Map) -> - maps:from_list([{to_bin(K), to_bin(V)} || {K, V} <- maps:to_list(Map)]); -env(Other) -> - throw({env, Other}). - -%%% ============================================================================ -%%% Network grant -%%% -%%% From wasi_net.erl (erlang_wasm), which parses the same rules for a WASM -%%% guest. Kept in step with it deliberately: a grant should mean one thing. -%%% ============================================================================ - -net(none) -> none; -net(undefined) -> none; -net(Map) when is_map(Map) -> - case maps:keys(maps:without([connect, listen, resolve], Map)) of - [] -> ok; - Extra -> throw({net, {unknown_keys, Extra}}) - end, - #{connect => rules(maps:get(connect, Map, [])), - listen => rules(maps:get(listen, Map, [])), - resolve => resolve(maps:get(resolve, Map, deny))}; -net(Other) -> - throw({net, Other}). - -resolve(allow) -> true; -resolve(deny) -> false; -resolve(Other) -> throw({net, {resolve, Other}}). - -rules(L) when is_list(L) -> [rule(R) || R <- L]; -rules(Other) -> throw({net, Other}). - -rule({Proto, Addr, Port}) when Proto =:= tcp; Proto =:= udp -> - {Proto, cidr(Addr), ports(Port)}; -rule(Other) -> - throw({net, {rule, Other}}). - -%% An address with no prefix length is one host: a full-width prefix. -cidr(Bin) when is_binary(Bin) -> - case binary:split(Bin, <<"/">>) of - [Addr] -> host(parse_or_fail(Addr)); - [Addr, Len] -> network(parse_or_fail(Addr), integer_or_fail(Len, Bin), Bin) - end; -cidr(Tuple) when tuple_size(Tuple) =:= 4; tuple_size(Tuple) =:= 8 -> - host(Tuple); -cidr(Other) -> - throw({net, {address, Other}}). - -host(Addr0) -> - Addr = normalise(Addr0), - {Addr, width(Addr)}. - -%% The prefix length is written in the notation the address was written in, so -%% a mapped base has to have its 96 mapping bits taken off with it. Below 96 -%% the prefix spans addresses inside and outside the mapped block at once, -%% which has no IPv4 meaning; refuse rather than guess which half was meant. -network(Addr, Bits, Written) -> - case normalise(Addr) of - A when tuple_size(A) =:= 4, Bits >= 0, Bits =< 32 -> - {mask(A, Bits), Bits}; - A when tuple_size(A) =:= 8, Bits >= 0, Bits =< 128 -> - {mask(A, Bits), Bits}; - V4 when Bits >= 96, Bits =< 128 -> - {mask(V4, Bits - 96), Bits - 96}; - _ -> - throw({net, {address, Written}}) - end. - -ports(any) -> {0, 65535}; -ports(P) when is_integer(P), P >= 0, P =< 65535 -> {P, P}; -ports({Lo, Hi}) when is_integer(Lo), is_integer(Hi), Lo >= 0, Lo =< Hi, - Hi =< 65535 -> {Lo, Hi}; -ports(Other) -> throw({net, {port, Other}}). - -parse_or_fail(Bin) -> - case inet:parse_address(binary_to_list(Bin)) of - {ok, Addr} -> normalise(Addr); - {error, _} -> throw({net, {address, Bin}}) - end. - -integer_or_fail(Bin, Written) -> - try binary_to_integer(Bin) - catch _:_ -> throw({net, {address, Written}}) - end. - -%% Fold an IPv4-mapped IPv6 address onto the IPv4 address it reaches, so -%% `::ffff:127.0.0.1' cannot walk past a `127.0.0.0/8' rule. The deprecated -%% IPv4-compatible block is left alone: `::0.0.0.1' and `::1' are the same -%% address, so folding it would make loopback ambiguous. -normalise({0, 0, 0, 0, 0, 16#ffff, X, Y}) -> - {X bsr 8, X band 16#ff, Y bsr 8, Y band 16#ff}; -normalise(Addr) -> - Addr. - -width(Addr) when tuple_size(Addr) =:= 4 -> 32; -width(Addr) when tuple_size(Addr) =:= 8 -> 128. - -%% Zeroing the host bits, so a rule written `10.1.2.3/8' means the same -%% network as `10.0.0.0/8' rather than never matching anything. -mask(Addr, Bits) -> - W = width(Addr), - from_int(to_int(Addr) band (((1 bsl Bits) - 1) bsl (W - Bits)), W). - -to_int(Addr) -> - Size = part_size(Addr), - lists:foldl(fun(P, Acc) -> (Acc bsl Size) bor P end, 0, tuple_to_list(Addr)). - -from_int(N, 32) -> - <> = <>, - {A, B, C, D}; -from_int(N, 128) -> - <> = <>, - {A, B, C, D, E, F, G, H}. - -part_size(Addr) when tuple_size(Addr) =:= 4 -> 8; -part_size(Addr) when tuple_size(Addr) =:= 8 -> 16. - -%%% ============================================================================ -%%% Wire form -%%% ============================================================================ - -net_json(none) -> - null; -net_json(#{connect := C, listen := L, resolve := R}) -> - #{<<"connect">> => [rule_json(Rule) || Rule <- C], - <<"listen">> => [rule_json(Rule) || Rule <- L], - <<"resolve">> => R}. - -%% The child matches with Python's `ipaddress', so rules cross as the CIDR -%% text that module reads. The address is already masked and folded here, so -%% both sides agree on what a rule covers without parsing it twice. -rule_json({Proto, {Addr, Bits}, {Lo, Hi}}) -> - #{<<"proto">> => atom_to_binary(Proto), - <<"cidr">> => iolist_to_binary([inet:ntoa(Addr), "/", integer_to_list(Bits)]), - <<"ports">> => [Lo, Hi]}. - -to_bin(B) when is_binary(B) -> B; -to_bin(L) when is_list(L) -> list_to_binary(L); -to_bin(A) when is_atom(A) -> atom_to_binary(A, utf8); -to_bin(Other) -> throw({not_a_string, Other}). diff --git a/src/py_context.erl b/src/py_context.erl index 210a74f..757df51 100644 --- a/src/py_context.erl +++ b/src/py_context.erl @@ -188,32 +188,7 @@ stop(Ctx) when is_pid(Ctx) -> new(Opts) when is_map(Opts) -> Mode = maps:get(mode, Opts, worker), Id = erlang:unique_integer([positive]), - case check_caps(Mode, Opts) of - ok -> start_link(Id, Mode, Opts); - {error, _} = Err -> Err - end. - -%% @private A capability grant is only meaningful where the interpreter is a -%% child process. Read it here, in the caller, so a malformed rule is the -%% configuration error it is rather than a connection refused much later. -check_caps(Mode, Opts) -> - case maps:get(caps, Opts, undefined) of - undefined -> - ok; - _ when Mode =/= isolated -> - {error, {caps_requires_isolated, Mode}}; - _ when is_map_key(env, Opts) -> - %% The `env' option adds to the child's environment and a grant - %% says what the whole of it is. Taking both would mean one of - %% them silently losing, so say so instead: `caps.env' is the - %% one that names an environment. - {error, {bad_caps, env_option_conflicts_with_caps_env}}; - Caps -> - case py_caps:validate(Caps) of - {ok, _} -> ok; - {error, _} = Err -> Err - end - end. + start_link(Id, Mode, Opts). %% @doc Alias for stop/1 for API consistency. -spec destroy(context()) -> ok. diff --git a/src/py_isolated.erl b/src/py_isolated.erl index 3a26721..566c4f8 100644 --- a/src/py_isolated.erl +++ b/src/py_isolated.erl @@ -446,38 +446,10 @@ resolve_exe(Exe) -> start_child(#data{opts = Opts} = St) -> case check_platform_opts(Opts) of - ok -> - case caps(Opts) of - {ok, _} -> start_child_1(St); - {error, _} = Err -> Err - end; - {error, _} = Err -> - Err + ok -> start_child_1(St); + {error, _} = Err -> Err end. -%% A grant is read here as well as in py_context:new/1, so a context started -%% by any other route still fails with the configuration error rather than -%% with a child that refuses everything. -caps(Opts) -> - case maps:get(caps, Opts, undefined) of - undefined -> - {ok, none}; - _ when is_map_key(env, Opts) -> - {error, {bad_caps, env_option_conflicts_with_caps_env}}; - Caps -> - case py_caps:validate(Caps) of - {ok, Valid} -> {ok, with_import_paths(Valid, Opts)}; - {error, _} = Err -> Err - end - end. - -%% A directory named in `paths' is one the child was told to import from, so -%% it is granted for reading. Saying it twice would be a trap, and leaving it -%% ungranted turns `paths' into an import error rather than a grant error. -with_import_paths(#{dirs := Dirs} = Caps, Opts) -> - Extra = [{to_bin(P), read} || P <- maps:get(paths, Opts, [])], - Caps#{dirs => Dirs ++ [D || D <- Extra, not lists:member(D, Dirs)]}. - %% cgroups exist only on Linux; rlimits are POSIX and apply everywhere. %% RLIMIT_AS is enforced by the kernel on Linux and FreeBSD; on macOS the %% child enforces `as' with a watchdog thread on its resident set. @@ -511,8 +483,7 @@ spawn_child(Python, Opts) -> ok = socket:bind(L, #{family => local, path => Path}), ok = socket:listen(L), Script = filename:join(priv_dir(), "py_isolated_child.py"), - Args = [Script, Path | rlimit_args(Opts) ++ cgroup_args(Opts) - ++ caps_args(Opts)], + Args = [Script, Path | rlimit_args(Opts) ++ cgroup_args(Opts)], PortOpts = [exit_status, stderr_to_stdout, binary, use_stdio, {args, Args}, {env, env_opt(Opts)}], Port = open_port({spawn_executable, Python}, PortOpts), @@ -1216,35 +1187,8 @@ cgroup_args(Opts) -> Dir -> ["--cgroup", to_list(Dir)] end. -%% The grant travels in argv rather than in the handshake because argv is -%% read in the child's prologue, before the socket exists and before any -%% user code can run. -caps_args(Opts) -> - case caps(Opts) of - {ok, none} -> []; - {ok, Caps} -> ["--caps-json", binary_to_list(py_caps:to_json(Caps))] - end. - -%% Without a grant the child inherits the VM's environment and the `env' -%% option adds to it. With one, it gets what the grant names and nothing -%% else, except the loader variables, which belong to whoever started the -%% node rather than to the workload and without which an interpreter built -%% against a private libpython does not start at all. env_opt(Opts) -> - User = [{to_list(K), to_list(V)} || {K, V} <- maps:to_list(maps:get(env, Opts, #{}))], - case caps(Opts) of - {ok, none} -> - User; - {ok, #{env := Granted}} -> - %% `User' is empty here: a grant and the `env' option together - %% are refused in caps/1, because the port keeps the last of two - %% settings for the same name and the option would win. - Keep = ["LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH", - "DYLD_FALLBACK_LIBRARY_PATH"], - Clear = [{Name, false} || {Name, _} <- os:env(), - not lists:member(Name, Keep)], - Clear ++ [{to_list(K), to_list(V)} || {K, V} <- maps:to_list(Granted)] - end. + [{to_list(K), to_list(V)} || {K, V} <- maps:to_list(maps:get(env, Opts, #{}))]. to_bin(A) when is_atom(A) -> atom_to_binary(A, utf8); to_bin(L) when is_list(L) -> unicode:characters_to_binary(L); diff --git a/test/coverage_audit.md b/test/coverage_audit.md index c0a977a..bb8a8ca 100644 --- a/test/coverage_audit.md +++ b/test/coverage_audit.md @@ -34,7 +34,6 @@ suite is visible. `scripts/check_code_map.sh` requires a row per module. | `py_channel` | `py_channel_SUITE`, `py_byte_channel_SUITE` | | `py_byte_channel` | `py_channel_SUITE`, `py_byte_channel_SUITE` | | `py_buffer` | `py_buffer_SUITE`, `py_isolated_buffer_SUITE` | -| `py_caps` | `py_isolated_caps_SUITE` | | `py_shm` | `py_isolated_shm_SUITE` | | `py_import` | `py_import_SUITE` | | `py_preload` | `py_preload_SUITE` | diff --git a/test/py_isolated_caps_SUITE.erl b/test/py_isolated_caps_SUITE.erl deleted file mode 100644 index 6caea88..0000000 --- a/test/py_isolated_caps_SUITE.erl +++ /dev/null @@ -1,628 +0,0 @@ -%% @doc The `caps' option: what an isolated child may reach. -%% -%% The path cases are the ones that matter. A capability set that opens the -%% right files is easy; one that reliably refuses the wrong ones is the whole -%% point. Each escape technique gets its own case, and each is refused as a -%% capability error rather than as a missing file, so the error cannot be used -%% to find out what exists outside a grant. -%% -%% The case names follow `wasi_SUITE' and `wasi_net_SUITE' in erlang_wasm, -%% whose grant model this implements, so the two can be read side by side. --module(py_isolated_caps_SUITE). - --include_lib("common_test/include/ct.hrl"). --include_lib("stdlib/include/assert.hrl"). - --export([all/0, init_per_suite/1, end_per_suite/1, - init_per_testcase/2, end_per_testcase/2]). - --export([ - reads_inside_a_grant/1, - parent_traversal_is_refused/1, - absolute_path_is_refused/1, - partial_traversal_is_refused/1, - symlink_escape_is_refused/1, - symlink_directory_prefix_is_refused/1, - a_symlink_inside_a_grant_is_followed/1, - a_symlink_cycle_is_refused_rather_than_followed/1, - missing_file_inside_a_grant_is_not_a_refusal/1, - a_read_grant_yields_no_write_whatever_the_flags/1, - a_write_grant_allows_create_and_unlink/1, - listing_is_granted_with_the_directory/1, - imports_still_work/1, - a_network_and_a_port_range/1, - an_ungranted_port_is_refused_even_where_something_listens/1, - binding_is_checked_against_listen_not_connect/1, - resolution_is_its_own_capability/1, - resolution_cannot_widen_a_grant/1, - a_wildcard_grant_really_is_a_wildcard/1, - no_net_key_is_no_network/1, - a_passed_fd_still_serves/1, - env_is_what_was_granted_and_nothing_else/1, - subprocess_is_refused/1, - ctypes_is_refused/1, - the_policy_cannot_be_switched_off_from_python/1, - signalling_another_process_is_refused/1, - unaudited_creators_are_taken_away/1, - every_resolver_is_gated_not_only_getaddrinfo/1, - the_env_option_cannot_widen_a_grant/1, - shared_memory_is_refused_under_a_capability_set/1, - a_user_fspath_runs_enforced/1, - a_unix_socket_is_not_a_file/1, - a_read_grant_cannot_become_a_write/1, - caps_survive_a_child_restart/1, - child_info_reports_the_grants/1, - caps_are_rejected_outside_isolated/1, - a_malformed_rule_is_a_configuration_error/1, - no_caps_changes_nothing/1 -]). - --define(TEST_MOD, py_test_caps). - -all() -> - [ - %% filesystem - reads_inside_a_grant, - parent_traversal_is_refused, - absolute_path_is_refused, - partial_traversal_is_refused, - symlink_escape_is_refused, - symlink_directory_prefix_is_refused, - a_symlink_inside_a_grant_is_followed, - a_symlink_cycle_is_refused_rather_than_followed, - missing_file_inside_a_grant_is_not_a_refusal, - a_read_grant_yields_no_write_whatever_the_flags, - a_write_grant_allows_create_and_unlink, - listing_is_granted_with_the_directory, - imports_still_work, - %% network - a_network_and_a_port_range, - an_ungranted_port_is_refused_even_where_something_listens, - binding_is_checked_against_listen_not_connect, - resolution_is_its_own_capability, - resolution_cannot_widen_a_grant, - a_wildcard_grant_really_is_a_wildcard, - no_net_key_is_no_network, - a_passed_fd_still_serves, - %% the rest - env_is_what_was_granted_and_nothing_else, - subprocess_is_refused, - ctypes_is_refused, - the_policy_cannot_be_switched_off_from_python, - signalling_another_process_is_refused, - unaudited_creators_are_taken_away, - every_resolver_is_gated_not_only_getaddrinfo, - the_env_option_cannot_widen_a_grant, - shared_memory_is_refused_under_a_capability_set, - a_user_fspath_runs_enforced, - a_unix_socket_is_not_a_file, - a_read_grant_cannot_become_a_write, - caps_survive_a_child_restart, - child_info_reports_the_grants, - caps_are_rejected_outside_isolated, - a_malformed_rule_is_a_configuration_error, - no_caps_changes_nothing - ]. - -init_per_suite(Config) -> - {ok, _} = application:ensure_all_started(erlang_python), - [{test_dir, filename:join(code:lib_dir(erlang_python), "test")} | Config]. - -end_per_suite(_Config) -> - ok = application:stop(erlang_python), - ok. - -%% A tree with everything the escape cases need, made fresh for each case so -%% one case cannot leave a symlink behind for the next. -%% -%% root/data/note.txt readable -%% root/data/sub/deep.txt readable, one level down -%% root/data/escape -> root/secret/key.txt -%% root/data/outdir -> root/secret -%% root/data/here -> note.txt (stays inside) -%% root/data/loop -> loop -%% root/secret/key.txt never granted -init_per_testcase(TestCase, Config) -> - Root = filename:join(?config(priv_dir, Config), atom_to_list(TestCase)), - Data = filename:join(Root, "data"), - Secret = filename:join(Root, "secret"), - ok = filelib:ensure_path(filename:join(Data, "sub")), - ok = filelib:ensure_path(Secret), - ok = file:write_file(filename:join(Data, "note.txt"), <<"inside">>), - ok = file:write_file(filename:join([Data, "sub", "deep.txt"]), <<"deep">>), - ok = file:write_file(filename:join(Secret, "key.txt"), <<"secret">>), - ok = file:make_symlink(filename:join(Secret, "key.txt"), - filename:join(Data, "escape")), - ok = file:make_symlink(Secret, filename:join(Data, "outdir")), - ok = file:make_symlink("note.txt", filename:join(Data, "here")), - ok = file:make_symlink("loop", filename:join(Data, "loop")), - [{root, Root}, {data, Data}, {secret, Secret} | Config]. - -end_per_testcase(_TestCase, _Config) -> - flush(), - ok. - -%%% ============================================================================ -%%% Filesystem -%%% ============================================================================ - -reads_inside_a_grant(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - {ok, <<"inside">>} = read(C, path(Config, "note.txt")), - {ok, <<"deep">>} = read(C, path(Config, "sub/deep.txt")), - {ok, <<"inside">>} = read(C, path(Config, "./note.txt")), - alive(C), - stop(C). - -parent_traversal_is_refused(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - refused(read(C, path(Config, "../secret/key.txt"))), - alive(C), - stop(C). - -absolute_path_is_refused(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - refused(read(C, "/etc/hosts")), - refused(read(C, filename:join(?config(secret, Config), "key.txt"))), - alive(C), - stop(C). - -%% Leaves the grant and comes back. Refused because the path leaves at any -%% point, not merely because of where it ends up; and the half of it that is -%% legal really is legal, or this case would pass just as well with `..' -%% refused outright. -partial_traversal_is_refused(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - refused(read(C, path(Config, "sub/../../secret/key.txt"))), - {ok, <<"inside">>} = read(C, path(Config, "sub/../note.txt")), - alive(C), - stop(C). - -symlink_escape_is_refused(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - refused(read(C, path(Config, "escape"))), - alive(C), - stop(C). - -symlink_directory_prefix_is_refused(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - refused(read(C, path(Config, "outdir/key.txt"))), - alive(C), - stop(C). - -a_symlink_inside_a_grant_is_followed(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - {ok, <<"inside">>} = read(C, path(Config, "here")), - alive(C), - stop(C). - -a_symlink_cycle_is_refused_rather_than_followed(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - refused(read(C, path(Config, "loop"))), - alive(C), - stop(C). - -%% A file that is not there is not a capability answer. Distinguishing the two -%% is the whole reason refusals are not `FileNotFoundError'. -missing_file_inside_a_grant_is_not_a_refusal(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - {error, {'FileNotFoundError', _}} = read(C, path(Config, "missing.txt")), - alive(C), - stop(C). - -a_read_grant_yields_no_write_whatever_the_flags(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - refused(call(C, write_file, [path(Config, "new.txt"), <<"x">>])), - refused(call(C, append_file, [path(Config, "note.txt"), <<"x">>])), - refused(call(C, truncate_file, [path(Config, "note.txt")])), - refused(call(C, remove_file, [path(Config, "note.txt")])), - {ok, <<"inside">>} = file:read_file(path(Config, "note.txt")), - alive(C), - stop(C). - -a_write_grant_allows_create_and_unlink(Config) -> - C = ctx(Config, #{dirs => [{data(Config), write}]}), - {ok, <<"ok">>} = call(C, write_file, [path(Config, "new.txt"), <<"written">>]), - {ok, <<"written">>} = file:read_file(path(Config, "new.txt")), - {ok, <<"ok">>} = call(C, remove_file, [path(Config, "new.txt")]), - false = filelib:is_regular(path(Config, "new.txt")), - %% Still only this grant: the neighbouring directory is untouched. - refused(call(C, write_file, - [filename:join(?config(secret, Config), "x"), <<"x">>])), - alive(C), - stop(C). - -listing_is_granted_with_the_directory(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - {ok, Names} = call(C, list_dir, [data(Config)]), - true = lists:member(<<"note.txt">>, Names), - refused(call(C, list_dir, [?config(secret, Config)])), - alive(C), - stop(C). - -%% The interpreter's own path is granted, or nothing would import at all. -imports_still_work(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - {ok, <<"[1, 2]">>} = py_context:eval( - C, <<"__import__('json').dumps([1,2])">>), - {ok, 4} = py_context:eval(C, <<"len(__import__('base64').b64encode(b'ab'))">>), - stop(C). - -%%% ============================================================================ -%%% Network -%%% ============================================================================ - -a_network_and_a_port_range(Config) -> - {LSock, Port} = listener(), - C = ctx(Config, #{net => #{connect => [{tcp, <<"127.0.0.0/8">>, - {Port, Port}}]}}), - {ok, <<"connected">>} = call(C, connect, [<<"127.0.0.1">>, Port]), - {ok, _} = gen_tcp:accept(LSock, 2000), - ok = gen_tcp:close(LSock), - alive(C), - stop(C). - -%% Something is accepting on this port, and the answer is the same one a dead -%% port would get: nothing was attempted. -an_ungranted_port_is_refused_even_where_something_listens(Config) -> - {LSock, Port} = listener(), - Granted = free_port(), - C = ctx(Config, #{net => #{connect => [{tcp, <<"127.0.0.1">>, Granted}]}}), - refused(call(C, connect, [<<"127.0.0.1">>, Port])), - {error, timeout} = gen_tcp:accept(LSock, 300), - ok = gen_tcp:close(LSock), - alive(C), - stop(C). - -%% Binding claims a local address, which is what `listen' grants. A connect -%% grant for the same address does not carry it. -binding_is_checked_against_listen_not_connect(Config) -> - Port = free_port(), - C = ctx(Config, #{net => #{connect => [{tcp, <<"127.0.0.1">>, any}], - listen => [{tcp, <<"127.0.0.1">>, Port}]}}), - {ok, <<"bound">>} = call(C, bind, [<<"127.0.0.1">>, Port]), - refused(call(C, bind, [<<"127.0.0.1">>, free_port()])), - alive(C), - stop(C). - -resolution_is_its_own_capability(Config) -> - C = ctx(Config, #{net => #{connect => [{tcp, <<"127.0.0.1">>, any}]}}), - refused(call(C, resolve, [<<"localhost">>])), - stop(C), - C2 = ctx(Config, #{net => #{connect => [{tcp, <<"127.0.0.1">>, any}], - resolve => allow}}), - {ok, _} = call(C2, resolve, [<<"localhost">>]), - alive(C2), - stop(C2). - -%% An address learned by resolving carries no authority from having been -%% resolved: the connect is still checked, and still refused. -resolution_cannot_widen_a_grant(Config) -> - {LSock, Port} = listener(), - C = ctx(Config, #{net => #{connect => [{tcp, <<"10.0.0.0/8">>, any}], - resolve => allow}}), - {ok, Addrs} = call(C, resolve, [<<"localhost">>]), - true = lists:member(<<"127.0.0.1">>, Addrs), - refused(call(C, connect, [<<"127.0.0.1">>, Port])), - {error, timeout} = gen_tcp:accept(LSock, 300), - ok = gen_tcp:close(LSock), - stop(C). - -%% An inverse case: the documented sharp edge is that nothing is denied -%% implicitly, so adding a hidden deny list has to break the build and force -%% the guide to be corrected. -a_wildcard_grant_really_is_a_wildcard(Config) -> - {LSock, Port} = listener(), - C = ctx(Config, #{net => #{connect => [{tcp, <<"0.0.0.0/0">>, any}]}}), - {ok, <<"connected">>} = call(C, connect, [<<"127.0.0.1">>, Port]), - {ok, _} = gen_tcp:accept(LSock, 2000), - ok = gen_tcp:close(LSock), - stop(C). - -no_net_key_is_no_network(Config) -> - {LSock, Port} = listener(), - C = ctx(Config, #{dirs => [{data(Config), read}]}), - refused(call(C, connect, [<<"127.0.0.1">>, Port])), - C2 = ctx(Config, #{net => #{}}), - refused(call(C2, connect, [<<"127.0.0.1">>, Port])), - {error, timeout} = gen_tcp:accept(LSock, 300), - ok = gen_tcp:close(LSock), - stop(C), - stop(C2). - -%% A socket Erlang opened and handed over needs no grant: the child was given -%% the descriptor, which is the capability. -a_passed_fd_still_serves(Config) -> - {ok, LSock} = gen_tcp:listen(0, [binary, {ip, {127,0,0,1}}, - {active, false}, {backlog, 8}]), - {ok, Port} = inet:port(LSock), - {ok, Fd} = inet:getfd(LSock), - C = ctx(Config, #{dirs => [{data(Config), read}]}), - {ok, ChildFd} = py_context:pass_fd(C, Fd), - ok = py_context:start_loop(C), - {ok, _} = py_context:submit_await(C, ?TEST_MOD, serve, [ChildFd], #{}, 10000), - {ok, Sock} = gen_tcp:connect({127,0,0,1}, Port, [binary, {active, false}], 2000), - ok = gen_tcp:send(Sock, <<"ping">>), - {ok, <<"pong">>} = gen_tcp:recv(Sock, 4, 5000), - ok = gen_tcp:close(Sock), - ok = gen_tcp:close(LSock), - ok = py_context:stop_loop(C), - stop(C). - -%%% ============================================================================ -%%% Environment, processes, shared memory, lifecycle -%%% ============================================================================ - -env_is_what_was_granted_and_nothing_else(Config) -> - true = os:putenv("EP_CAPS_SECRET", "leaked"), - C = ctx(Config, #{env => #{<<"EP_CAPS_GRANTED">> => <<"yes">>}}), - {ok, <<"yes">>} = call(C, getenv, [<<"EP_CAPS_GRANTED">>]), - {ok, none} = call(C, getenv, [<<"EP_CAPS_SECRET">>]), - {ok, none} = call(C, getenv, [<<"HOME">>]), - stop(C), - %% Without a capability set the child inherits as it always did. - C2 = ctx(Config, no_caps), - {ok, <<"leaked">>} = call(C2, getenv, [<<"EP_CAPS_SECRET">>]), - stop(C2), - true = os:unsetenv("EP_CAPS_SECRET"). - -subprocess_is_refused(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - refused(call(C, run_subprocess, [])), - refused(py_context:eval(C, <<"__import__('os').fork()">>)), - alive(C), - stop(C). - -ctypes_is_refused(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - refused(py_context:eval(C, <<"__import__('ctypes').CDLL(None)">>)), - alive(C), - stop(C). - -%% The hook must not read anything the workload can assign to, or the policy -%% is off the moment code says so. -the_policy_cannot_be_switched_off_from_python(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - refused(read(C, "/etc/hosts")), - %% Every name the hook used to resolve when it ran, assigned at once. - ok = py_context:exec(C, <<"import _erlang_impl._caps as c\n" - "c._summary = None\n" - "c._local = type('x', (), {'busy': True})()\n" - "c._writes = lambda *a: False\n" - "c._PATH_EVENTS = {}\n" - "c._RESOLVE_EVENTS = frozenset()\n" - "c._SUBPROCESS_EVENTS = frozenset()\n" - "c._make_enforcer = None\n" - "c.os = None\n">>), - refused(read(C, "/etc/hosts")), - refused(call(C, run_subprocess, [])), - %% And the levers that used to let Python widen a grant are gone. - {ok, false} = py_context:eval( - C, <<"hasattr(__import__('_erlang_impl._caps', fromlist=['x'])," - " 'allow_path')">>), - {ok, false} = py_context:eval( - C, <<"hasattr(__import__('_erlang_impl._caps', fromlist=['x'])," - " '_walk')">>), - alive(C), - stop(C). - -%% The child shares the node's user and its parent is the BEAM, so an -%% unchecked signal is a way to take the node down. -signalling_another_process_is_refused(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - refused(py_context:eval(C, <<"__import__('os').kill(__import__('os')" - ".getppid(), 0)">>)), - refused(py_context:eval(C, <<"__import__('os').killpg(__import__('os')" - ".getpgrp(), 0)">>)), - refused(py_context:eval(C, <<"__import__('os').kill(1, 0)">>)), - %% Signalling itself is its own business. - {ok, none} = py_context:eval(C, <<"__import__('os').kill(__import__('os')" - ".getpid(), 0)">>), - alive(C), - stop(C). - -%% CPython raises no audit event for these, so they cannot be refused and -%% are taken away instead. Their absence is the assertion. -unaudited_creators_are_taken_away(Config) -> - C = ctx(Config, #{dirs => [{data(Config), write}]}), - {ok, false} = py_context:eval(C, <<"hasattr(__import__('os'), 'mkfifo')">>), - {ok, false} = py_context:eval(C, <<"hasattr(__import__('os'), 'mknod')">>), - {ok, false} = py_context:eval(C, <<"hasattr(__import__('posix'), 'mkfifo')">>), - {ok, false} = py_context:eval(C, <<"hasattr(__import__('posix'), 'mknod')">>), - alive(C), - stop(C). - -%% Gating only getaddrinfo would leave every other resolver as a way out, -%% and a name lookup is a message to whoever answers it. -every_resolver_is_gated_not_only_getaddrinfo(Config) -> - C = ctx(Config, #{net => #{connect => [{tcp, <<"127.0.0.1">>, any}]}}), - refused(call(C, resolve, [<<"localhost">>])), - refused(py_context:eval(C, <<"__import__('socket').gethostbyname('localhost')">>)), - refused(py_context:eval(C, <<"__import__('socket').gethostbyname_ex('localhost')">>)), - refused(py_context:eval(C, <<"__import__('socket').gethostbyaddr('127.0.0.1')">>)), - refused(py_context:eval(C, <<"__import__('socket').getnameinfo(('127.0.0.1',80),0)">>)), - refused(py_context:eval(C, <<"__import__('socket').gethostname()">>)), - alive(C), - stop(C). - -%% The `env' option adds to the environment and a grant says what the whole -%% of it is; the port keeps the last setting for a name, so taking both -%% would let the option quietly win. -the_env_option_cannot_widen_a_grant(_Config) -> - {error, {bad_caps, env_option_conflicts_with_caps_env}} = - py_context:new(#{mode => isolated, - caps => #{env => #{<<"A">> => <<"1">>}}, - env => #{<<"SECRET">> => <<"leaked">>}}), - {error, {bad_caps, env_option_conflicts_with_caps_env}} = - py_context:new(#{mode => isolated, caps => #{}, - env => #{<<"SECRET">> => <<"leaked">>}}), - ok. - -%% Shared memory does not combine with a capability set yet: a region -%% arrives as a path, and the only way to grant it would be to hand over the -%% directory holding every region this node owns. Passing the descriptor is -%% the fix, and it is not here yet, so the refusal has to be legible. -shared_memory_is_refused_under_a_capability_set(Config) -> - case py_shm:available() of - false -> - {skip, "iommap not available"}; - true -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - {ok, Shm} = py_shm:new(4096), - refused(py_context:call(C, ?TEST_MOD, shm_write, - [Shm, <<"payload">>], #{}, 10000)), - %% Erlang still has it, unharmed. - ok = py_shm:write(Shm, 0, <<"payload">>), - {ok, <<"payload">>} = py_shm:read(Shm, 0, 7), - alive(C), - ok = py_shm:close(Shm), - stop(C) - end. - -%% Path conversion happens before the re-entrancy guard, so a `__fspath__' -%% method is user code that runs with the hook live rather than a window in -%% which everything is allowed. -a_user_fspath_runs_enforced(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - {ok, false} = py_context:call(C, ?TEST_MOD, read_through_fspath, - [path(Config, "note.txt")], #{}, 10000), - alive(C), - stop(C). - -%% Reaching a Unix socket is talking to whatever is behind it, which a -%% directory grant says nothing about. -a_unix_socket_is_not_a_file(Config) -> - %% Its own short directory: an AF_UNIX path is capped near 104 bytes and - %% `connect' rejects a longer one before the hook ever sees it, which - %% would make this case pass for the wrong reason. - Dir = "/tmp/ep_caps_u" ++ integer_to_list(erlang:unique_integer([positive])), - ok = filelib:ensure_path(Dir), - C = ctx(Config, #{dirs => [{Dir, write}]}), - try - %% The directory is granted for writing, so the path is reachable as - %% a file; talking through it is not what that grant said. - {ok, <<"ok">>} = call(C, write_file, [Dir ++ "/plain", <<"x">>]), - refused(call(C, unix_connect, [Dir ++ "/sock"])), - refused(call(C, unix_bind, [Dir ++ "/mine.sock"])), - alive(C) - after - stop(C), - _ = file:del_dir_r(Dir) - end. - -%% Every route from a read grant to a write, including the ones CPython -%% does not announce by path. -a_read_grant_cannot_become_a_write(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - refused(call(C, write_file, [path(Config, "note.txt"), <<"x">>])), - refused(call(C, truncate_by_descriptor, [path(Config, "note.txt")])), - {ok, <<"inside">>} = file:read_file(path(Config, "note.txt")), - alive(C), - stop(C). - -caps_survive_a_child_restart(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}]}), - refused(read(C, "/etc/hosts")), - ok = py_context:kill(C), - {ok, <<"inside">>} = read(C, path(Config, "note.txt")), - refused(read(C, "/etc/hosts")), - refused(call(C, run_subprocess, [])), - stop(C). - -child_info_reports_the_grants(Config) -> - C = ctx(Config, #{dirs => [{data(Config), read}], - net => #{connect => [{tcp, <<"10.0.0.0/8">>, 443}]}}), - {ok, Info} = py_context:child_info(C), - Caps = maps:get(caps, Info), - Dirs = maps:get(<<"dirs">>, Caps), - Bin = list_to_binary(data(Config)), - true = lists:keymember(Bin, 1, Dirs), - #{<<"connect">> := [<<"tcp 10.0.0.0/8 443-443">>]} = maps:get(<<"net">>, Caps), - stop(C). - -caps_are_rejected_outside_isolated(_Config) -> - {error, {caps_requires_isolated, worker}} = - py_context:new(#{mode => worker, caps => #{}}), - {error, {caps_requires_isolated, owngil}} = - py_context:new(#{mode => owngil, caps => #{}}), - ok. - -%% A malformed rule is a configuration error and is reported as one here, -%% rather than met later as a refused connection: a capability set that -%% silently refuses everything looks exactly like one that works. -a_malformed_rule_is_a_configuration_error(_Config) -> - {error, {bad_caps, {net, {address, <<"nope">>}}}} = - py_context:new(#{mode => isolated, - caps => #{net => #{connect => [{tcp, <<"nope">>, 80}]}}}), - {error, {bad_caps, {dir_not_absolute, "rel"}}} = - py_context:new(#{mode => isolated, caps => #{dirs => [{"rel", read}]}}), - {error, {bad_caps, {net, {port, -1}}}} = - py_context:new(#{mode => isolated, - caps => #{net => #{listen => [{tcp, <<"127.0.0.1">>, -1}]}}}), - {error, {bad_caps, {unknown_keys, [bogus]}}} = - py_context:new(#{mode => isolated, caps => #{bogus => 1}}), - ok. - -no_caps_changes_nothing(Config) -> - C = ctx(Config, no_caps), - {ok, _} = read(C, "/etc/hosts"), - {ok, 4} = py_context:eval(C, <<"2+2">>), - stop(C). - -%%% ============================================================================ -%%% Helpers -%%% ============================================================================ - -ctx(Config, no_caps) -> - new(Config, #{}); -ctx(Config, Caps) -> - new(Config, #{caps => Caps}). - -new(Config, Extra) -> - TestDir = ?config(test_dir, Config), - Opts = maps:merge(#{mode => isolated, paths => [TestDir]}, Extra), - {ok, C} = py_context:new(Opts), - C. - -stop(C) -> - ok = py_context:stop(C). - -%% Every case ends with the context still working: a refusal must not have -%% left the child broken. -alive(C) -> - {ok, 4} = py_context:eval(C, <<"2+2">>). - -data(Config) -> ?config(data, Config). - -path(Config, Rel) -> filename:join(data(Config), Rel). - -read(C, Path) -> - call(C, read_file, [to_bin(Path)]). - -call(C, Fun, Args) -> - py_context:call(C, ?TEST_MOD, Fun, [to_bin(A) || A <- Args], #{}, 10000). - -%% A capability error, and never a missing-file error: the refusal says -%% nothing about whether the path exists. -refused({error, {'CapabilityError', _}}) -> ok; -refused(Other) -> ct:fail({expected_refusal, Other}). - -listener() -> - {ok, LSock} = gen_tcp:listen(0, [binary, {ip, {127,0,0,1}}, - {active, false}, {backlog, 8}]), - {ok, Port} = inet:port(LSock), - {LSock, Port}. - -free_port() -> - {ok, S} = gen_tcp:listen(0, [{ip, {127,0,0,1}}]), - {ok, P} = inet:port(S), - ok = gen_tcp:close(S), - P. - -to_bin(B) when is_binary(B) -> B; -to_bin(L) when is_list(L) -> list_to_binary(L); -to_bin(I) when is_integer(I) -> I; -to_bin(T) when is_tuple(T) -> T. - -flush() -> - receive _ -> flush() after 0 -> ok end. diff --git a/test/py_test_caps.py b/test/py_test_caps.py deleted file mode 100644 index 1d32e93..0000000 --- a/test/py_test_caps.py +++ /dev/null @@ -1,171 +0,0 @@ -"""Helpers for py_isolated_caps_SUITE. - -Each one is the smallest Python that performs one operation a capability set -either grants or refuses. They return a plain term on success and let the -exception through on refusal, so the suite sees `{error, {'CapabilityError', -Msg}}` and can tell it apart from a missing file. -""" - -import asyncio -import os -import socket -import subprocess - -_servers = {} - - -def _bytes(value): - """Erlang binaries arrive as `str`; `{bytes, B}` is what arrives as bytes.""" - return value.encode() if isinstance(value, str) else value - - -def _text(value): - return value.decode() if isinstance(value, bytes) else value - - -# --- filesystem ------------------------------------------------------------- - -def read_file(path): - with open(path, 'rb') as fh: - return fh.read() - - -def write_file(path, data): - with open(path, 'wb') as fh: - fh.write(_bytes(data)) - return 'ok' - - -def append_file(path, data): - with open(path, 'ab') as fh: - fh.write(_bytes(data)) - return 'ok' - - -def truncate_file(path): - os.truncate(path, 0) - return 'ok' - - -def remove_file(path): - os.remove(path) - return 'ok' - - -def list_dir(path): - return sorted(os.listdir(path)) - - -# --- network ---------------------------------------------------------------- - -def connect(host, port): - sock = socket.socket() - try: - sock.settimeout(5) - sock.connect((_text(host), port)) - return 'connected' - finally: - sock.close() - - -def bind(host, port): - sock = socket.socket() - try: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind((_text(host), port)) - return 'bound' - finally: - sock.close() - - -def resolve(name): - name = _text(name) - return sorted({info[4][0] for info in socket.getaddrinfo(name, 80)}) - - -class _Echo(asyncio.Protocol): - def connection_made(self, transport): - self.transport = transport - - def data_received(self, data): - self.transport.write(b'pong') - - -async def serve(fd): - """Accept on a descriptor Erlang passed over and answer one request.""" - import erlang - _servers[fd] = await erlang.server.serve(fd, _Echo) - return 'serving' - - -# --- the rest --------------------------------------------------------------- - -def getenv(name): - name = _text(name) - value = os.environ.get(name) - return None if value is None else value - - -def run_subprocess(): - subprocess.run(['true'], check=False) - return 'ran' - - -def shm_write(region, data): - data = _bytes(data) - region[0:len(data)] = data - return 'ok' - - -class _Fspath: - """A path object whose conversion tries to read outside every grant.""" - - def __init__(self, path): - self.path = path - self.leaked = None - - def __fspath__(self): - try: - with open('/etc/hosts'): - self.leaked = True - except Exception: - self.leaked = False - return self.path - - -def read_through_fspath(path): - """Did the conversion get an unchecked read? It must not.""" - obj = _Fspath(_text(path)) - try: - with open(obj): - pass - except Exception: - pass - return obj.leaked - - -def truncate_by_descriptor(path): - fd = os.open(_text(path), os.O_RDONLY) - try: - os.truncate(fd, 0) - return 'truncated' - finally: - os.close(fd) - - -def unix_connect(path): - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - try: - sock.connect(_text(path)) - return 'connected' - finally: - sock.close() - - -def unix_bind(path): - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - try: - sock.bind(_text(path)) - return 'bound' - finally: - sock.close()