From 9b090603a04e69ec710e14f6c8bd93011516c5a1 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 24 Jun 2023 17:19:05 +1000 Subject: [PATCH] webassembly: Implement runPythonAsync() for top-level async code. With this commit, `interpreter.runPythonAsync(code)` can now be used to run Python code that uses `await` at the top level. That will yield up to JavaScript and produce a thenable, which the JavaScript runtime can then resume. Also implemented is the ability for Python code to await on JavaScript promises/thenables. For example, outer JavaScript code can await on `runPythonAsync(code)` which then runs Python code that does `await js.fetch(url)`. The entire chain of calls will be suspended until the fetch completes. Signed-off-by: Damien George --- ports/webassembly/Makefile | 2 + ports/webassembly/api.js | 10 ++ ports/webassembly/main.c | 6 ++ ports/webassembly/mpconfigport.h | 1 + ports/webassembly/objjsproxy.c | 163 ++++++++++++++++++++++++++++++- ports/webassembly/objpyproxy.js | 23 +++++ ports/webassembly/proxy_c.c | 83 +++++++++++++++- ports/webassembly/proxy_js.js | 10 +- 8 files changed, 291 insertions(+), 7 deletions(-) diff --git a/ports/webassembly/Makefile b/ports/webassembly/Makefile index 2a5669392e..05e0c1f0f9 100644 --- a/ports/webassembly/Makefile +++ b/ports/webassembly/Makefile @@ -47,6 +47,7 @@ CFLAGS += $(INC) EXPORTED_FUNCTIONS_EXTRA += ,\ _mp_js_do_exec,\ + _mp_js_do_exec_async,\ _mp_js_do_import,\ _mp_js_register_js_module,\ _proxy_c_init,\ @@ -58,6 +59,7 @@ EXPORTED_FUNCTIONS_EXTRA += ,\ _proxy_c_to_js_get_type,\ _proxy_c_to_js_has_attr,\ _proxy_c_to_js_lookup_attr,\ + _proxy_c_to_js_resume,\ _proxy_c_to_js_store_attr,\ _proxy_convert_mp_to_js_obj_cside diff --git a/ports/webassembly/api.js b/ports/webassembly/api.js index dfe7561768..ec0601c612 100644 --- a/ports/webassembly/api.js +++ b/ports/webassembly/api.js @@ -140,6 +140,16 @@ export async function loadMicroPython(options) { ); return proxy_convert_mp_to_js_obj_jsside_with_free(value); }, + runPythonAsync(code) { + const value = Module._malloc(3 * 4); + Module.ccall( + "mp_js_do_exec_async", + "number", + ["string", "pointer"], + [code, value], + ); + return proxy_convert_mp_to_js_obj_jsside_with_free(value); + }, }; } diff --git a/ports/webassembly/main.c b/ports/webassembly/main.c index c1c7a88840..5bb4222aaa 100644 --- a/ports/webassembly/main.c +++ b/ports/webassembly/main.c @@ -169,6 +169,12 @@ void mp_js_do_exec(const char *src, uint32_t *out) { } } +void mp_js_do_exec_async(const char *src, uint32_t *out) { + mp_compile_allow_top_level_await = true; + mp_js_do_exec(src, out); + mp_compile_allow_top_level_await = false; +} + #if MICROPY_GC_SPLIT_HEAP_AUTO // The largest new region that is available to become Python heap. diff --git a/ports/webassembly/mpconfigport.h b/ports/webassembly/mpconfigport.h index fc7ba2f82d..ae5dfa6fa5 100644 --- a/ports/webassembly/mpconfigport.h +++ b/ports/webassembly/mpconfigport.h @@ -39,6 +39,7 @@ #endif #define MICROPY_ALLOC_PATH_MAX (256) +#define MICROPY_COMP_ALLOW_TOP_LEVEL_AWAIT (1) #define MICROPY_READER_VFS (MICROPY_VFS) #define MICROPY_ENABLE_GC (1) #define MICROPY_ENABLE_PYSTACK (1) diff --git a/ports/webassembly/objjsproxy.c b/ports/webassembly/objjsproxy.c index a28b791cf2..5e2aeb6a36 100644 --- a/ports/webassembly/objjsproxy.c +++ b/ports/webassembly/objjsproxy.c @@ -32,6 +32,16 @@ #include "py/runtime.h" #include "proxy_c.h" +EM_JS(bool, has_attr, (int jsref, const char *str), { + const base = proxy_js_ref[jsref]; + const attr = UTF8ToString(str); + if (attr in base) { + return true; + } else { + return false; + } +}); + // *FORMAT-OFF* EM_JS(bool, lookup_attr, (int jsref, const char *str, uint32_t * out), { const base = proxy_js_ref[jsref]; @@ -299,19 +309,166 @@ static mp_obj_t jsproxy_it_iternext(mp_obj_t self_in) { } } -static mp_obj_t jsproxy_getiter(mp_obj_t o_in, mp_obj_iter_buf_t *iter_buf) { +static mp_obj_t jsproxy_new_it(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { assert(sizeof(jsproxy_it_t) <= sizeof(mp_obj_iter_buf_t)); + mp_obj_jsproxy_t *self = MP_OBJ_TO_PTR(self_in); jsproxy_it_t *o = (jsproxy_it_t *)iter_buf; o->base.type = &mp_type_polymorph_iter; o->iternext = jsproxy_it_iternext; - o->ref = mp_obj_jsproxy_get_ref(o_in); + o->ref = self->ref; o->cur = 0; - o->len = js_get_len(o->ref); + o->len = js_get_len(self->ref); + return MP_OBJ_FROM_PTR(o); +} + +/******************************************************************************/ +// jsproxy generator + +enum { + JSOBJ_GEN_STATE_WAITING, + JSOBJ_GEN_STATE_COMPLETED, + JSOBJ_GEN_STATE_EXHAUSTED, +}; + +typedef struct _jsproxy_gen_t { + mp_obj_base_t base; + mp_obj_t thenable; + int state; +} jsproxy_gen_t; + +mp_vm_return_kind_t jsproxy_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t throw_value, mp_obj_t *ret_val) { + jsproxy_gen_t *self = MP_OBJ_TO_PTR(self_in); + switch (self->state) { + case JSOBJ_GEN_STATE_WAITING: + self->state = JSOBJ_GEN_STATE_COMPLETED; + *ret_val = self->thenable; + return MP_VM_RETURN_YIELD; + + case JSOBJ_GEN_STATE_COMPLETED: + self->state = JSOBJ_GEN_STATE_EXHAUSTED; + *ret_val = send_value; + return MP_VM_RETURN_NORMAL; + + case JSOBJ_GEN_STATE_EXHAUSTED: + default: + // Trying to resume an already stopped generator. + // This is an optimised "raise StopIteration(None)". + *ret_val = mp_const_none; + return MP_VM_RETURN_NORMAL; + } +} + +static mp_obj_t jsproxy_gen_resume_and_raise(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t throw_value, bool raise_stop_iteration) { + mp_obj_t ret; + switch (jsproxy_gen_resume(self_in, send_value, throw_value, &ret)) { + case MP_VM_RETURN_NORMAL: + default: + // A normal return is a StopIteration, either raise it or return + // MP_OBJ_STOP_ITERATION as an optimisation. + if (ret == mp_const_none) { + ret = MP_OBJ_NULL; + } + if (raise_stop_iteration) { + mp_raise_StopIteration(ret); + } else { + return mp_make_stop_iteration(ret); + } + + case MP_VM_RETURN_YIELD: + return ret; + + case MP_VM_RETURN_EXCEPTION: + nlr_raise(ret); + } +} + +static mp_obj_t jsproxy_gen_instance_iternext(mp_obj_t self_in) { + return jsproxy_gen_resume_and_raise(self_in, mp_const_none, MP_OBJ_NULL, false); +} + +static mp_obj_t jsproxy_gen_instance_send(mp_obj_t self_in, mp_obj_t send_value) { + return jsproxy_gen_resume_and_raise(self_in, send_value, MP_OBJ_NULL, true); +} +static MP_DEFINE_CONST_FUN_OBJ_2(jsproxy_gen_instance_send_obj, jsproxy_gen_instance_send); + +static mp_obj_t jsproxy_gen_instance_throw(size_t n_args, const mp_obj_t *args) { + // The signature of this function is: throw(type[, value[, traceback]]) + // CPython will pass all given arguments through the call chain and process them + // at the point they are used (native generators will handle them differently to + // user-defined generators with a throw() method). To save passing multiple + // values, MicroPython instead does partial processing here to reduce it down to + // one argument and passes that through: + // - if only args[1] is given, or args[2] is given but is None, args[1] is + // passed through (in the standard case it is an exception class or instance) + // - if args[2] is given and not None it is passed through (in the standard + // case it would be an exception instance and args[1] its corresponding class) + // - args[3] is always ignored + + mp_obj_t exc = args[1]; + if (n_args > 2 && args[2] != mp_const_none) { + exc = args[2]; + } + + return jsproxy_gen_resume_and_raise(args[0], mp_const_none, exc, true); +} +static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(jsproxy_gen_instance_throw_obj, 2, 4, jsproxy_gen_instance_throw); + +static mp_obj_t jsproxy_gen_instance_close(mp_obj_t self_in) { + mp_obj_t ret; + switch (jsproxy_gen_resume(self_in, mp_const_none, MP_OBJ_FROM_PTR(&mp_const_GeneratorExit_obj), &ret)) { + case MP_VM_RETURN_YIELD: + mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("generator ignored GeneratorExit")); + + // Swallow GeneratorExit (== successful close), and re-raise any other + case MP_VM_RETURN_EXCEPTION: + // ret should always be an instance of an exception class + if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(ret)), MP_OBJ_FROM_PTR(&mp_type_GeneratorExit))) { + return mp_const_none; + } + nlr_raise(ret); + + default: + // The only choice left is MP_VM_RETURN_NORMAL which is successful close + return mp_const_none; + } +} +static MP_DEFINE_CONST_FUN_OBJ_1(jsproxy_gen_instance_close_obj, jsproxy_gen_instance_close); + +static const mp_rom_map_elem_t jsproxy_gen_instance_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&jsproxy_gen_instance_close_obj) }, + { MP_ROM_QSTR(MP_QSTR_send), MP_ROM_PTR(&jsproxy_gen_instance_send_obj) }, + { MP_ROM_QSTR(MP_QSTR_throw), MP_ROM_PTR(&jsproxy_gen_instance_throw_obj) }, +}; +static MP_DEFINE_CONST_DICT(jsproxy_gen_instance_locals_dict, jsproxy_gen_instance_locals_dict_table); + +MP_DEFINE_CONST_OBJ_TYPE( + mp_type_jsproxy_gen, + MP_QSTR_generator, + MP_TYPE_FLAG_ITER_IS_ITERNEXT, + iter, jsproxy_gen_instance_iternext, + locals_dict, &jsproxy_gen_instance_locals_dict + ); + +static mp_obj_t jsproxy_new_gen(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { + assert(sizeof(jsproxy_gen_t) <= sizeof(mp_obj_iter_buf_t)); + jsproxy_gen_t *o = (jsproxy_gen_t *)iter_buf; + o->base.type = &mp_type_jsproxy_gen; + o->thenable = self_in; + o->state = JSOBJ_GEN_STATE_WAITING; return MP_OBJ_FROM_PTR(o); } /******************************************************************************/ +static mp_obj_t jsproxy_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { + mp_obj_jsproxy_t *self = MP_OBJ_TO_PTR(self_in); + if (has_attr(self->ref, "then")) { + return jsproxy_new_gen(self_in, iter_buf); + } else { + return jsproxy_new_it(self_in, iter_buf); + } +} + MP_DEFINE_CONST_OBJ_TYPE( mp_type_jsproxy, MP_QSTR_JsProxy, diff --git a/ports/webassembly/objpyproxy.js b/ports/webassembly/objpyproxy.js index 52670b66e9..9ba06283ea 100644 --- a/ports/webassembly/objpyproxy.js +++ b/ports/webassembly/objpyproxy.js @@ -159,6 +159,9 @@ const py_proxy_handler = { if (prop === "_ref") { return target._ref; } + if (prop === "then") { + return null; + } const value = Module._malloc(3 * 4); Module.ccall( "proxy_c_to_js_lookup_attr", @@ -189,3 +192,23 @@ const py_proxy_handler = { ); }, }; + +// PyProxy of a Python generator, that implements the thenable interface. +class PyProxyThenable { + constructor(ref) { + this._ref = ref; + } + + then(resolve, reject) { + const values = Module._malloc(3 * 3 * 4); + proxy_convert_js_to_mp_obj_jsside(resolve, values + 3 * 4); + proxy_convert_js_to_mp_obj_jsside(reject, values + 2 * 3 * 4); + Module.ccall( + "proxy_c_to_js_resume", + "null", + ["number", "pointer"], + [this._ref, values], + ); + return proxy_convert_mp_to_js_obj_jsside_with_free(values); + } +} diff --git a/ports/webassembly/proxy_c.c b/ports/webassembly/proxy_c.c index 809dd44dde..1e4573ce0b 100644 --- a/ports/webassembly/proxy_c.c +++ b/ports/webassembly/proxy_c.c @@ -27,6 +27,7 @@ #include #include +#include "emscripten.h" #include "py/builtin.h" #include "py/runtime.h" #include "proxy_c.h" @@ -42,8 +43,9 @@ enum { PROXY_KIND_MP_FLOAT = 4, PROXY_KIND_MP_STR = 5, PROXY_KIND_MP_CALLABLE = 6, - PROXY_KIND_MP_OBJECT = 7, - PROXY_KIND_MP_JSPROXY = 8, + PROXY_KIND_MP_GENERATOR = 7, + PROXY_KIND_MP_OBJECT = 8, + PROXY_KIND_MP_JSPROXY = 9, }; enum { @@ -115,6 +117,8 @@ void proxy_convert_mp_to_js_obj_cside(mp_obj_t obj, uint32_t *out) { } else { if (mp_obj_is_callable(obj)) { kind = PROXY_KIND_MP_CALLABLE; + } else if (mp_obj_is_type(obj, &mp_type_gen_instance)) { + kind = PROXY_KIND_MP_GENERATOR; } else { kind = PROXY_KIND_MP_OBJECT; } @@ -279,3 +283,78 @@ void proxy_c_to_js_get_dict(uint32_t c_ref, uint32_t *out) { out[0] = map->alloc; out[1] = (uintptr_t)map->table; } + +/******************************************************************************/ +// Bridge Python generator to JavaScript thenable. + +static const mp_obj_fun_builtin_var_t resume_obj; + +EM_JS(void, js_then_resolve, (uint32_t * resolve, uint32_t * reject), { + const resolve_js = proxy_convert_mp_to_js_obj_jsside(resolve); + const reject_js = proxy_convert_mp_to_js_obj_jsside(reject); + resolve_js(null); +}); + +EM_JS(void, js_then_reject, (uint32_t * resolve, uint32_t * reject), { + const resolve_js = proxy_convert_mp_to_js_obj_jsside(resolve); + const reject_js = proxy_convert_mp_to_js_obj_jsside(reject); + reject_js(null); +}); + +// *FORMAT-OFF* +EM_JS(void, js_then_continue, (int jsref, uint32_t * py_resume, uint32_t * resolve, uint32_t * reject, uint32_t * out), { + const py_resume_js = proxy_convert_mp_to_js_obj_jsside(py_resume); + const resolve_js = proxy_convert_mp_to_js_obj_jsside(resolve); + const reject_js = proxy_convert_mp_to_js_obj_jsside(reject); + const ret = proxy_js_ref[jsref].then((x) => {py_resume_js(x, resolve_js, reject_js);}, reject_js); + proxy_convert_js_to_mp_obj_jsside(ret, out); +}); +// *FORMAT-ON* + +static mp_obj_t proxy_resume_execute(mp_obj_t self_in, mp_obj_t value, mp_obj_t resolve, mp_obj_t reject) { + mp_obj_t ret_value; + mp_vm_return_kind_t ret_kind = mp_resume(self_in, value, MP_OBJ_NULL, &ret_value); + + uint32_t out_resolve[PVN]; + uint32_t out_reject[PVN]; + proxy_convert_mp_to_js_obj_cside(resolve, out_resolve); + proxy_convert_mp_to_js_obj_cside(reject, out_reject); + + if (ret_kind == MP_VM_RETURN_NORMAL) { + js_then_resolve(out_resolve, out_reject); + return mp_const_none; + } else if (ret_kind == MP_VM_RETURN_YIELD) { + // ret_value should be a JS thenable + mp_obj_t py_resume = mp_obj_new_bound_meth(MP_OBJ_FROM_PTR(&resume_obj), self_in); + int ref = mp_obj_jsproxy_get_ref(ret_value); + uint32_t out_py_resume[PVN]; + proxy_convert_mp_to_js_obj_cside(py_resume, out_py_resume); + uint32_t out[PVN]; + js_then_continue(ref, out_py_resume, out_resolve, out_reject, out); + return proxy_convert_js_to_mp_obj_cside(out); + } else { + // MP_VM_RETURN_EXCEPTION; + js_then_reject(out_resolve, out_reject); + nlr_raise(ret_value); + } +} + +static mp_obj_t resume_fun(size_t n_args, const mp_obj_t *args) { + return proxy_resume_execute(args[0], args[1], args[2], args[3]); +} +static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(resume_obj, 4, 4, resume_fun); + +void proxy_c_to_js_resume(uint32_t c_ref, uint32_t *args) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + mp_obj_t obj = proxy_c_get_obj(c_ref); + mp_obj_t resolve = proxy_convert_js_to_mp_obj_cside(args + 1 * 3); + mp_obj_t reject = proxy_convert_js_to_mp_obj_cside(args + 2 * 3); + mp_obj_t ret = proxy_resume_execute(obj, mp_const_none, resolve, reject); + nlr_pop(); + return proxy_convert_mp_to_js_obj_cside(ret, args); + } else { + // uncaught exception + return proxy_convert_mp_to_js_exc_cside(nlr.ret_val, args); + } +} diff --git a/ports/webassembly/proxy_js.js b/ports/webassembly/proxy_js.js index 1835bdfdfa..7a0a1bbe89 100644 --- a/ports/webassembly/proxy_js.js +++ b/ports/webassembly/proxy_js.js @@ -34,8 +34,9 @@ const PROXY_KIND_MP_INT = 3; const PROXY_KIND_MP_FLOAT = 4; const PROXY_KIND_MP_STR = 5; const PROXY_KIND_MP_CALLABLE = 6; -const PROXY_KIND_MP_OBJECT = 7; -const PROXY_KIND_MP_JSPROXY = 8; +const PROXY_KIND_MP_GENERATOR = 7; +const PROXY_KIND_MP_OBJECT = 8; +const PROXY_KIND_MP_JSPROXY = 9; const PROXY_KIND_JS_NULL = 1; const PROXY_KIND_JS_BOOLEAN = 2; @@ -122,6 +123,9 @@ function proxy_convert_js_to_mp_obj_jsside(js_obj, out) { } else if (js_obj instanceof PyProxy) { kind = PROXY_KIND_JS_PYPROXY; Module.setValue(out + 4, js_obj._ref, "i32"); + } else if (js_obj instanceof PyProxyThenable) { + kind = PROXY_KIND_JS_PYPROXY; + Module.setValue(out + 4, js_obj._ref, "i32"); } else { kind = PROXY_KIND_JS_OBJECT; const id = proxy_js_ref.length; @@ -193,6 +197,8 @@ function proxy_convert_mp_to_js_obj_jsside(value) { obj = (...args) => { return proxy_call_python(id, args); }; + } else if (kind === PROXY_KIND_MP_GENERATOR) { + obj = new PyProxyThenable(id); } else { // PROXY_KIND_MP_OBJECT const target = new PyProxy(id);