From 85858e72dfdc3e941c2e620e94de05ad663138b1 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Wed, 29 Jan 2020 14:27:33 +1100 Subject: [PATCH] py/objexcept: Allow compression of exception message text. The decompression of error-strings is only done if the string is accessed via printing or via er.args. Tests are added for this feature to ensure the decompression works. --- py/compile.c | 2 +- py/emitinlinethumb.c | 2 +- py/nativeglue.h | 2 +- py/obj.h | 6 +- py/objexcept.c | 95 ++++++++++++++++--- py/runtime.c | 10 +- py/runtime.h | 10 +- tests/micropython/heapalloc_exc_compressed.py | 48 ++++++++++ .../heapalloc_exc_compressed.py.exp | 6 ++ .../heapalloc_exc_compressed_emg_exc.py | 41 ++++++++ .../heapalloc_exc_compressed_emg_exc.py.exp | 4 + tools/tinytest-codegen.py | 2 + 12 files changed, 198 insertions(+), 30 deletions(-) create mode 100644 tests/micropython/heapalloc_exc_compressed.py create mode 100644 tests/micropython/heapalloc_exc_compressed.py.exp create mode 100644 tests/micropython/heapalloc_exc_compressed_emg_exc.py create mode 100644 tests/micropython/heapalloc_exc_compressed_emg_exc.py.exp diff --git a/py/compile.c b/py/compile.c index 750cb9c3d8..5daeb6e18d 100644 --- a/py/compile.c +++ b/py/compile.c @@ -197,7 +197,7 @@ STATIC void compile_error_set_line(compiler_t *comp, mp_parse_node_t pn) { } } -STATIC void compile_syntax_error(compiler_t *comp, mp_parse_node_t pn, const char *msg) { +STATIC void compile_syntax_error(compiler_t *comp, mp_parse_node_t pn, mp_rom_error_text_t msg) { // only register the error if there has been no other error if (comp->compile_error == MP_OBJ_NULL) { comp->compile_error = mp_obj_new_exception_msg(&mp_type_SyntaxError, msg); diff --git a/py/emitinlinethumb.c b/py/emitinlinethumb.c index 358ccac336..7cf6f1cbb3 100644 --- a/py/emitinlinethumb.c +++ b/py/emitinlinethumb.c @@ -59,7 +59,7 @@ struct _emit_inline_asm_t { qstr *label_lookup; }; -STATIC void emit_inline_thumb_error_msg(emit_inline_asm_t *emit, const char *msg) { +STATIC void emit_inline_thumb_error_msg(emit_inline_asm_t *emit, mp_rom_error_text_t msg) { *emit->error_slot = mp_obj_new_exception_msg(&mp_type_SyntaxError, msg); } diff --git a/py/nativeglue.h b/py/nativeglue.h index 9e320eff04..908ae3f653 100644 --- a/py/nativeglue.h +++ b/py/nativeglue.h @@ -145,7 +145,7 @@ typedef struct _mp_fun_table_t { #if defined(__GNUC__) NORETURN // Only certain compilers support no-return attributes in function pointer declarations #endif - void (*raise_msg)(const mp_obj_type_t *exc_type, const char *msg); + void (*raise_msg)(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg); const mp_obj_type_t *(*obj_get_type)(mp_const_obj_t o_in); mp_obj_t (*obj_new_str)(const char *data, size_t len); mp_obj_t (*obj_new_bytes)(const byte *data, size_t len); diff --git a/py/obj.h b/py/obj.h index e3299cb8ee..969f3033b9 100644 --- a/py/obj.h +++ b/py/obj.h @@ -731,10 +731,10 @@ mp_obj_t mp_obj_new_complex(mp_float_t real, mp_float_t imag); mp_obj_t mp_obj_new_exception(const mp_obj_type_t *exc_type); mp_obj_t mp_obj_new_exception_arg1(const mp_obj_type_t *exc_type, mp_obj_t arg); mp_obj_t mp_obj_new_exception_args(const mp_obj_type_t *exc_type, size_t n_args, const mp_obj_t *args); -mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, const char *msg); -mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...); // counts args by number of % symbols in fmt, excluding %%; can only handle void* sizes (ie no float/double!) +mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg); +mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, ...); // counts args by number of % symbols in fmt, excluding %%; can only handle void* sizes (ie no float/double!) #ifdef va_start -mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const char *fmt, va_list arg); // same fmt restrictions as above +mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, va_list arg); // same fmt restrictions as above #endif mp_obj_t mp_obj_new_fun_bc(mp_obj_t def_args, mp_obj_t def_kw_args, const byte *code, const mp_uint_t *const_table); mp_obj_t mp_obj_new_fun_native(mp_obj_t def_args_in, mp_obj_t def_kw_args, const void *fun_data, const mp_uint_t *const_table); diff --git a/py/objexcept.c b/py/objexcept.c index ef99019d36..594c4c650f 100644 --- a/py/objexcept.c +++ b/py/objexcept.c @@ -38,6 +38,15 @@ #include "py/gc.h" #include "py/mperrno.h" +// Extract the MP_MAX_UNCOMPRESSED_TEXT_LEN macro from "genhdr/compressed.data.h" +#if MICROPY_ROM_TEXT_COMPRESSION +#define MP_MATCH_COMPRESSED(...) // Ignore +#define MP_COMPRESSED_DATA(...) // Ignore +#include "genhdr/compressed.data.h" +#undef MP_MATCH_COMPRESSED +#undef MP_COMPRESSED_DATA +#endif + // Number of items per traceback entry (file, line, block) #define TRACEBACK_ENTRY_LEN (3) @@ -57,6 +66,7 @@ #define EMG_BUF_TUPLE_OFFSET (EMG_BUF_TRACEBACK_OFFSET + EMG_BUF_TRACEBACK_SIZE) #define EMG_BUF_TUPLE_SIZE(n_args) (sizeof(mp_obj_tuple_t) + n_args * sizeof(mp_obj_t)) #define EMG_BUF_STR_OFFSET (EMG_BUF_TUPLE_OFFSET + EMG_BUF_TUPLE_SIZE(1)) +#define EMG_BUF_STR_BUF_OFFSET (EMG_BUF_STR_OFFSET + sizeof(mp_obj_str_t)) #if MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE > 0 #define mp_emergency_exception_buf_size MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE @@ -100,6 +110,40 @@ mp_obj_t mp_alloc_emergency_exception_buf(mp_obj_t size_in) { #endif #endif // MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF +STATIC void decompress_error_text_maybe(mp_obj_exception_t *o) { + #if MICROPY_ROM_TEXT_COMPRESSION + if (o->args->len == 1 && mp_obj_is_type(o->args->items[0], &mp_type_str)) { + mp_obj_str_t *o_str = MP_OBJ_TO_PTR(o->args->items[0]); + if (MP_IS_COMPRESSED_ROM_STRING(o_str->data)) { + byte *buf = m_new_maybe(byte, MP_MAX_UNCOMPRESSED_TEXT_LEN + 1); + if (!buf) { + #if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF + // Try and use the emergency exception buf if enough space is available. + buf = (byte *)((uint8_t *)MP_STATE_VM(mp_emergency_exception_buf) + EMG_BUF_STR_BUF_OFFSET); + size_t avail = (uint8_t *)MP_STATE_VM(mp_emergency_exception_buf) + mp_emergency_exception_buf_size - buf; + if (avail < MP_MAX_UNCOMPRESSED_TEXT_LEN + 1) { + // No way to decompress, fallback to no message text. + o->args = (mp_obj_tuple_t *)&mp_const_empty_tuple_obj; + return; + } + #else + o->args = (mp_obj_tuple_t *)&mp_const_empty_tuple_obj; + return; + #endif + } + mp_decompress_rom_string(buf, (mp_rom_error_text_t)o_str->data); + o_str->data = buf; + o_str->len = strlen((const char *)buf); + o_str->hash = 0; + } + // Lazily compute the string hash. + if (o_str->hash == 0) { + o_str->hash = qstr_compute_hash(o_str->data, o_str->len); + } + } + #endif +} + void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { mp_obj_exception_t *o = MP_OBJ_TO_PTR(o_in); mp_print_kind_t k = kind & ~PRINT_EXC_SUBCLASS; @@ -112,6 +156,8 @@ void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kin mp_print_str(print, ": "); } + decompress_error_text_maybe(o); + if (k == PRINT_STR || k == PRINT_EXC) { if (o->args == NULL || o->args->len == 0) { mp_print_str(print, ""); @@ -131,6 +177,7 @@ void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kin return; } } + mp_obj_tuple_print(print, MP_OBJ_FROM_PTR(o->args), kind); } @@ -189,6 +236,7 @@ mp_obj_t mp_obj_exception_get_value(mp_obj_t self_in) { if (self->args->len == 0) { return mp_const_none; } else { + decompress_error_text_maybe(self); return self->args->items[0]; } } @@ -210,6 +258,7 @@ void mp_obj_exception_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { return; } if (attr == MP_QSTR_args) { + decompress_error_text_maybe(self); dest[0] = MP_OBJ_FROM_PTR(self->args); } else if (self->base.type == &mp_type_StopIteration && attr == MP_QSTR_value) { dest[0] = mp_obj_exception_get_value(self_in); @@ -323,7 +372,7 @@ mp_obj_t mp_obj_new_exception_args(const mp_obj_type_t *exc_type, size_t n_args, return mp_obj_exception_make_new(exc_type, n_args, 0, args); } -mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, const char *msg) { +mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg) { // Check that the given type is an exception type assert(exc_type->make_new == mp_obj_exception_make_new); @@ -348,9 +397,13 @@ mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, const char *msg // Create the string object and call mp_obj_exception_make_new to create the exception o_str->base.type = &mp_type_str; - o_str->len = strlen(msg); + o_str->len = strlen((const char *)msg); o_str->data = (const byte *)msg; + #if MICROPY_ROM_TEXT_COMPRESSION + o_str->hash = 0; // will be computed only if string object is accessed + #else o_str->hash = qstr_compute_hash(o_str->data, o_str->len); + #endif mp_obj_t arg = MP_OBJ_FROM_PTR(o_str); return mp_obj_exception_make_new(exc_type, 1, 0, &arg); } @@ -388,7 +441,7 @@ STATIC void exc_add_strn(void *data, const char *str, size_t len) { pr->len += len; } -mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...) { +mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, ...) { va_list args; va_start(args, fmt); mp_obj_t exc = mp_obj_new_exception_msg_vlist(exc_type, fmt, args); @@ -396,7 +449,7 @@ mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, const char return exc; } -mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const char *fmt, va_list args) { +mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, va_list args) { assert(fmt != NULL); // Check that the given type is an exception type @@ -404,7 +457,7 @@ mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const cha // Try to allocate memory for the message mp_obj_str_t *o_str = m_new_obj_maybe(mp_obj_str_t); - size_t o_str_alloc = strlen(fmt) + 1; + size_t o_str_alloc = strlen((const char *)fmt) + 1; byte *o_str_buf = m_new_maybe(byte, o_str_alloc); bool used_emg_buf = false; @@ -415,29 +468,39 @@ mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const cha if ((o_str == NULL || o_str_buf == NULL) && mp_emergency_exception_buf_size >= EMG_BUF_STR_OFFSET + sizeof(mp_obj_str_t) + 16) { used_emg_buf = true; - o_str = (mp_obj_str_t *)((uint8_t *)MP_STATE_VM(mp_emergency_exception_buf) - + EMG_BUF_STR_OFFSET); - o_str_buf = (byte *)&o_str[1]; - o_str_alloc = (uint8_t *)MP_STATE_VM(mp_emergency_exception_buf) - + mp_emergency_exception_buf_size - o_str_buf; + o_str = (mp_obj_str_t *)((uint8_t *)MP_STATE_VM(mp_emergency_exception_buf) + EMG_BUF_STR_OFFSET); + o_str_buf = (byte *)((uint8_t *)MP_STATE_VM(mp_emergency_exception_buf) + EMG_BUF_STR_BUF_OFFSET); + o_str_alloc = (uint8_t *)MP_STATE_VM(mp_emergency_exception_buf) + mp_emergency_exception_buf_size - o_str_buf; } #endif if (o_str == NULL) { - // No memory for the string object so create the exception with no args + // No memory for the string object so create the exception with no args. + // The exception will only have a type and no message (compression is irrelevant). return mp_obj_exception_make_new(exc_type, 0, 0, NULL); } if (o_str_buf == NULL) { // No memory for the string buffer: assume that the fmt string is in ROM - // and use that data as the data of the string + // and use that data as the data of the string. + // The string will point directly to the compressed data -- will need to be decompressed + // prior to display (this case is identical to mp_obj_new_exception_msg above). o_str->len = o_str_alloc - 1; // will be equal to strlen(fmt) o_str->data = (const byte *)fmt; } else { - // We have some memory to format the string + // We have some memory to format the string. + // TODO: Optimise this to format-while-decompressing (and not require the temp stack space). struct _exc_printer_t exc_pr = {!used_emg_buf, o_str_alloc, 0, o_str_buf}; mp_print_t print = {&exc_pr, exc_add_strn}; - mp_vprintf(&print, fmt, args); + const char *fmt2 = (const char *)fmt; + #if MICROPY_ROM_TEXT_COMPRESSION + byte decompressed[MP_MAX_UNCOMPRESSED_TEXT_LEN]; + if (MP_IS_COMPRESSED_ROM_STRING(fmt)) { + mp_decompress_rom_string(decompressed, fmt); + fmt2 = (const char *)decompressed; + } + #endif + mp_vprintf(&print, fmt2, args); exc_pr.buf[exc_pr.len] = '\0'; o_str->len = exc_pr.len; o_str->data = exc_pr.buf; @@ -445,7 +508,11 @@ mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const cha // Create the string object and call mp_obj_exception_make_new to create the exception o_str->base.type = &mp_type_str; + #if MICROPY_ROM_TEXT_COMPRESSION + o_str->hash = 0; // will be computed only if string object is accessed + #else o_str->hash = qstr_compute_hash(o_str->data, o_str->len); + #endif mp_obj_t arg = MP_OBJ_FROM_PTR(o_str); return mp_obj_exception_make_new(exc_type, 1, 0, &arg); } diff --git a/py/runtime.c b/py/runtime.c index 647ff5d98f..49863631a3 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -1500,7 +1500,7 @@ NORETURN void m_malloc_fail(size_t num_bytes) { "memory allocation failed, allocating %u bytes", (uint)num_bytes); } -NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, const char *msg) { +NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg) { if (msg == NULL) { nlr_raise(mp_obj_new_exception(exc_type)); } else { @@ -1508,7 +1508,7 @@ NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, const char *msg) { } } -NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...) { +NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, ...) { va_list args; va_start(args, fmt); mp_obj_t exc = mp_obj_new_exception_msg_vlist(exc_type, fmt, args); @@ -1516,11 +1516,11 @@ NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, nlr_raise(exc); } -NORETURN void mp_raise_ValueError(const char *msg) { +NORETURN void mp_raise_ValueError(mp_rom_error_text_t msg) { mp_raise_msg(&mp_type_ValueError, msg); } -NORETURN void mp_raise_TypeError(const char *msg) { +NORETURN void mp_raise_TypeError(mp_rom_error_text_t msg) { mp_raise_msg(&mp_type_TypeError, msg); } @@ -1528,7 +1528,7 @@ NORETURN void mp_raise_OSError(int errno_) { nlr_raise(mp_obj_new_exception_arg1(&mp_type_OSError, MP_OBJ_NEW_SMALL_INT(errno_))); } -NORETURN void mp_raise_NotImplementedError(const char *msg) { +NORETURN void mp_raise_NotImplementedError(mp_rom_error_text_t msg) { mp_raise_msg(&mp_type_NotImplementedError, msg); } diff --git a/py/runtime.h b/py/runtime.h index 85c377f556..53aed4429c 100644 --- a/py/runtime.h +++ b/py/runtime.h @@ -162,11 +162,11 @@ mp_obj_t mp_import_from(mp_obj_t module, qstr name); void mp_import_all(mp_obj_t module); #define mp_raise_type(exc_type) mp_raise_msg(exc_type, NULL) -NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, const char *msg); -NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...); -NORETURN void mp_raise_ValueError(const char *msg); -NORETURN void mp_raise_TypeError(const char *msg); -NORETURN void mp_raise_NotImplementedError(const char *msg); +NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, mp_rom_error_text_t msg); +NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, ...); +NORETURN void mp_raise_ValueError(mp_rom_error_text_t msg); +NORETURN void mp_raise_TypeError(mp_rom_error_text_t msg); +NORETURN void mp_raise_NotImplementedError(mp_rom_error_text_t msg); NORETURN void mp_raise_OSError(int errno_); NORETURN void mp_raise_recursion_depth(void); diff --git a/tests/micropython/heapalloc_exc_compressed.py b/tests/micropython/heapalloc_exc_compressed.py new file mode 100644 index 0000000000..79e423ca0f --- /dev/null +++ b/tests/micropython/heapalloc_exc_compressed.py @@ -0,0 +1,48 @@ +import micropython + +# Tests both code paths for built-in exception raising. +# mp_obj_new_exception_msg_varg (exception requires decompression at raise-time to format) +# mp_obj_new_exception_msg (decompression can be deferred) + +# NameError uses mp_obj_new_exception_msg_varg for NameError("name '%q' isn't defined") +# set.pop uses mp_obj_new_exception_msg for KeyError("pop from an empty set") + +# Tests that deferred decompression works both via print(e) and accessing the message directly via e.args. + +a = set() + +# First test the regular case (can use heap for allocating the decompression buffer). +try: + name() +except NameError as e: + print(type(e).__name__, e) + +try: + a.pop() +except KeyError as e: + print(type(e).__name__, e) + +try: + name() +except NameError as e: + print(e.args[0]) + +try: + a.pop() +except KeyError as e: + print(e.args[0]) + +# Then test that it still works when the heap is locked (i.e. in ISR context). +micropython.heap_lock() + +try: + name() +except NameError as e: + print(type(e).__name__) + +try: + a.pop() +except KeyError as e: + print(type(e).__name__) + +micropython.heap_unlock() diff --git a/tests/micropython/heapalloc_exc_compressed.py.exp b/tests/micropython/heapalloc_exc_compressed.py.exp new file mode 100644 index 0000000000..32d1642f86 --- /dev/null +++ b/tests/micropython/heapalloc_exc_compressed.py.exp @@ -0,0 +1,6 @@ +NameError name 'name' isn't defined +KeyError pop from an empty set +name 'name' isn't defined +pop from an empty set +NameError +KeyError diff --git a/tests/micropython/heapalloc_exc_compressed_emg_exc.py b/tests/micropython/heapalloc_exc_compressed_emg_exc.py new file mode 100644 index 0000000000..86ade07862 --- /dev/null +++ b/tests/micropython/heapalloc_exc_compressed_emg_exc.py @@ -0,0 +1,41 @@ +import micropython + +# Does the full test from heapalloc_exc_compressed.py but while the heap is +# locked (this can only work when the emergency exception buf is enabled). + +# Some ports need to allocate heap for the emgergency exception buffer. +try: + micropython.alloc_emergency_exception_buf(256) +except AttributeError: + pass + +a = set() + + +def test(): + micropython.heap_lock() + + try: + name() + except NameError as e: + print(type(e).__name__, e) + + try: + a.pop() + except KeyError as e: + print(type(e).__name__, e) + + try: + name() + except NameError as e: + print(e.args[0]) + + try: + a.pop() + except KeyError as e: + print(e.args[0]) + + micropython.heap_unlock() + + +test() diff --git a/tests/micropython/heapalloc_exc_compressed_emg_exc.py.exp b/tests/micropython/heapalloc_exc_compressed_emg_exc.py.exp new file mode 100644 index 0000000000..7c368712a2 --- /dev/null +++ b/tests/micropython/heapalloc_exc_compressed_emg_exc.py.exp @@ -0,0 +1,4 @@ +NameError name 'name' isn't defined +KeyError pop from an empty set +name 'name' isn't defined +pop from an empty set diff --git a/tools/tinytest-codegen.py b/tools/tinytest-codegen.py index 64f88edb30..116a217b4b 100755 --- a/tools/tinytest-codegen.py +++ b/tools/tinytest-codegen.py @@ -89,6 +89,8 @@ exclude_tests = ( # different filename in output "micropython/emg_exc.py", "micropython/heapalloc_traceback.py", + # don't have emergency exception buffer + "micropython/heapalloc_exc_compressed_emg_exc.py", # pattern matching in .exp "micropython/meminfo.py", # needs sys stdfiles