From 7e14680a83525bf0822ef9cab899a5625496d662 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Mon, 26 Sep 2022 11:02:31 +1000 Subject: [PATCH] py/objringio: Add micropython.RingIO() interface for general use. This commit adds a new `RingIO` type which exposes the internal ring-buffer code for general use in Python programs. It has the stream interface making it similar to `StringIO` and `BytesIO`, except `RingIO` has a fixed buffer size and is automatically safe when reads and writes are in different threads or an IRQ. This new type is enabled at the "extra features" ROM level. Signed-off-by: Andrew Leech --- docs/library/micropython.rst | 68 +++++++++++++ ports/unix/coverage.c | 18 ++++ py/modmicropython.c | 3 + py/mpconfig.h | 5 + py/obj.h | 1 + py/objringio.c | 130 +++++++++++++++++++++++++ py/py.cmake | 1 + py/py.mk | 1 + py/ringbuf.c | 26 +---- py/ringbuf.h | 37 ++++++- tests/micropython/ringio.py | 48 +++++++++ tests/micropython/ringio.py.exp | 16 +++ tests/micropython/ringio_async.py | 36 +++++++ tests/micropython/ringio_async.py.exp | 4 + tests/ports/unix/extra_coverage.py.exp | 5 + 15 files changed, 372 insertions(+), 27 deletions(-) create mode 100644 py/objringio.c create mode 100644 tests/micropython/ringio.py create mode 100644 tests/micropython/ringio.py.exp create mode 100644 tests/micropython/ringio_async.py create mode 100644 tests/micropython/ringio_async.py.exp diff --git a/docs/library/micropython.rst b/docs/library/micropython.rst index 31b24903f1..4d5a064a7a 100644 --- a/docs/library/micropython.rst +++ b/docs/library/micropython.rst @@ -155,3 +155,71 @@ Functions There is a finite queue to hold the scheduled functions and `schedule()` will raise a `RuntimeError` if the queue is full. + +Classes +------- + +.. class:: RingIO(size) +.. class:: RingIO(buffer) + :noindex: + + Provides a fixed-size ringbuffer for bytes with a stream interface. Can be + considered like a fifo queue variant of `io.BytesIO`. + + When created with integer size a suitable buffer will be allocated. + Alternatively a `bytearray` or similar buffer protocol object can be provided + to the constructor for in-place use. + + The classic ringbuffer algorithm is used which allows for any size buffer + to be used however one byte will be consumed for tracking. If initialised + with an integer size this will be accounted for, for example ``RingIO(16)`` + will allocate a 17 byte buffer internally so it can hold 16 bytes of data. + When passing in a pre-allocated buffer however one byte less than its + original length will be available for storage, eg. ``RingIO(bytearray(16))`` + will only hold 15 bytes of data. + + A RingIO instance can be IRQ / thread safe when used to pass data in a single + direction eg. when written to in an IRQ and read from in a non-IRQ function + (or vice versa). This does not hold if you try to eg. write to a single instance + from both IRQ and non-IRQ code, this would often cause data corruption. + + .. method:: RingIO.any() + + Returns an integer counting the number of characters that can be read. + + .. method:: RingIO.read([nbytes]) + + Read available characters. This is a non-blocking function. If ``nbytes`` + is specified then read at most that many bytes, otherwise read as much + data as possible. + + Return value: a bytes object containing the bytes read. Will be + zero-length bytes object if no data is available. + + .. method:: RingIO.readline([nbytes]) + + Read a line, ending in a newline character or return if one exists in + the buffer, else return available bytes in buffer. If ``nbytes`` is + specified then read at most that many bytes. + + Return value: a bytes object containing the line read. + + .. method:: RingIO.readinto(buf[, nbytes]) + + Read available bytes into the provided ``buf``. If ``nbytes`` is + specified then read at most that many bytes. Otherwise, read at + most ``len(buf)`` bytes. + + Return value: Integer count of the number of bytes read into ``buf``. + + .. method:: RingIO.write(buf) + + Non-blocking write of bytes from ``buf`` into the ringbuffer, limited + by the available space in the ringbuffer. + + Return value: Integer count of bytes written. + + .. method:: RingIO.close() + + No-op provided as part of standard `stream` interface. Has no effect + on data in the ringbuffer. diff --git a/ports/unix/coverage.c b/ports/unix/coverage.c index 67052ea704..2b65b47fc5 100644 --- a/ports/unix/coverage.c +++ b/ports/unix/coverage.c @@ -692,6 +692,24 @@ static mp_obj_t extra_coverage(void) { ringbuf.iget = 0; ringbuf_put(&ringbuf, 0xaa); mp_printf(&mp_plat_print, "%d\n", ringbuf_get16(&ringbuf)); + + // ringbuf_put_bytes() / ringbuf_get_bytes() functions. + ringbuf.iput = 0; + ringbuf.iget = 0; + uint8_t *put = (uint8_t *)"abc123"; + uint8_t get[7] = {0}; + mp_printf(&mp_plat_print, "%d\n", ringbuf_put_bytes(&ringbuf, put, 7)); + mp_printf(&mp_plat_print, "%d\n", ringbuf_get_bytes(&ringbuf, get, 7)); + mp_printf(&mp_plat_print, "%s\n", get); + // Prefill ringbuffer. + for (size_t i = 0; i < sizeof(buf) - 3; ++i) { + ringbuf_put(&ringbuf, i); + } + // Should fail - too full. + mp_printf(&mp_plat_print, "%d\n", ringbuf_put_bytes(&ringbuf, put, 7)); + // Should fail - buffer too big. + uint8_t large[sizeof(buf) + 5] = {0}; + mp_printf(&mp_plat_print, "%d\n", ringbuf_put_bytes(&ringbuf, large, sizeof(large))); } // pairheap diff --git a/py/modmicropython.c b/py/modmicropython.c index daf03807c8..1bf0a000c2 100644 --- a/py/modmicropython.c +++ b/py/modmicropython.c @@ -200,6 +200,9 @@ static const mp_rom_map_elem_t mp_module_micropython_globals_table[] = { #if MICROPY_KBD_EXCEPTION { MP_ROM_QSTR(MP_QSTR_kbd_intr), MP_ROM_PTR(&mp_micropython_kbd_intr_obj) }, #endif + #if MICROPY_PY_MICROPYTHON_RINGIO + { MP_ROM_QSTR(MP_QSTR_RingIO), MP_ROM_PTR(&mp_type_ringio) }, + #endif #if MICROPY_ENABLE_SCHEDULER { MP_ROM_QSTR(MP_QSTR_schedule), MP_ROM_PTR(&mp_micropython_schedule_obj) }, #endif diff --git a/py/mpconfig.h b/py/mpconfig.h index 5c10007a19..b5414312c7 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -1306,6 +1306,11 @@ typedef double mp_float_t; #define MICROPY_PY_MICROPYTHON_HEAP_LOCKED (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) #endif +// Support for micropython.RingIO() +#ifndef MICROPY_PY_MICROPYTHON_RINGIO +#define MICROPY_PY_MICROPYTHON_RINGIO (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#endif + // Whether to provide "array" module. Note that large chunk of the // underlying code is shared with "bytearray" builtin type, so to // get real savings, it should be disabled too. diff --git a/py/obj.h b/py/obj.h index 81ee75aebc..eac2b89ea8 100644 --- a/py/obj.h +++ b/py/obj.h @@ -843,6 +843,7 @@ extern const mp_obj_type_t mp_type_bound_meth; extern const mp_obj_type_t mp_type_property; extern const mp_obj_type_t mp_type_stringio; extern const mp_obj_type_t mp_type_bytesio; +extern const mp_obj_type_t mp_type_ringio; extern const mp_obj_type_t mp_type_reversed; extern const mp_obj_type_t mp_type_polymorph_iter; #if MICROPY_ENABLE_FINALISER diff --git a/py/objringio.c b/py/objringio.c new file mode 100644 index 0000000000..ba1ec25307 --- /dev/null +++ b/py/objringio.c @@ -0,0 +1,130 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2024 Andrew Leech + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "ringbuf.h" +#include "py/mpconfig.h" + +#if MICROPY_PY_MICROPYTHON_RINGIO + +#include "py/runtime.h" +#include "py/stream.h" + +typedef struct _micropython_ringio_obj_t { + mp_obj_base_t base; + ringbuf_t ringbuffer; +} micropython_ringio_obj_t; + +static mp_obj_t micropython_ringio_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { + mp_arg_check_num(n_args, n_kw, 1, 1, false); + mp_int_t buff_size = -1; + mp_buffer_info_t bufinfo = {NULL, 0, 0}; + + if (!mp_get_buffer(args[0], &bufinfo, MP_BUFFER_RW)) { + buff_size = mp_obj_get_int(args[0]); + } + micropython_ringio_obj_t *self = mp_obj_malloc(micropython_ringio_obj_t, type); + if (bufinfo.buf != NULL) { + // buffer passed in, use it directly for ringbuffer. + self->ringbuffer.buf = bufinfo.buf; + self->ringbuffer.size = bufinfo.len; + self->ringbuffer.iget = self->ringbuffer.iput = 0; + } else { + // Allocate new buffer, add one extra to buff_size as ringbuf consumes one byte for tracking. + ringbuf_alloc(&(self->ringbuffer), buff_size + 1); + } + return MP_OBJ_FROM_PTR(self); +} + +static mp_uint_t micropython_ringio_read(mp_obj_t self_in, void *buf_in, mp_uint_t size, int *errcode) { + micropython_ringio_obj_t *self = MP_OBJ_TO_PTR(self_in); + size = MIN(size, ringbuf_avail(&self->ringbuffer)); + ringbuf_memcpy_get_internal(&(self->ringbuffer), buf_in, size); + *errcode = 0; + return size; +} + +static mp_uint_t micropython_ringio_write(mp_obj_t self_in, const void *buf_in, mp_uint_t size, int *errcode) { + micropython_ringio_obj_t *self = MP_OBJ_TO_PTR(self_in); + size = MIN(size, ringbuf_free(&self->ringbuffer)); + ringbuf_memcpy_put_internal(&(self->ringbuffer), buf_in, size); + *errcode = 0; + return size; +} + +static mp_uint_t micropython_ringio_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_t arg, int *errcode) { + micropython_ringio_obj_t *self = MP_OBJ_TO_PTR(self_in); + switch (request) { + case MP_STREAM_POLL: { + mp_uint_t ret = 0; + if ((arg & MP_STREAM_POLL_RD) && ringbuf_avail(&self->ringbuffer) > 0) { + ret |= MP_STREAM_POLL_RD; + } + if ((arg & MP_STREAM_POLL_WR) && ringbuf_free(&self->ringbuffer) > 0) { + ret |= MP_STREAM_POLL_WR; + } + return ret; + } + case MP_STREAM_CLOSE: + return 0; + } + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; +} + +static mp_obj_t micropython_ringio_any(mp_obj_t self_in) { + micropython_ringio_obj_t *self = MP_OBJ_TO_PTR(self_in); + return MP_OBJ_NEW_SMALL_INT(ringbuf_avail(&self->ringbuffer)); +} +static MP_DEFINE_CONST_FUN_OBJ_1(micropython_ringio_any_obj, micropython_ringio_any); + +static const mp_rom_map_elem_t micropython_ringio_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_any), MP_ROM_PTR(µpython_ringio_any_obj) }, + { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) }, + { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) }, + { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) }, + { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) }, + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, + +}; +static MP_DEFINE_CONST_DICT(micropython_ringio_locals_dict, micropython_ringio_locals_dict_table); + +static const mp_stream_p_t ringio_stream_p = { + .read = micropython_ringio_read, + .write = micropython_ringio_write, + .ioctl = micropython_ringio_ioctl, + .is_text = false, +}; + +MP_DEFINE_CONST_OBJ_TYPE( + mp_type_ringio, + MP_QSTR_RingIO, + MP_TYPE_FLAG_NONE, + make_new, micropython_ringio_make_new, + protocol, &ringio_stream_p, + locals_dict, µpython_ringio_locals_dict + ); + +#endif // MICROPY_PY_MICROPYTHON_RINGIO diff --git a/py/py.cmake b/py/py.cmake index 03c559c206..d7e2b0a05d 100644 --- a/py/py.cmake +++ b/py/py.cmake @@ -95,6 +95,7 @@ set(MICROPY_SOURCE_PY ${MICROPY_PY_DIR}/objproperty.c ${MICROPY_PY_DIR}/objrange.c ${MICROPY_PY_DIR}/objreversed.c + ${MICROPY_PY_DIR}/objringio.c ${MICROPY_PY_DIR}/objset.c ${MICROPY_PY_DIR}/objsingleton.c ${MICROPY_PY_DIR}/objslice.c diff --git a/py/py.mk b/py/py.mk index 0d4958ccba..16240a5b9c 100644 --- a/py/py.mk +++ b/py/py.mk @@ -167,6 +167,7 @@ PY_CORE_O_BASENAME = $(addprefix py/,\ objnamedtuple.o \ objrange.o \ objreversed.o \ + objringio.o \ objset.o \ objsingleton.o \ objslice.o \ diff --git a/py/ringbuf.c b/py/ringbuf.c index 10dca62081..5f77271a06 100644 --- a/py/ringbuf.c +++ b/py/ringbuf.c @@ -24,8 +24,6 @@ * THE SOFTWARE. */ -#include - #include "ringbuf.h" int ringbuf_get16(ringbuf_t *r) { @@ -83,17 +81,7 @@ int ringbuf_get_bytes(ringbuf_t *r, uint8_t *data, size_t data_len) { if (ringbuf_avail(r) < data_len) { return (r->size <= data_len) ? -2 : -1; } - uint32_t iget = r->iget; - uint32_t iget_a = (iget + data_len) % r->size; - uint8_t *datap = data; - if (iget_a < iget) { - // Copy part of the data from the space left at the end of the buffer - memcpy(datap, r->buf + iget, r->size - iget); - datap += (r->size - iget); - iget = 0; - } - memcpy(datap, r->buf + iget, iget_a - iget); - r->iget = iget_a; + ringbuf_memcpy_get_internal(r, data, data_len); return 0; } @@ -105,16 +93,6 @@ int ringbuf_put_bytes(ringbuf_t *r, const uint8_t *data, size_t data_len) { if (ringbuf_free(r) < data_len) { return (r->size <= data_len) ? -2 : -1; } - uint32_t iput = r->iput; - uint32_t iput_a = (iput + data_len) % r->size; - const uint8_t *datap = data; - if (iput_a < iput) { - // Copy part of the data to the end of the buffer - memcpy(r->buf + iput, datap, r->size - iput); - datap += (r->size - iput); - iput = 0; - } - memcpy(r->buf + iput, datap, iput_a - iput); - r->iput = iput_a; + ringbuf_memcpy_put_internal(r, data, data_len); return 0; } diff --git a/py/ringbuf.h b/py/ringbuf.h index c8508c07ed..d5aed429c5 100644 --- a/py/ringbuf.h +++ b/py/ringbuf.h @@ -28,10 +28,9 @@ #include #include +#include -#ifdef _MSC_VER -#include "py/mpconfig.h" // For inline. -#endif +#include "py/mpconfig.h" typedef struct _ringbuf_t { uint8_t *buf; @@ -91,6 +90,38 @@ static inline size_t ringbuf_avail(ringbuf_t *r) { return (r->size + r->iput - r->iget) % r->size; } +static inline void ringbuf_memcpy_get_internal(ringbuf_t *r, uint8_t *data, size_t data_len) { + // No bounds / space checking is performed here so ensure available size is checked before running this + // otherwise data loss or buffer overflow can occur. + uint32_t iget = r->iget; + uint32_t iget_a = (iget + data_len) % r->size; + uint8_t *datap = data; + if (iget_a < iget) { + // Copy part of the data from the space left at the end of the buffer + memcpy(datap, r->buf + iget, r->size - iget); + datap += (r->size - iget); + iget = 0; + } + memcpy(datap, r->buf + iget, iget_a - iget); + r->iget = iget_a; +} + +static inline void ringbuf_memcpy_put_internal(ringbuf_t *r, const uint8_t *data, size_t data_len) { + // No bounds / space checking is performed here so ensure free size is checked before running this + // otherwise data loss or buffer overflow can occur. + uint32_t iput = r->iput; + uint32_t iput_a = (iput + data_len) % r->size; + const uint8_t *datap = data; + if (iput_a < iput) { + // Copy part of the data to the end of the buffer + memcpy(r->buf + iput, datap, r->size - iput); + datap += (r->size - iput); + iput = 0; + } + memcpy(r->buf + iput, datap, iput_a - iput); + r->iput = iput_a; +} + // Note: big-endian. No-op if not enough room available for both bytes. int ringbuf_get16(ringbuf_t *r); int ringbuf_peek16(ringbuf_t *r); diff --git a/tests/micropython/ringio.py b/tests/micropython/ringio.py new file mode 100644 index 0000000000..45774bc2ee --- /dev/null +++ b/tests/micropython/ringio.py @@ -0,0 +1,48 @@ +# Check that micropython.RingIO works correctly. + +import micropython + +try: + micropython.RingIO +except AttributeError: + print("SKIP") + raise SystemExit + +rb = micropython.RingIO(16) +print(rb) + +print(rb.any()) + +rb.write(b"\x00") +print(rb.any()) + +rb.write(b"\x00") +print(rb.any()) + +print(rb.read(2)) +print(rb.any()) + + +rb.write(b"\x00\x01") +print(rb.read()) + +print(rb.read(1)) + +# Try to write more data than can fit at one go. +print(rb.write(b"\x00\x01" * 10)) +print(rb.write(b"\x00")) +print(rb.read()) + + +ba = bytearray(17) +rb = micropython.RingIO(ba) +print(rb) +print(rb.write(b"\x00\x01" * 10)) +print(rb.write(b"\x00")) +print(rb.read()) + +try: + # Size must be int. + micropython.RingIO(None) +except TypeError as ex: + print(ex) diff --git a/tests/micropython/ringio.py.exp b/tests/micropython/ringio.py.exp new file mode 100644 index 0000000000..b27ea472c7 --- /dev/null +++ b/tests/micropython/ringio.py.exp @@ -0,0 +1,16 @@ + +0 +1 +2 +b'\x00\x00' +0 +b'\x00\x01' +b'' +16 +0 +b'\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01' + +16 +0 +b'\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01\x00\x01' +can't convert NoneType to int diff --git a/tests/micropython/ringio_async.py b/tests/micropython/ringio_async.py new file mode 100644 index 0000000000..2a4befc352 --- /dev/null +++ b/tests/micropython/ringio_async.py @@ -0,0 +1,36 @@ +# Check that micropython.RingIO works correctly with asyncio.Stream. + +import micropython + +try: + import asyncio + + asyncio.StreamWriter + micropython.RingIO +except (AttributeError, ImportError): + print("SKIP") + raise SystemExit + +rb = micropython.RingIO(16) +rba = asyncio.StreamWriter(rb) + +data = b"ABC123" * 20 +print("w", len(data)) + + +async def data_writer(): + global data + rba.write(data) + await rba.drain() + + +async def main(): + task = asyncio.create_task(data_writer()) + await asyncio.sleep_ms(10) + read = await rba.readexactly(len(data)) + print(read) + print("r", len(read)) + print(read == data) + + +asyncio.run(main()) diff --git a/tests/micropython/ringio_async.py.exp b/tests/micropython/ringio_async.py.exp new file mode 100644 index 0000000000..dfeb71d890 --- /dev/null +++ b/tests/micropython/ringio_async.py.exp @@ -0,0 +1,4 @@ +w 120 +b'ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123' +r 120 +True diff --git a/tests/ports/unix/extra_coverage.py.exp b/tests/ports/unix/extra_coverage.py.exp index a2b11638a3..176db8e9f8 100644 --- a/tests/ports/unix/extra_coverage.py.exp +++ b/tests/ports/unix/extra_coverage.py.exp @@ -144,6 +144,11 @@ cc99 22ff -1 -1 +0 +0 +abc123 +-1 +-2 # pairheap create: 0 0 0 0 pop all: 0 1 2 3