Browse Source
This will be replaced with a new deflate module providing the same functionality, with an optional frozen Python wrapper providing a replacement zlib module. binascii.crc32 is temporarily disabled. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared <jim.mussared@gmail.com>pull/11905/head
Jim Mussared
1 year ago
committed by
Damien George
Damien George
22 changed files with 4 additions and 482 deletions
@ -1,13 +0,0 @@ |
|||
# Location of top-level MicroPython directory
|
|||
MPY_DIR = ../../.. |
|||
|
|||
# Name of module (different to built-in zlib so it can coexist)
|
|||
MOD = zlib_$(ARCH) |
|||
|
|||
# Source files (.c or .py)
|
|||
SRC = zlib.c |
|||
|
|||
# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
|
|||
ARCH = x64 |
|||
|
|||
include $(MPY_DIR)/py/dynruntime.mk |
|||
@ -1,35 +0,0 @@ |
|||
#define MICROPY_PY_ZLIB (1) |
|||
|
|||
#include "py/dynruntime.h" |
|||
|
|||
#if !defined(__linux__) |
|||
void *memset(void *s, int c, size_t n) { |
|||
return mp_fun_table.memset_(s, c, n); |
|||
} |
|||
#endif |
|||
|
|||
mp_obj_full_type_t decompio_type; |
|||
|
|||
#include "extmod/modzlib.c" |
|||
|
|||
mp_map_elem_t decompio_locals_dict_table[3]; |
|||
STATIC MP_DEFINE_CONST_DICT(decompio_locals_dict, decompio_locals_dict_table); |
|||
|
|||
mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { |
|||
MP_DYNRUNTIME_INIT_ENTRY |
|||
|
|||
decompio_type.base.type = mp_fun_table.type_type; |
|||
decompio_type.name = MP_QSTR_DecompIO; |
|||
MP_OBJ_TYPE_SET_SLOT(&decompio_type, make_new, &decompio_make_new, 0); |
|||
MP_OBJ_TYPE_SET_SLOT(&decompio_type, protocol, &decompio_stream_p, 1); |
|||
decompio_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_read), MP_OBJ_FROM_PTR(&mp_stream_read_obj) }; |
|||
decompio_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readinto), MP_OBJ_FROM_PTR(&mp_stream_readinto_obj) }; |
|||
decompio_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readline), MP_OBJ_FROM_PTR(&mp_stream_unbuffered_readline_obj) }; |
|||
MP_OBJ_TYPE_SET_SLOT(&decompio_type, locals_dict, (void*)&decompio_locals_dict, 2); |
|||
|
|||
mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_zlib)); |
|||
mp_store_global(MP_QSTR_decompress, MP_OBJ_FROM_PTR(&mod_zlib_decompress_obj)); |
|||
mp_store_global(MP_QSTR_DecompIO, MP_OBJ_FROM_PTR(&decompio_type)); |
|||
|
|||
MP_DYNRUNTIME_INIT_EXIT |
|||
} |
|||
@ -1,239 +0,0 @@ |
|||
/*
|
|||
* This file is part of the MicroPython project, http://micropython.org/
|
|||
* |
|||
* The MIT License (MIT) |
|||
* |
|||
* Copyright (c) 2014-2016 Paul Sokolovsky |
|||
* |
|||
* Permission is hereby granted, free of charge, to any person obtaining a copy |
|||
* of this software and associated documentation files (the "Software"), to deal |
|||
* in the Software without restriction, including without limitation the rights |
|||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
|||
* copies of the Software, and to permit persons to whom the Software is |
|||
* furnished to do so, subject to the following conditions: |
|||
* |
|||
* The above copyright notice and this permission notice shall be included in |
|||
* all copies or substantial portions of the Software. |
|||
* |
|||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
|||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
|||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
|||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
|||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
|||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
|||
* THE SOFTWARE. |
|||
*/ |
|||
|
|||
#include <stdio.h> |
|||
#include <string.h> |
|||
|
|||
#include "py/runtime.h" |
|||
#include "py/stream.h" |
|||
#include "py/mperrno.h" |
|||
|
|||
#if MICROPY_PY_ZLIB |
|||
|
|||
#include "lib/uzlib/tinf.h" |
|||
|
|||
#if 0 // print debugging info
|
|||
#define DEBUG_printf DEBUG_printf |
|||
#else // don't print debugging info
|
|||
#define DEBUG_printf(...) (void)0 |
|||
#endif |
|||
|
|||
typedef struct _mp_obj_decompio_t { |
|||
mp_obj_base_t base; |
|||
mp_obj_t src_stream; |
|||
TINF_DATA decomp; |
|||
bool eof; |
|||
} mp_obj_decompio_t; |
|||
|
|||
STATIC int read_src_stream(TINF_DATA *data) { |
|||
byte *p = (void *)data; |
|||
p -= offsetof(mp_obj_decompio_t, decomp); |
|||
mp_obj_decompio_t *self = (mp_obj_decompio_t *)p; |
|||
|
|||
const mp_stream_p_t *stream = mp_get_stream(self->src_stream); |
|||
int err; |
|||
byte c; |
|||
mp_uint_t out_sz = stream->read(self->src_stream, &c, 1, &err); |
|||
if (out_sz == MP_STREAM_ERROR) { |
|||
mp_raise_OSError(err); |
|||
} |
|||
if (out_sz == 0) { |
|||
mp_raise_type(&mp_type_EOFError); |
|||
} |
|||
return c; |
|||
} |
|||
|
|||
STATIC mp_obj_t decompio_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { |
|||
mp_arg_check_num(n_args, n_kw, 1, 2, false); |
|||
mp_get_stream_raise(args[0], MP_STREAM_OP_READ); |
|||
mp_obj_decompio_t *o = mp_obj_malloc(mp_obj_decompio_t, type); |
|||
memset(&o->decomp, 0, sizeof(o->decomp)); |
|||
o->decomp.readSource = read_src_stream; |
|||
o->src_stream = args[0]; |
|||
o->eof = false; |
|||
|
|||
mp_int_t dict_opt = 0; |
|||
uint dict_sz; |
|||
if (n_args > 1) { |
|||
dict_opt = mp_obj_get_int(args[1]); |
|||
} |
|||
|
|||
if (dict_opt >= 16) { |
|||
int st = uzlib_gzip_parse_header(&o->decomp); |
|||
if (st != TINF_OK) { |
|||
goto header_error; |
|||
} |
|||
dict_sz = 1 << (dict_opt - 16); |
|||
} else if (dict_opt >= 0) { |
|||
dict_opt = uzlib_zlib_parse_header(&o->decomp); |
|||
if (dict_opt < 0) { |
|||
header_error: |
|||
mp_raise_ValueError(MP_ERROR_TEXT("compression header")); |
|||
} |
|||
// RFC 1950 section 2.2:
|
|||
// CINFO is the base-2 logarithm of the LZ77 window size,
|
|||
// minus eight (CINFO=7 indicates a 32K window size)
|
|||
dict_sz = 1 << (dict_opt + 8); |
|||
} else { |
|||
dict_sz = 1 << -dict_opt; |
|||
} |
|||
|
|||
uzlib_uncompress_init(&o->decomp, m_new(byte, dict_sz), dict_sz); |
|||
return MP_OBJ_FROM_PTR(o); |
|||
} |
|||
|
|||
STATIC mp_uint_t decompio_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) { |
|||
mp_obj_decompio_t *o = MP_OBJ_TO_PTR(o_in); |
|||
if (o->eof) { |
|||
return 0; |
|||
} |
|||
|
|||
o->decomp.dest = buf; |
|||
o->decomp.dest_limit = (byte *)buf + size; |
|||
int st = uzlib_uncompress_chksum(&o->decomp); |
|||
if (st == TINF_DONE) { |
|||
o->eof = true; |
|||
} |
|||
if (st < 0) { |
|||
DEBUG_printf("uncompress error=" INT_FMT "\n", st); |
|||
*errcode = MP_EINVAL; |
|||
return MP_STREAM_ERROR; |
|||
} |
|||
return o->decomp.dest - (byte *)buf; |
|||
} |
|||
|
|||
#if !MICROPY_ENABLE_DYNRUNTIME |
|||
STATIC const mp_rom_map_elem_t decompio_locals_dict_table[] = { |
|||
{ MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) }, |
|||
{ MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) }, |
|||
{ MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) }, |
|||
}; |
|||
|
|||
STATIC MP_DEFINE_CONST_DICT(decompio_locals_dict, decompio_locals_dict_table); |
|||
#endif |
|||
|
|||
STATIC const mp_stream_p_t decompio_stream_p = { |
|||
.read = decompio_read, |
|||
}; |
|||
|
|||
#if !MICROPY_ENABLE_DYNRUNTIME |
|||
STATIC MP_DEFINE_CONST_OBJ_TYPE( |
|||
decompio_type, |
|||
MP_QSTR_DecompIO, |
|||
MP_TYPE_FLAG_NONE, |
|||
make_new, decompio_make_new, |
|||
protocol, &decompio_stream_p, |
|||
locals_dict, &decompio_locals_dict |
|||
); |
|||
#endif |
|||
|
|||
STATIC mp_obj_t mod_zlib_decompress(size_t n_args, const mp_obj_t *args) { |
|||
mp_obj_t data = args[0]; |
|||
mp_buffer_info_t bufinfo; |
|||
mp_get_buffer_raise(data, &bufinfo, MP_BUFFER_READ); |
|||
|
|||
TINF_DATA *decomp = m_new_obj(TINF_DATA); |
|||
memset(decomp, 0, sizeof(*decomp)); |
|||
DEBUG_printf("sizeof(TINF_DATA)=" UINT_FMT "\n", sizeof(*decomp)); |
|||
uzlib_uncompress_init(decomp, NULL, 0); |
|||
mp_uint_t dest_buf_size = (bufinfo.len + 15) & ~15; |
|||
byte *dest_buf = m_new(byte, dest_buf_size); |
|||
|
|||
decomp->dest = dest_buf; |
|||
decomp->dest_limit = dest_buf + dest_buf_size; |
|||
DEBUG_printf("zlib: Initial out buffer: " UINT_FMT " bytes\n", dest_buf_size); |
|||
decomp->source = bufinfo.buf; |
|||
decomp->source_limit = (byte *)bufinfo.buf + bufinfo.len; |
|||
|
|||
int st; |
|||
bool is_zlib = true; |
|||
|
|||
if (n_args > 1 && MP_OBJ_SMALL_INT_VALUE(args[1]) < 0) { |
|||
is_zlib = false; |
|||
} |
|||
|
|||
if (is_zlib) { |
|||
st = uzlib_zlib_parse_header(decomp); |
|||
if (st < 0) { |
|||
goto error; |
|||
} |
|||
} |
|||
|
|||
while (1) { |
|||
st = uzlib_uncompress_chksum(decomp); |
|||
if (st < 0) { |
|||
goto error; |
|||
} |
|||
if (st == TINF_DONE) { |
|||
break; |
|||
} |
|||
size_t offset = decomp->dest - dest_buf; |
|||
dest_buf = m_renew(byte, dest_buf, dest_buf_size, dest_buf_size + 256); |
|||
dest_buf_size += 256; |
|||
decomp->dest = dest_buf + offset; |
|||
decomp->dest_limit = decomp->dest + 256; |
|||
} |
|||
|
|||
mp_uint_t final_sz = decomp->dest - dest_buf; |
|||
DEBUG_printf("zlib: Resizing from " UINT_FMT " to final size: " UINT_FMT " bytes\n", dest_buf_size, final_sz); |
|||
dest_buf = (byte *)m_renew(byte, dest_buf, dest_buf_size, final_sz); |
|||
mp_obj_t res = mp_obj_new_bytearray_by_ref(final_sz, dest_buf); |
|||
m_del_obj(TINF_DATA, decomp); |
|||
return res; |
|||
|
|||
error: |
|||
mp_raise_type_arg(&mp_type_ValueError, MP_OBJ_NEW_SMALL_INT(st)); |
|||
} |
|||
STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mod_zlib_decompress_obj, 1, 3, mod_zlib_decompress); |
|||
|
|||
#if !MICROPY_ENABLE_DYNRUNTIME |
|||
STATIC const mp_rom_map_elem_t mp_module_zlib_globals_table[] = { |
|||
{ MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_zlib) }, |
|||
{ MP_ROM_QSTR(MP_QSTR_decompress), MP_ROM_PTR(&mod_zlib_decompress_obj) }, |
|||
{ MP_ROM_QSTR(MP_QSTR_DecompIO), MP_ROM_PTR(&decompio_type) }, |
|||
}; |
|||
|
|||
STATIC MP_DEFINE_CONST_DICT(mp_module_zlib_globals, mp_module_zlib_globals_table); |
|||
|
|||
const mp_obj_module_t mp_module_zlib = { |
|||
.base = { &mp_type_module }, |
|||
.globals = (mp_obj_dict_t *)&mp_module_zlib_globals, |
|||
}; |
|||
|
|||
|
|||
MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_zlib, mp_module_zlib); |
|||
#endif |
|||
|
|||
// Source files #include'd here to make sure they're compiled in
|
|||
// only if module is enabled by config setting.
|
|||
|
|||
#include "lib/uzlib/tinflate.c" |
|||
#include "lib/uzlib/tinfzlib.c" |
|||
#include "lib/uzlib/tinfgzip.c" |
|||
#include "lib/uzlib/adler32.c" |
|||
#include "lib/uzlib/crc32.c" |
|||
|
|||
#endif // MICROPY_PY_ZLIB
|
|||
@ -1,33 +0,0 @@ |
|||
try: |
|||
import zlib |
|||
import io |
|||
except ImportError: |
|||
print("SKIP") |
|||
raise SystemExit |
|||
|
|||
|
|||
# Raw DEFLATE bitstream |
|||
buf = io.BytesIO(b"\xcbH\xcd\xc9\xc9\x07\x00") |
|||
inp = zlib.DecompIO(buf, -8) |
|||
print(buf.seek(0, 1)) |
|||
print(inp.read(1)) |
|||
print(buf.seek(0, 1)) |
|||
print(inp.read(2)) |
|||
print(inp.read()) |
|||
print(buf.seek(0, 1)) |
|||
print(inp.read(1)) |
|||
print(inp.read()) |
|||
print(buf.seek(0, 1)) |
|||
|
|||
|
|||
# zlib bitstream (with 256 byte window size) |
|||
inp = zlib.DecompIO(io.BytesIO(b"\x08\x9930\xa0=\x00\x00\xb3q\x12\xc1")) |
|||
print(inp.read(10)) |
|||
print(inp.read()) |
|||
|
|||
# zlib bitstream, wrong checksum |
|||
inp = zlib.DecompIO(io.BytesIO(b"\x08\x9930\xa0=\x00\x00\xb3q\x12\xc0")) |
|||
try: |
|||
print(inp.read()) |
|||
except OSError as e: |
|||
print(repr(e)) |
|||
@ -1,12 +0,0 @@ |
|||
0 |
|||
b'h' |
|||
2 |
|||
b'el' |
|||
b'lo' |
|||
7 |
|||
b'' |
|||
b'' |
|||
7 |
|||
b'0000000000' |
|||
b'000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000' |
|||
OSError(22,) |
|||
@ -1,60 +0,0 @@ |
|||
try: |
|||
import zlib |
|||
import io |
|||
except ImportError: |
|||
print("SKIP") |
|||
raise SystemExit |
|||
|
|||
|
|||
# gzip bitstream |
|||
buf = io.BytesIO( |
|||
b"\x1f\x8b\x08\x08\x99\x0c\xe5W\x00\x03hello\x00\xcbH\xcd\xc9\xc9\x07\x00\x86\xa6\x106\x05\x00\x00\x00" |
|||
) |
|||
inp = zlib.DecompIO(buf, 16 + 8) |
|||
print(buf.seek(0, 1)) |
|||
print(inp.read(1)) |
|||
print(buf.seek(0, 1)) |
|||
print(inp.read(2)) |
|||
print(inp.read()) |
|||
print(buf.seek(0, 1)) |
|||
print(inp.read(1)) |
|||
print(inp.read()) |
|||
print(buf.seek(0, 1)) |
|||
|
|||
# Check FHCRC field |
|||
buf = io.BytesIO( |
|||
b"\x1f\x8b\x08\x02\x99\x0c\xe5W\x00\x03\x00\x00\xcbH\xcd\xc9\xc9\x07\x00\x86\xa6\x106\x05\x00\x00\x00" |
|||
) |
|||
inp = zlib.DecompIO(buf, 16 + 8) |
|||
print(inp.read()) |
|||
|
|||
# Check FEXTRA field |
|||
buf = io.BytesIO( |
|||
b"\x1f\x8b\x08\x04\x99\x0c\xe5W\x00\x03\x01\x00X\xcbH\xcd\xc9\xc9\x07\x00\x86\xa6\x106\x05\x00\x00\x00" |
|||
) |
|||
inp = zlib.DecompIO(buf, 16 + 8) |
|||
print(inp.read()) |
|||
|
|||
# broken header |
|||
buf = io.BytesIO( |
|||
b"\x1f\x8c\x08\x08\x99\x0c\xe5W\x00\x03hello\x00\xcbH\xcd\xc9\xc9\x07\x00\x86\xa6\x106\x05\x00\x00\x00" |
|||
) |
|||
try: |
|||
inp = zlib.DecompIO(buf, 16 + 8) |
|||
except ValueError: |
|||
print("ValueError") |
|||
|
|||
# broken crc32 |
|||
buf = io.BytesIO( |
|||
b"\x1f\x8b\x08\x08\x99\x0c\xe5W\x00\x03hello\x00\xcbH\xcd\xc9\xc9\x07\x00\x86\xa7\x106\x05\x00\x00\x00" |
|||
) |
|||
inp = zlib.DecompIO(buf, 16 + 8) |
|||
try: |
|||
inp.read(6) |
|||
except OSError as e: |
|||
print(repr(e)) |
|||
|
|||
# broken uncompressed size - not checked so far |
|||
# buf = io.BytesIO(b'\x1f\x8b\x08\x08\x99\x0c\xe5W\x00\x03hello\x00\xcbH\xcd\xc9\xc9\x07\x00\x86\xa6\x106\x06\x00\x00\x00') |
|||
# inp = zlib.DecompIO(buf, 16 + 8) |
|||
# inp.read(6) |
|||
@ -1,13 +0,0 @@ |
|||
16 |
|||
b'h' |
|||
18 |
|||
b'el' |
|||
b'lo' |
|||
31 |
|||
b'' |
|||
b'' |
|||
31 |
|||
b'hello' |
|||
b'hello' |
|||
ValueError |
|||
OSError(22,) |
|||
@ -1,57 +0,0 @@ |
|||
try: |
|||
import zlib |
|||
except ImportError: |
|||
print("SKIP") |
|||
raise SystemExit |
|||
|
|||
PATTERNS = [ |
|||
# Packed results produced by CPy's zlib.compress() |
|||
(b"0", b"x\x9c3\x00\x00\x001\x001"), |
|||
(b"a", b"x\x9cK\x04\x00\x00b\x00b"), |
|||
(b"0" * 100, b"x\x9c30\xa0=\x00\x00\xb3q\x12\xc1"), |
|||
( |
|||
bytes(range(64)), |
|||
b"x\x9cc`dbfaec\xe7\xe0\xe4\xe2\xe6\xe1\xe5\xe3\x17\x10\x14\x12\x16\x11\x15\x13\x97\x90\x94\x92\x96\x91\x95\x93WPTRVQUS\xd7\xd0\xd4\xd2\xd6\xd1\xd5\xd370426153\xb7\xb0\xb4\xb2\xb6\xb1\xb5\xb3\x07\x00\xaa\xe0\x07\xe1", |
|||
), |
|||
(b"hello", b"x\x01\x01\x05\x00\xfa\xffhello\x06,\x02\x15"), # compression level 0 |
|||
# adaptive/dynamic huffman tree |
|||
( |
|||
b"13371813150|13764518736|12345678901", |
|||
b"x\x9c\x05\xc1\x81\x01\x000\x04\x04\xb1\x95\\\x1f\xcfn\x86o\x82d\x06Qq\xc8\x9d\xc5X}<e\xb5g\x83\x0f\x89X\x07\xab", |
|||
), |
|||
# dynamic Huffman tree with "case 17" (repeat code for 3-10 times) |
|||
( |
|||
b">I}\x00\x951D>I}\x00\x951D>I}\x00\x951D>I}\x00\x951D", |
|||
b"x\x9c\x05\xc11\x01\x00\x00\x00\x010\x95\x14py\x84\x12C_\x9bR\x8cV\x8a\xd1J1Z)F\x1fw`\x089", |
|||
), |
|||
] |
|||
|
|||
for unpacked, packed in PATTERNS: |
|||
assert zlib.decompress(packed) == unpacked |
|||
print(unpacked) |
|||
|
|||
|
|||
# Raw DEFLATE bitstream |
|||
v = b"\xcbH\xcd\xc9\xc9\x07\x00" |
|||
exp = b"hello" |
|||
out = zlib.decompress(v, -15) |
|||
assert out == exp |
|||
print(exp) |
|||
# Even when you ask CPython zlib.compress to produce Raw DEFLATE stream, |
|||
# it returns it with adler2 and oriignal size appended, as if it was a |
|||
# zlib stream. Make sure there're no random issues decompressing such. |
|||
v = b"\xcbH\xcd\xc9\xc9\x07\x00\x86\xa6\x106\x05\x00\x00\x00" |
|||
out = zlib.decompress(v, -15) |
|||
assert out == exp |
|||
|
|||
# this should error |
|||
try: |
|||
zlib.decompress(b"abc") |
|||
except Exception: |
|||
print("Exception") |
|||
|
|||
# invalid block type |
|||
try: |
|||
zlib.decompress(b"\x07", -15) # final-block, block-type=3 (invalid) |
|||
except Exception as er: |
|||
print("Exception") |
|||
Loading…
Reference in new issue