diff --git a/RELEASES.rst b/RELEASES.rst index cb77c3d0..73bfd44c 100644 --- a/RELEASES.rst +++ b/RELEASES.rst @@ -2012,8 +2012,8 @@ Planned * Add an fmod() self test (GH-1108) -* Reduce RAM built-ins initdata limitations for custom bindings by bumping - bit count for normal and function properties from 6 to 8 (GH-FIXME) +* Reduce RAM built-ins initdata limitations for custom bindings by using a + shared varuint encoding in the bit-packed initdata stream (GH-1151) * Fix JSON stringify fastpath handling of array gaps in JX and JC; they incorrectly stringified as 'null' (like in JSON) instead of 'undefined' diff --git a/src-input/duk_hthread_builtins.c b/src-input/duk_hthread_builtins.c index f05f5841..2a190651 100644 --- a/src-input/duk_hthread_builtins.c +++ b/src-input/duk_hthread_builtins.c @@ -14,14 +14,10 @@ #define DUK__BIDX_BITS 7 #define DUK__STRIDX_BITS 9 /* XXX: try to optimize to 8 (would now be possible, <200 used) */ #define DUK__NATIDX_BITS 8 -#define DUK__NUM_NORMAL_PROPS_BITS 8 -#define DUK__NUM_FUNC_PROPS_BITS 8 #define DUK__PROP_FLAGS_BITS 3 #define DUK__LENGTH_PROP_BITS 3 #define DUK__NARGS_BITS 3 #define DUK__PROP_TYPE_BITS 3 -#define DUK__MAGIC_BITS 16 -#define DUK__ACCESSOR_MAGIC_BITS 2 /* just a few shared accessors now */ #define DUK__NARGS_VARARGS_MARKER 0x07 #define DUK__NO_CLASS_MARKER 0x00 /* 0 = DUK_HOBJECT_CLASS_NONE */ @@ -292,7 +288,7 @@ DUK_INTERNAL void duk_hthread_create_builtin_objects(duk_hthread *thr) { } /* Cast converts magic to 16-bit signed value */ - magic = (duk_int16_t) duk_bd_decode_flagged(bd, DUK__MAGIC_BITS, 0 /*def_value*/); + magic = (duk_int16_t) duk_bd_decode_varuint(bd); ((duk_hnatfunc *) h)->magic = magic; } else if (class_num == DUK_HOBJECT_CLASS_ARRAY) { duk_push_array(ctx); @@ -406,7 +402,7 @@ DUK_INTERNAL void duk_hthread_create_builtin_objects(duk_hthread *thr) { } /* normal valued properties */ - num = (duk_small_uint_t) duk_bd_decode(bd, DUK__NUM_NORMAL_PROPS_BITS); + num = (duk_small_uint_t) duk_bd_decode_varuint(bd); DUK_DDD(DUK_DDDPRINT("built-in object %ld, %ld normal valued properties", (long) i, (long) num)); for (j = 0; j < num; j++) { duk_small_uint_t defprop_flags; @@ -479,7 +475,7 @@ DUK_INTERNAL void duk_hthread_create_builtin_objects(duk_hthread *thr) { case DUK__PROP_TYPE_ACCESSOR: { duk_small_uint_t natidx_getter = (duk_small_uint_t) duk_bd_decode(bd, DUK__NATIDX_BITS); duk_small_uint_t natidx_setter = (duk_small_uint_t) duk_bd_decode(bd, DUK__NATIDX_BITS); - duk_small_uint_t accessor_magic = (duk_small_uint_t) duk_bd_decode(bd, DUK__ACCESSOR_MAGIC_BITS); + duk_small_uint_t accessor_magic = (duk_small_uint_t) duk_bd_decode_varuint(bd); duk_c_function c_func_getter; duk_c_function c_func_setter; @@ -517,7 +513,7 @@ DUK_INTERNAL void duk_hthread_create_builtin_objects(duk_hthread *thr) { } /* native function properties */ - num = (duk_small_uint_t) duk_bd_decode(bd, DUK__NUM_FUNC_PROPS_BITS); + num = (duk_small_uint_t) duk_bd_decode_varuint(bd); DUK_DDD(DUK_DDDPRINT("built-in object %ld, %ld function valued properties", (long) i, (long) num)); for (j = 0; j < num; j++) { duk_hstring *h_key; @@ -549,7 +545,7 @@ DUK_INTERNAL void duk_hthread_create_builtin_objects(duk_hthread *thr) { (c_nargs == DUK_VARARGS ? (long) -1 : (long) c_nargs))); /* Cast converts magic to 16-bit signed value */ - magic = (duk_int16_t) duk_bd_decode_flagged(bd, DUK__MAGIC_BITS, 0); + magic = (duk_int16_t) duk_bd_decode_varuint(bd); #if defined(DUK_USE_LIGHTFUNC_BUILTINS) lightfunc_eligible = diff --git a/src-input/duk_util.h b/src-input/duk_util.h index 870cb2de..a003e20f 100644 --- a/src-input/duk_util.h +++ b/src-input/duk_util.h @@ -508,9 +508,10 @@ DUK_INTERNAL_DECL duk_uint32_t duk_util_hashbytes(const duk_uint8_t *data, duk_s DUK_INTERNAL_DECL duk_uint32_t duk_util_get_hash_prime(duk_uint32_t size); #endif -DUK_INTERNAL_DECL duk_int32_t duk_bd_decode(duk_bitdecoder_ctx *ctx, duk_small_int_t bits); -DUK_INTERNAL_DECL duk_small_int_t duk_bd_decode_flag(duk_bitdecoder_ctx *ctx); -DUK_INTERNAL_DECL duk_int32_t duk_bd_decode_flagged(duk_bitdecoder_ctx *ctx, duk_small_int_t bits, duk_int32_t def_value); +DUK_INTERNAL_DECL duk_uint32_t duk_bd_decode(duk_bitdecoder_ctx *ctx, duk_small_int_t bits); +DUK_INTERNAL_DECL duk_small_uint_t duk_bd_decode_flag(duk_bitdecoder_ctx *ctx); +DUK_INTERNAL_DECL duk_uint32_t duk_bd_decode_flagged(duk_bitdecoder_ctx *ctx, duk_small_int_t bits, duk_uint32_t def_value); +DUK_INTERNAL_DECL duk_uint32_t duk_bd_decode_varuint(duk_bitdecoder_ctx *ctx); DUK_INTERNAL_DECL duk_small_uint_t duk_bd_decode_bitpacked_string(duk_bitdecoder_ctx *bd, duk_uint8_t *out); DUK_INTERNAL_DECL void duk_be_encode(duk_bitencoder_ctx *ctx, duk_uint32_t data, duk_small_int_t bits); diff --git a/src-input/duk_util_bitdecoder.c b/src-input/duk_util_bitdecoder.c index 58c625bc..fa77e0e6 100644 --- a/src-input/duk_util_bitdecoder.c +++ b/src-input/duk_util_bitdecoder.c @@ -8,7 +8,7 @@ * When reading past bitstream end, zeroes are shifted in. The result * is signed to match duk_bd_decode_flagged. */ -DUK_INTERNAL duk_int32_t duk_bd_decode(duk_bitdecoder_ctx *ctx, duk_small_int_t bits) { +DUK_INTERNAL duk_uint32_t duk_bd_decode(duk_bitdecoder_ctx *ctx, duk_small_int_t bits) { duk_small_int_t shift; duk_uint32_t mask; duk_uint32_t tmp; @@ -53,22 +53,44 @@ DUK_INTERNAL duk_int32_t duk_bd_decode(duk_bitdecoder_ctx *ctx, duk_small_int_t return tmp; } -DUK_INTERNAL duk_small_int_t duk_bd_decode_flag(duk_bitdecoder_ctx *ctx) { - return (duk_small_int_t) duk_bd_decode(ctx, 1); +DUK_INTERNAL duk_small_uint_t duk_bd_decode_flag(duk_bitdecoder_ctx *ctx) { + return (duk_small_uint_t) duk_bd_decode(ctx, 1); } /* Decode a one-bit flag, and if set, decode a value of 'bits', otherwise return * default value. Return value is signed so that negative marker value can be * used by caller as a "not present" value. */ -DUK_INTERNAL duk_int32_t duk_bd_decode_flagged(duk_bitdecoder_ctx *ctx, duk_small_int_t bits, duk_int32_t def_value) { +DUK_INTERNAL duk_uint32_t duk_bd_decode_flagged(duk_bitdecoder_ctx *ctx, duk_small_int_t bits, duk_uint32_t def_value) { if (duk_bd_decode_flag(ctx)) { - return (duk_int32_t) duk_bd_decode(ctx, bits); + return duk_bd_decode(ctx, bits); } else { return def_value; } } +/* Shared varint encoding. Match dukutil.py BitEncode.varuint(). */ +DUK_INTERNAL duk_uint32_t duk_bd_decode_varuint(duk_bitdecoder_ctx *ctx) { + duk_small_uint_t t; + + /* Numbers 0-3 dominate input heavily, so use an initial flag byte for + * them, encoding into 3 bits. Other numbers are most often in [4,63] + * so encode that range efficiently with 7 bits. Reserve a few special + * values for rare long encodings. + */ + if (duk_bd_decode_flag(ctx)) { + t = duk_bd_decode(ctx, 6); + if (t == 0) { + return duk_bd_decode(ctx, 16); + } else if (t == 1) { + return duk_bd_decode(ctx, 32); + } + return (t - 2) + 4; /* Covers [4,65] */ + } else { + return duk_bd_decode(ctx, 2); + } +} + /* Decode a bit packed string from a custom format used by genbuiltins.py. * This function is here because it's used for both heap and thread inits. * Caller must supply the output buffer whose size is NOT checked! diff --git a/tools/dukutil.py b/tools/dukutil.py index b5d74bb8..408a7756 100644 --- a/tools/dukutil.py +++ b/tools/dukutil.py @@ -10,9 +10,15 @@ class BitEncoder: "Bitstream encoder." _bits = None + _varuint_dist = None + _varuint_count = None + _varuint_bits = None def __init__(self): self._bits = [] + self._varuint_dist = [ 0 ] * 65536 + self._varuint_count = 0 + self._varuint_bits = 0 def bits(self, x, nbits): if (x >> nbits) != 0: @@ -26,6 +32,32 @@ class BitEncoder: for shift in xrange(7, -1, -1): # 7, 6, ..., 0 self._bits.append((ch >> shift) & 0x01) + # Shared varint encoding. + def varuint(self, x): + assert(x >= 0) + if x <= 0xffff: + self._varuint_dist[x] += 1 + self._varuint_count += 1 + + if x <= 3: + self.bits(0, 1) + self.bits(x, 2) + self._varuint_bits += 3 + elif x <= 65: + self.bits(1, 1) + self.bits(x - 4 + 2, 6) + self._varuint_bits += 7 + elif x <= 65535: + self.bits(1, 1) + self.bits(0, 6) + self.bits(x, 16) + self._varuint_bits += 7 + 16 + else: + self.bits(1, 1) + self.bits(1, 6) + self.bits(x, 32) + self._varuint_bits += 7 + 32 + def getNumBits(self): "Get current number of encoded bits." return len(self._bits) diff --git a/tools/genbuiltins.py b/tools/genbuiltins.py index b68fe620..0f135ddd 100644 --- a/tools/genbuiltins.py +++ b/tools/genbuiltins.py @@ -1350,14 +1350,10 @@ CLASS_BITS = 5 BIDX_BITS = 7 STRIDX_BITS = 9 # would be nice to optimize to 8 NATIDX_BITS = 8 -NUM_NORMAL_PROPS_BITS = 8 -NUM_FUNC_PROPS_BITS = 8 PROP_FLAGS_BITS = 3 LENGTH_PROP_BITS = 3 NARGS_BITS = 3 PROP_TYPE_BITS = 3 -MAGIC_BITS = 16 -ACCESSOR_MAGIC_BITS = 2 NARGS_VARARGS_MARKER = 0x07 NO_CLASS_MARKER = 0x00 # 0 = DUK_HOBJECT_CLASS_NONE @@ -1518,6 +1514,10 @@ def gen_ramstr_initdata_bitpacked(meta): # end marker not necessary, C code knows length from define + if be._varuint_count > 0: + logger.debug('Varuint distribution:') + logger.debug(json.dumps(be._varuint_dist[0:1024])) + logger.debug('Varuint efficiency: %f bits/value' % (float(be._varuint_bits) / float(be._varuint_count))) res = be.getByteString() logger.debug(('%d ram strings, %d bytes of string init data, %d maximum string length, ' + \ @@ -1683,13 +1683,9 @@ def gen_ramobj_initdata_for_object(meta, be, bi, string_to_stridx, natfunc_name_ # Convert signed magic to 16-bit unsigned for encoding magic = resolve_magic(bi.get('magic'), objid_to_bidx) & 0xffff - if magic != 0: - assert(magic >= 0) - assert(magic < (1 << MAGIC_BITS)) - be.bits(1, 1) - be.bits(magic, MAGIC_BITS) - else: - be.bits(0, 1) + assert(magic >= 0) + assert(magic <= 0xffff) + be.varuint(magic) # Generate RAM object initdata for an object's properties. def gen_ramobj_initdata_for_props(meta, be, bi, string_to_stridx, natfunc_name_to_natidx, objid_to_bidx, double_byte_order): @@ -1778,7 +1774,7 @@ def gen_ramobj_initdata_for_props(meta, be, bi, string_to_stridx, natfunc_name_t else: values.append(prop) - be.bits(len(values), NUM_NORMAL_PROPS_BITS) + be.varuint(len(values)) for valspec in values: count_normal_props += 1 @@ -1881,7 +1877,7 @@ def gen_ramobj_initdata_for_props(meta, be, bi, string_to_stridx, natfunc_name_t assert(getter_magic == setter_magic) _natidx(getter_natfun) _natidx(setter_natfun) - be.bits(getter_magic, ACCESSOR_MAGIC_BITS) + be.varuint(getter_magic) elif val['type'] == 'lightfunc': logger.warning('RAM init data format doesn\'t support "lightfunc" now, value replaced with "undefined": %r' % valspec) be.bits(PROP_TYPE_UNDEFINED, PROP_TYPE_BITS) @@ -1890,7 +1886,7 @@ def gen_ramobj_initdata_for_props(meta, be, bi, string_to_stridx, natfunc_name_t else: raise Exception('unsupported value: %s' % repr(val)) - be.bits(len(functions), NUM_FUNC_PROPS_BITS) + be.varuint(len(functions)) for funprop in functions: count_function_props += 1 @@ -1918,13 +1914,9 @@ def gen_ramobj_initdata_for_props(meta, be, bi, string_to_stridx, natfunc_name_t # (there are quite a lot of function properties) # Convert signed magic to 16-bit unsigned for encoding magic = resolve_magic(funobj.get('magic'), objid_to_bidx) & 0xffff - if magic != 0: - assert(magic >= 0) - assert(magic < (1 << MAGIC_BITS)) - be.bits(1, 1) - be.bits(magic, MAGIC_BITS) - else: - be.bits(0, 1) + assert(magic >= 0) + assert(magic <= 0xffff) + be.varuint(magic) return count_normal_props, count_function_props @@ -1987,6 +1979,10 @@ def gen_ramobj_initdata_bitpacked(meta, native_funcs, natfunc_name_to_natidx, do count_normal_props += count_obj_normal count_function_props += count_obj_func + if be._varuint_count > 0: + logger.debug('varuint distribution:') + logger.debug(json.dumps(be._varuint_dist[0:1024])) + logger.debug('Varuint efficiency: %f bits/value' % (float(be._varuint_bits) / float(be._varuint_count))) romobj_init_data = be.getByteString() #logger.debug(repr(romobj_init_data)) #logger.debug(len(romobj_init_data))