github
/
duktape
mirror of https://github.com/svaarala/duktape.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Merge pull request #1151 from svaarala/varint-bits-initdata 

Use a shared varuint bit-packed type for some built-ins initdata fields
pull/1152/head
Sami Vaarala 8 years ago
committed by GitHub
parent
e5adf5ccd2 a79aa870e3
commit
47b96b2cbe
6 changed files with 87 additions and 40 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      RELEASES.rst
  2. 14
      src-input/duk_hthread_builtins.c
  3. 7
      src-input/duk_util.h
  4. 32
      src-input/duk_util_bitdecoder.c
  5. 32
      tools/dukutil.py
  6. 38
      tools/genbuiltins.py

4
RELEASES.rst
View File

@ -2012,8 +2012,8 @@ Planned
* Add an fmod() self test (GH-1108)
* Reduce RAM built-ins initdata limitations for custom bindings by bumping
bit count for normal and function properties from 6 to 8 (GH-FIXME)
* Reduce RAM built-ins initdata limitations for custom bindings by using a
shared varuint encoding in the bit-packed initdata stream (GH-1151)
* Fix JSON stringify fastpath handling of array gaps in JX and JC; they
incorrectly stringified as 'null' (like in JSON) instead of 'undefined'

14
src-input/duk_hthread_builtins.c
View File

@ -14,14 +14,10 @@
#define DUK__BIDX_BITS 7
#define DUK__STRIDX_BITS 9 /* XXX: try to optimize to 8 (would now be possible, <200 used) */
#define DUK__NATIDX_BITS 8
#define DUK__NUM_NORMAL_PROPS_BITS 8
#define DUK__NUM_FUNC_PROPS_BITS 8
#define DUK__PROP_FLAGS_BITS 3
#define DUK__LENGTH_PROP_BITS 3
#define DUK__NARGS_BITS 3
#define DUK__PROP_TYPE_BITS 3
#define DUK__MAGIC_BITS 16
#define DUK__ACCESSOR_MAGIC_BITS 2 /* just a few shared accessors now */
#define DUK__NARGS_VARARGS_MARKER 0x07
#define DUK__NO_CLASS_MARKER 0x00 /* 0 = DUK_HOBJECT_CLASS_NONE */
@ -292,7 +288,7 @@ DUK_INTERNAL void duk_hthread_create_builtin_objects(duk_hthread *thr) {
}
/* Cast converts magic to 16-bit signed value */
magic = (duk_int16_t) duk_bd_decode_flagged(bd, DUK__MAGIC_BITS, 0 /*def_value*/);
magic = (duk_int16_t) duk_bd_decode_varuint(bd);
((duk_hnatfunc *) h)->magic = magic;
} else if (class_num == DUK_HOBJECT_CLASS_ARRAY) {
duk_push_array(ctx);
@ -406,7 +402,7 @@ DUK_INTERNAL void duk_hthread_create_builtin_objects(duk_hthread *thr) {
}
/* normal valued properties */
num = (duk_small_uint_t) duk_bd_decode(bd, DUK__NUM_NORMAL_PROPS_BITS);
num = (duk_small_uint_t) duk_bd_decode_varuint(bd);
DUK_DDD(DUK_DDDPRINT("built-in object %ld, %ld normal valued properties", (long) i, (long) num));
for (j = 0; j < num; j++) {
duk_small_uint_t defprop_flags;
@ -479,7 +475,7 @@ DUK_INTERNAL void duk_hthread_create_builtin_objects(duk_hthread *thr) {
case DUK__PROP_TYPE_ACCESSOR: {
duk_small_uint_t natidx_getter = (duk_small_uint_t) duk_bd_decode(bd, DUK__NATIDX_BITS);
duk_small_uint_t natidx_setter = (duk_small_uint_t) duk_bd_decode(bd, DUK__NATIDX_BITS);
duk_small_uint_t accessor_magic = (duk_small_uint_t) duk_bd_decode(bd, DUK__ACCESSOR_MAGIC_BITS);
duk_small_uint_t accessor_magic = (duk_small_uint_t) duk_bd_decode_varuint(bd);
duk_c_function c_func_getter;
duk_c_function c_func_setter;
@ -517,7 +513,7 @@ DUK_INTERNAL void duk_hthread_create_builtin_objects(duk_hthread *thr) {
}
/* native function properties */
num = (duk_small_uint_t) duk_bd_decode(bd, DUK__NUM_FUNC_PROPS_BITS);
num = (duk_small_uint_t) duk_bd_decode_varuint(bd);
DUK_DDD(DUK_DDDPRINT("built-in object %ld, %ld function valued properties", (long) i, (long) num));
for (j = 0; j < num; j++) {
duk_hstring *h_key;
@ -549,7 +545,7 @@ DUK_INTERNAL void duk_hthread_create_builtin_objects(duk_hthread *thr) {
(c_nargs == DUK_VARARGS ? (long) -1 : (long) c_nargs)));
/* Cast converts magic to 16-bit signed value */
magic = (duk_int16_t) duk_bd_decode_flagged(bd, DUK__MAGIC_BITS, 0);
magic = (duk_int16_t) duk_bd_decode_varuint(bd);
#if defined(DUK_USE_LIGHTFUNC_BUILTINS)
lightfunc_eligible =

7
src-input/duk_util.h
View File

@ -508,9 +508,10 @@ DUK_INTERNAL_DECL duk_uint32_t duk_util_hashbytes(const duk_uint8_t *data, duk_s
DUK_INTERNAL_DECL duk_uint32_t duk_util_get_hash_prime(duk_uint32_t size);
#endif
DUK_INTERNAL_DECL duk_int32_t duk_bd_decode(duk_bitdecoder_ctx *ctx, duk_small_int_t bits);
DUK_INTERNAL_DECL duk_small_int_t duk_bd_decode_flag(duk_bitdecoder_ctx *ctx);
DUK_INTERNAL_DECL duk_int32_t duk_bd_decode_flagged(duk_bitdecoder_ctx *ctx, duk_small_int_t bits, duk_int32_t def_value);
DUK_INTERNAL_DECL duk_uint32_t duk_bd_decode(duk_bitdecoder_ctx *ctx, duk_small_int_t bits);
DUK_INTERNAL_DECL duk_small_uint_t duk_bd_decode_flag(duk_bitdecoder_ctx *ctx);
DUK_INTERNAL_DECL duk_uint32_t duk_bd_decode_flagged(duk_bitdecoder_ctx *ctx, duk_small_int_t bits, duk_uint32_t def_value);
DUK_INTERNAL_DECL duk_uint32_t duk_bd_decode_varuint(duk_bitdecoder_ctx *ctx);
DUK_INTERNAL_DECL duk_small_uint_t duk_bd_decode_bitpacked_string(duk_bitdecoder_ctx *bd, duk_uint8_t *out);
DUK_INTERNAL_DECL void duk_be_encode(duk_bitencoder_ctx *ctx, duk_uint32_t data, duk_small_int_t bits);

32
src-input/duk_util_bitdecoder.c
View File

@ -8,7 +8,7 @@
* When reading past bitstream end, zeroes are shifted in. The result
* is signed to match duk_bd_decode_flagged.
*/
DUK_INTERNAL duk_int32_t duk_bd_decode(duk_bitdecoder_ctx *ctx, duk_small_int_t bits) {
DUK_INTERNAL duk_uint32_t duk_bd_decode(duk_bitdecoder_ctx *ctx, duk_small_int_t bits) {
duk_small_int_t shift;
duk_uint32_t mask;
duk_uint32_t tmp;
@ -53,22 +53,44 @@ DUK_INTERNAL duk_int32_t duk_bd_decode(duk_bitdecoder_ctx *ctx, duk_small_int_t
return tmp;
}
DUK_INTERNAL duk_small_int_t duk_bd_decode_flag(duk_bitdecoder_ctx *ctx) {
return (duk_small_int_t) duk_bd_decode(ctx, 1);
DUK_INTERNAL duk_small_uint_t duk_bd_decode_flag(duk_bitdecoder_ctx *ctx) {
return (duk_small_uint_t) duk_bd_decode(ctx, 1);
}
/* Decode a one-bit flag, and if set, decode a value of 'bits', otherwise return
* default value. Return value is signed so that negative marker value can be
* used by caller as a "not present" value.
*/
DUK_INTERNAL duk_int32_t duk_bd_decode_flagged(duk_bitdecoder_ctx *ctx, duk_small_int_t bits, duk_int32_t def_value) {
DUK_INTERNAL duk_uint32_t duk_bd_decode_flagged(duk_bitdecoder_ctx *ctx, duk_small_int_t bits, duk_uint32_t def_value) {
if (duk_bd_decode_flag(ctx)) {
return (duk_int32_t) duk_bd_decode(ctx, bits);
return duk_bd_decode(ctx, bits);
} else {
return def_value;
}
}
/* Shared varint encoding. Match dukutil.py BitEncode.varuint(). */
DUK_INTERNAL duk_uint32_t duk_bd_decode_varuint(duk_bitdecoder_ctx *ctx) {
duk_small_uint_t t;
/* Numbers 0-3 dominate input heavily, so use an initial flag byte for
* them, encoding into 3 bits. Other numbers are most often in [4,63]
* so encode that range efficiently with 7 bits. Reserve a few special
* values for rare long encodings.
*/
if (duk_bd_decode_flag(ctx)) {
t = duk_bd_decode(ctx, 6);
if (t == 0) {
return duk_bd_decode(ctx, 16);
} else if (t == 1) {
return duk_bd_decode(ctx, 32);
}
return (t - 2) + 4; /* Covers [4,65] */
} else {
return duk_bd_decode(ctx, 2);
}
}
/* Decode a bit packed string from a custom format used by genbuiltins.py.
* This function is here because it's used for both heap and thread inits.
* Caller must supply the output buffer whose size is NOT checked!

32
tools/dukutil.py
View File

@ -10,9 +10,15 @@ class BitEncoder:
"Bitstream encoder."
_bits = None
_varuint_dist = None
_varuint_count = None
_varuint_bits = None
def __init__(self):
self._bits = []
self._varuint_dist = [ 0 ] * 65536
self._varuint_count = 0
self._varuint_bits = 0
def bits(self, x, nbits):
if (x >> nbits) != 0:
@ -26,6 +32,32 @@ class BitEncoder:
for shift in xrange(7, -1, -1): # 7, 6, ..., 0
self._bits.append((ch >> shift) & 0x01)
# Shared varint encoding.
def varuint(self, x):
assert(x >= 0)
if x <= 0xffff:
self._varuint_dist[x] += 1
self._varuint_count += 1
if x <= 3:
self.bits(0, 1)
self.bits(x, 2)
self._varuint_bits += 3
elif x <= 65:
self.bits(1, 1)
self.bits(x - 4 + 2, 6)
self._varuint_bits += 7
elif x <= 65535:
self.bits(1, 1)
self.bits(0, 6)
self.bits(x, 16)
self._varuint_bits += 7 + 16
else:
self.bits(1, 1)
self.bits(1, 6)
self.bits(x, 32)
self._varuint_bits += 7 + 32
def getNumBits(self):
"Get current number of encoded bits."
return len(self._bits)

38
tools/genbuiltins.py
View File

@ -1350,14 +1350,10 @@ CLASS_BITS = 5
BIDX_BITS = 7
STRIDX_BITS = 9 # would be nice to optimize to 8
NATIDX_BITS = 8
NUM_NORMAL_PROPS_BITS = 8
NUM_FUNC_PROPS_BITS = 8
PROP_FLAGS_BITS = 3
LENGTH_PROP_BITS = 3
NARGS_BITS = 3
PROP_TYPE_BITS = 3
MAGIC_BITS = 16
ACCESSOR_MAGIC_BITS = 2
NARGS_VARARGS_MARKER = 0x07
NO_CLASS_MARKER = 0x00 # 0 = DUK_HOBJECT_CLASS_NONE
@ -1518,6 +1514,10 @@ def gen_ramstr_initdata_bitpacked(meta):
# end marker not necessary, C code knows length from define
if be._varuint_count > 0:
logger.debug('Varuint distribution:')
logger.debug(json.dumps(be._varuint_dist[0:1024]))
logger.debug('Varuint efficiency: %f bits/value' % (float(be._varuint_bits) / float(be._varuint_count)))
res = be.getByteString()
logger.debug(('%d ram strings, %d bytes of string init data, %d maximum string length, ' + \
@ -1683,13 +1683,9 @@ def gen_ramobj_initdata_for_object(meta, be, bi, string_to_stridx, natfunc_name_
# Convert signed magic to 16-bit unsigned for encoding
magic = resolve_magic(bi.get('magic'), objid_to_bidx) & 0xffff
if magic != 0:
assert(magic >= 0)
assert(magic < (1 << MAGIC_BITS))
be.bits(1, 1)
be.bits(magic, MAGIC_BITS)
else:
be.bits(0, 1)
assert(magic >= 0)
assert(magic <= 0xffff)
be.varuint(magic)
# Generate RAM object initdata for an object's properties.
def gen_ramobj_initdata_for_props(meta, be, bi, string_to_stridx, natfunc_name_to_natidx, objid_to_bidx, double_byte_order):
@ -1778,7 +1774,7 @@ def gen_ramobj_initdata_for_props(meta, be, bi, string_to_stridx, natfunc_name_t
else:
values.append(prop)
be.bits(len(values), NUM_NORMAL_PROPS_BITS)
be.varuint(len(values))
for valspec in values:
count_normal_props += 1
@ -1881,7 +1877,7 @@ def gen_ramobj_initdata_for_props(meta, be, bi, string_to_stridx, natfunc_name_t
assert(getter_magic == setter_magic)
_natidx(getter_natfun)
_natidx(setter_natfun)
be.bits(getter_magic, ACCESSOR_MAGIC_BITS)
be.varuint(getter_magic)
elif val['type'] == 'lightfunc':
logger.warning('RAM init data format doesn\'t support "lightfunc" now, value replaced with "undefined": %r' % valspec)
be.bits(PROP_TYPE_UNDEFINED, PROP_TYPE_BITS)
@ -1890,7 +1886,7 @@ def gen_ramobj_initdata_for_props(meta, be, bi, string_to_stridx, natfunc_name_t
else:
raise Exception('unsupported value: %s' % repr(val))
be.bits(len(functions), NUM_FUNC_PROPS_BITS)
be.varuint(len(functions))
for funprop in functions:
count_function_props += 1
@ -1918,13 +1914,9 @@ def gen_ramobj_initdata_for_props(meta, be, bi, string_to_stridx, natfunc_name_t
# (there are quite a lot of function properties)
# Convert signed magic to 16-bit unsigned for encoding
magic = resolve_magic(funobj.get('magic'), objid_to_bidx) & 0xffff
if magic != 0:
assert(magic >= 0)
assert(magic < (1 << MAGIC_BITS))
be.bits(1, 1)
be.bits(magic, MAGIC_BITS)
else:
be.bits(0, 1)
assert(magic >= 0)
assert(magic <= 0xffff)
be.varuint(magic)
return count_normal_props, count_function_props
@ -1987,6 +1979,10 @@ def gen_ramobj_initdata_bitpacked(meta, native_funcs, natfunc_name_to_natidx, do
count_normal_props += count_obj_normal
count_function_props += count_obj_func
if be._varuint_count > 0:
logger.debug('varuint distribution:')
logger.debug(json.dumps(be._varuint_dist[0:1024]))
logger.debug('Varuint efficiency: %f bits/value' % (float(be._varuint_bits) / float(be._varuint_count)))
romobj_init_data = be.getByteString()
#logger.debug(repr(romobj_init_data))
#logger.debug(len(romobj_init_data))

Write Preview
Loading…
Cancel
Save