From 7b7efd80785b7125713bc61abde0cbc167264797 Mon Sep 17 00:00:00 2001 From: Sami Vaarala Date: Sat, 21 Mar 2020 16:44:58 +0200 Subject: [PATCH] Add JS-based RAM initdata support --- src-tools/lib/builtins/magic.js | 151 ++++++ src-tools/lib/builtins/metadata/gc.js | 134 +++++ src-tools/lib/builtins/ram_initdata.js | 672 +++++++++++++++++++++++++ src-tools/lib/util/bstr.js | 54 ++ src-tools/lib/util/clone.js | 9 + src-tools/lib/util/string_util.js | 18 + 6 files changed, 1038 insertions(+) create mode 100644 src-tools/lib/builtins/magic.js create mode 100644 src-tools/lib/builtins/metadata/gc.js create mode 100644 src-tools/lib/builtins/ram_initdata.js diff --git a/src-tools/lib/builtins/magic.js b/src-tools/lib/builtins/magic.js new file mode 100644 index 00000000..db469172 --- /dev/null +++ b/src-tools/lib/builtins/magic.js @@ -0,0 +1,151 @@ +'use strict'; + +const { assert } = require('../util/assert'); +const { createBareObject } = require('../util/bare'); + +const mathOneargMagic = createBareObject({ + fabs: 0, // BI_MATH_FABS_IDX + acos: 1, // BI_MATH_ACOS_IDX + asin: 2, // BI_MATH_ASIN_IDX + atan: 3, // BI_MATH_ATAN_IDX + ceil: 4, // BI_MATH_CEIL_IDX + cos: 5, // BI_MATH_COS_IDX + exp: 6, // BI_MATH_EXP_IDX + floor: 7, // BI_MATH_FLOOR_IDX + log: 8, // BI_MATH_LOG_IDX + round: 9, // BI_MATH_ROUND_IDX + sin: 10, // BI_MATH_SIN_IDX + sqrt: 11, // BI_MATH_SQRT_IDX + tan: 12, // BI_MATH_TAN_IDX + cbrt: 13, // BI_MATH_CBRT_IDX + log2: 14, // BI_MATH_LOG2_IDX + log10: 15, // BI_MATH_LOG10_IDX + trunc: 16 // BI_MATH_TRUNC_IDX +}); + +const mathTwoargMagic = createBareObject({ + atan2: 0, // BI_MATH_ATAN2_IDX + pow: 1 // BI_MATH_POW_IDX +}); + +const arrayIterMagic = createBareObject({ + every: 0, // BI_ARRAY_ITER_EVERY + some: 1, // BI_ARRAY_ITER_SOME + forEach: 2, // BI_ARRAY_ITER_FOREACH + map: 3, // BI_ARRAY_ITER_MAP + filter: 4 // BI_ARRAY_ITER_FILTER +}); + +// Magic value for typedarray/node.js buffer read field operations. +function magicReadField(elem, signed, bigendian, typedarray) { + assert(typeof signed === 'boolean'); + assert(typeof bigendian === 'boolean'); + assert(typeof typedarray === 'boolean'); + + // Must match duk__FLD_xxx in duk_bi_buffer.c. + var elemNum = createBareObject({ + '8bit': 0, + '16bit': 1, + '32bit': 2, + 'float': 3, + 'double': 4, + 'varint': 5 + })[elem]; + var signedNum = (signed ? 1 : 0); + var bigendianNum = (bigendian ? 1 : 0); + var typedarrayNum = (typedarray ? 1 : 0); + return elemNum + (signedNum << 4) + (bigendianNum << 3) + (typedarrayNum << 5); +} +exports.magicReadField = magicReadField; + +// Magic value for typedarray/node.js buffer write field operations. +function magicWriteField(elem, signed, bigendian, typedarray) { + return magicReadField(elem, signed, bigendian, typedarray); +} +exports.magicWriteField = magicWriteField; + +// Magic value for typedarray constructors. +function magicTypedArrayConstructor(elem, shift) { + // Must match duk_hbufobj.h header. + var elemNum = createBareObject({ + 'uint8': 0, + 'uint8clamped': 1, + 'int8': 2, + 'uint16': 3, + 'int16': 4, + 'uint32': 5, + 'int32': 6, + 'float32': 7, + 'float64': 8 + })[elem]; + return (elemNum << 2) + shift; +} +exports.magicTypedArrayConstructor = magicTypedArrayConstructor; + +function resolveMagic(elem, objIdToBidx) { + var tmp; + + console.debug('resolve magic:', elem); + + if (elem === void 0 || elem === null) { + return 0; + } + if (typeof elem === 'number') { + elem = { type: 'plain', value: elem }; + } + if (!(typeof elem === 'object' && elem !== null)) { + throw new TypeError('invalid magic'); + } + + switch (elem.type) { + case 'bidx': { + // Maps to thr->builtins[]. + tmp = objIdToBidx[elem.id]; + if (typeof tmp !== 'number') { + throw new TypeError('invalid bidx magic: ' + elem.id); + } + return tmp; + } + case 'plain': { + let v = Math.floor(elem.value); + if (!(v >= -0x8000 && v <= 0x7fff)) { + throw new TypeError('invalid plain value for magic: ' + v); + } + return v; + } + case 'math_onearg': { + tmp = mathOneargMagic[elem.funcname]; + if (typeof tmp !== 'number') { + throw new TypeError('invalid math_onearg magic: ' + elem.funcname); + } + return tmp; + } + case 'math_twoarg': { + tmp = mathTwoargMagic[elem.funcname]; + if (typeof tmp !== 'number') { + throw new TypeError('invalid math_twoarg magic: ' + elem.funcname); + } + return tmp; + } + case 'array_iter': { + tmp = arrayIterMagic[elem.funcname]; + if (typeof tmp !== 'number') { + throw new TypeError('invalid array_iter magic: ' + elem.funcname); + } + return tmp; + } + case 'typedarray_constructor': { + return magicTypedArrayConstructor(elem.elem, elem.shift); + } + case 'buffer_readfield': { + return magicReadField(elem.elem, elem.signed, elem.bigendian, elem.typedarray); + } + case 'buffer_writefield': { + return magicWriteField(elem.elem, elem.signed, elem.bigendian, elem.typedarray); + } + default: { + throw new TypeError('invalid magic type: ' + elem.type); + } + } +} +exports.resolveMagic = resolveMagic; diff --git a/src-tools/lib/builtins/metadata/gc.js b/src-tools/lib/builtins/metadata/gc.js new file mode 100644 index 00000000..b0c97a95 --- /dev/null +++ b/src-tools/lib/builtins/metadata/gc.js @@ -0,0 +1,134 @@ +/* + * Garbage collect built-in objects and strings based on reachability roots. + */ + +'use strict'; + +const { + walkObjects, + walkStrings, + walkObjectProperties, + walkObjectsAndProperties, + propDefault +} = require('./util'); +const { createBareObject } = require('../../util/bare'); + +// Mark objects with 'bidx' forcibly reachable. +function markBidxObjectsReachable(meta) { + walkObjects(meta, (o) => { + if (propDefault(o, 'bidx_used', false)) { + o._force_reachable = 'bidx_used'; + } + }); +} +exports.markBidxObjectsReachable = markBidxObjectsReachable; + +// Mark strings with actively referenced 'stridx' forcibly reachable. +function markStridxStringsReachable(meta) { + walkStrings(meta, (s) => { + if (s.stridx_used) { + s._force_reachable = 'stridx'; + } + }); +} +exports.markStridxStringsReachable = markStridxStringsReachable; + +// Delete objects and strings not reachable from reachability roots or +// forced to be reachable. Such objects can't be reached at runtime +// so they're useless in RAM or ROM init data. +function removeUnreachableObjectsAndStrings(meta) { + var reachable = createBareObject({}); + + // First prune objects: keep only reachable and forced objects. + + walkObjects(meta, (o) => { + if (propDefault(o, '_force_reachable', false)) { + reachable[o.id] = true; + } + }); + + function markId(objId) { + if (objId) { + reachable[objId] = true; + } + } + + // Keep marking until steady state. + console.debug('original object count: ' + meta.objects.length); + for (;;) { + let reachableCount = Object.keys(reachable).length; + + walkObjects(meta, (o) => { + if (!reachable[o.id]) { + return; + } + markId(o.internal_prototype); + walkObjectProperties(o, (p) => { + // Shorthand has been normalized so no need + // to support it here. + let v = p.value; + if (typeof v === 'object' && v !== null) { + switch (v.type) { + case 'object': + markId(v.id); + break; + case 'accessor': + markId(v.getter_id); + markId(v.setter_id); + break; + } + } + }); + }); + + let newReachableCount = Object.keys(reachable).length; + console.debug('mark reachable, reachable count ' + reachableCount + ' -> ' + newReachableCount); + + if (reachableCount === newReachableCount) { + break; + } + } + + let numDeleted = 0; + meta.objects = meta.objects.filter((o) => { + if (reachable[o.id]) { + return true; + } else { + console.debug('object ' + o.id + ' not reachable, dropping: ' + JSON.stringify(o)); + numDeleted++; + return false; + } + }); + + // Then prune strings: keep only reachable and forced strings. + // (This is not very relevant for RAM initdata because only + // strings with stridx are ultimately kept.) + + var reachableStrings = {}; + var numDeletedStrings = 0; + walkObjectsAndProperties(meta, null, (p, o) => { + void o; + reachableStrings[p.key] = true; + if (typeof p.value === 'string') { + reachableStrings[p.value] = true; + } + }); + meta.strings = meta.strings.filter((s) => { + if (reachableStrings[s.str]) { + return true; + } else if (s._force_reachable) { + console.debug('string not reachable but forced, keep:', s.str); + return true; + } else { + console.debug('string not reachable, drop:', s.str); + numDeletedStrings++; + return false; + } + }); + + if (numDeleted > 0 || numDeletedStrings > 0) { + console.log('deleted ' + numDeleted + ' unreachable objects, ' + + numDeletedStrings + ' unreachable strings'); + } +} +exports.removeUnreachableObjectsAndStrings = removeUnreachableObjectsAndStrings; diff --git a/src-tools/lib/builtins/ram_initdata.js b/src-tools/lib/builtins/ram_initdata.js new file mode 100644 index 00000000..3aa0d078 --- /dev/null +++ b/src-tools/lib/builtins/ram_initdata.js @@ -0,0 +1,672 @@ +'use strict'; + +const { BitEncoder } = require('../util/bitencoder'); +const { bitpack5BitBstr } = require('../formats/bitpack_5bit'); +const { walkObjectsAndProperties, findObjectById, findPropertyByKey, propDefault } = require('./metadata/util'); +const { classToNumber } = require('./classnames'); +const { resolveMagic } = require('./magic'); +const { assert } = require('../util/assert'); +const { hexDecode } = require('../util/hex'); +const { shallowCloneArray } = require('../util/clone'); +const { createBareObject } = require('../util/bare'); + +// Default property attributes. +const LENGTH_PROPERTY_ATTRIBUTES = 'c'; +//const ACCESSOR_PROPERTY_ATTRIBUTES = 'c'; +const DEFAULT_DATA_PROPERTY_ATTRIBUTES = 'wc'; +const DEFAULT_FUNC_PROPERTY_ATTRIBUTES = 'wc'; + +// Encoding constants (must match duk_hthread_builtins.c). +const PROP_FLAGS_BITS = 3; +const LENGTH_PROP_BITS = 3; +const NARGS_BITS = 3; +const PROP_TYPE_BITS = 3; + +const NARGS_VARARGS_MARKER = 0x07; + +const PROP_TYPE_DOUBLE = 0; +const PROP_TYPE_STRING = 1; +const PROP_TYPE_STRIDX = 2; +const PROP_TYPE_BUILTIN = 3; +const PROP_TYPE_UNDEFINED = 4; +const PROP_TYPE_BOOLEAN_TRUE = 5; +const PROP_TYPE_BOOLEAN_FALSE = 6; +const PROP_TYPE_ACCESSOR = 7; + +// Property descriptor flags, must match duk_hobject.h. +const PROPDESC_FLAG_WRITABLE = (1 << 0); +const PROPDESC_FLAG_ENUMERABLE = (1 << 1); +const PROPDESC_FLAG_CONFIGURABLE = (1 << 2); +const PROPDESC_FLAG_ACCESSOR = (1 << 3); + +// Encode property flags for RAM initializers. +function encodePropertyFlags(flags) { + var res = 0; + var nflags = 0; + if (flags.indexOf('w') >= 0) { + nflags++; + res |= PROPDESC_FLAG_WRITABLE; + } + if (flags.indexOf('e') >= 0) { + nflags++; + res |= PROPDESC_FLAG_ENUMERABLE; + } + if (flags.indexOf('c') >= 0) { + nflags++; + res |= PROPDESC_FLAG_CONFIGURABLE; + } + if (flags.indexOf('a') >= 0) { + nflags++; + res |= PROPDESC_FLAG_ACCESSOR; + } + + if (nflags !== flags.length) { + throw new TypeError('invalid property flags: ' + flags); + } + + return res; +} + +// Get helper maps for RAM objects. +function getRamobjNativeFuncMaps(meta) { + var nativeFound = {}; + var nativeFuncs = []; + var natfuncNameToNatidx = {}; + + nativeFuncs.push(null); // natidx 0 is reserved for NULL + + walkObjectsAndProperties(meta, (o) => { + if (typeof o.native !== 'undefined') { + nativeFound[o.native] = true; + } + }, (p, o) => { + void o; + let val = p.value; + if (typeof val === 'object' && val !== null) { + switch (val.type) { + case 'accessor': + if (typeof val.getter_id !== 'undefined') { + let getter = findObjectById(meta, val.getter_id); + nativeFound[getter.native] = true; + } + if (typeof val.setter_id !== 'undefined') { + let setter = findObjectById(meta, val.setter_id); + nativeFound[setter.native] = true; + } + break; + case 'object': + { + let target = findObjectById(meta, val.id); + if (typeof target.native !== 'undefined') { + nativeFound[target.native] = true; + } + } + break; + case 'lightfunc': + // No lightfunc support for RAM initializer now. + break; + } + } + }); + + Object.keys(nativeFound).sort().forEach((k, idx) => { + void idx; + natfuncNameToNatidx[k] = nativeFuncs.length; + nativeFuncs.push(k); // native func names + }); + + return { nativeFuncs, natfuncNameToNatidx }; +} +exports.getRamobjNativeFuncMaps = getRamobjNativeFuncMaps; + +// Generate bit-packed RAM string init data. +function generateRamStringInitDataBitpacked(meta) { + var be = new BitEncoder(); + + var maxLen = 0; + var stats = createBareObject({ + numInputBytes: 0, + numOptimal: 0, + numLookup1: 0, + numLookup2: 0, + numSwitch1: 0, + numSwitch: 0, + numEightBit: 0 + }); + + for (let strObj of meta.strings_stridx) { + let s = strObj.str; + maxLen = Math.max(maxLen, s.length); + bitpack5BitBstr(be, s, stats); + } + + // End marker not necessary, C code knows length from define. + + console.debug('RAM string init data: ' + be.getStatsString()); + let res = be.getBytes(); + + console.debug(meta.strings_stridx.length + ' ram strings, ' + stats.numInputBytes + ' input data bytes, ' + + res.length + ' bytes of string init data, ' + maxLen + ' maximum string length, ' + + (res.length * 8 / stats.numInputBytes) + ' bits/char, ' + + 'encoding stats: ' + JSON.stringify(stats)); + + return { data: res, maxLen: maxLen }; +} +exports.generateRamStringInitDataBitpacked = generateRamStringInitDataBitpacked; + +// Helper to find a property from a property list, remove it from the +// property list, and return the removed property. +function stealProp(props, key, opts) { + opts = opts || {}; + for (let i = 0; i < props.length; i++) { + let prop = props[i]; + if (prop.key === key) { + let isAccessor = (typeof prop.value === 'object' && prop.value !== null && prop.value.type === 'accessor'); + if (!isAccessor || opts.allowAccessor) { + var res = props.splice(i, 1); + return res[0]; + } + } + } +} + +// Generate init data for an object, no properties yet. +function generateRamObjectInitDataForObject(meta, be, obj, stringToStridx, natfuncNameToNatidx, objIdToBidx) { + function emitStridx(strval) { + var stridx = stringToStridx[strval]; + be.varuint(stridx); + } + void emitStridx; + function emitStridxOrString(strval) { + var stridx = stringToStridx[strval]; + if (typeof stridx === 'number') { + be.varuint(stridx + 1); + } else { + be.varuint(0); + bitpack5BitBstr(be, strval, null); + } + } + function emitNatidx(nativeName) { + var natidx = natfuncNameToNatidx[nativeName]; + be.varuint(natidx); + } + + var classNum = classToNumber(obj.class); + be.varuint(classNum); + + var props = shallowCloneArray(obj.properties); // Clone so we can steal. + + var propProto = stealProp(props, 'prototype', { allowAccessor: false }); + void propProto; + var propConstructor = stealProp(props, 'constructor', { allowAccessor: false }); + void propConstructor; + var propName = stealProp(props, 'name', { allowAccessor: false }); + var propLength = stealProp(props, 'length', { allowAccessor: false }); + + var length = -1; // default value, signifies varargs + if (propLength) { + assert(typeof propLength.value === 'number'); + length = propLength.value; + be.bits(1, 1); // flag: have length + be.bits(length, LENGTH_PROP_BITS); + } else { + be.bits(0, 1); // flag: no length + } + + // The attributes for 'length' are standard ("none") except for + // Array.prototype.length which must be writable (this is handled + // separately in duk_hthread_builtins.c). + + var lenAttrs = LENGTH_PROPERTY_ATTRIBUTES; + if (propLength) { + lenAttrs = propLength.attributes; + } + if (lenAttrs !== LENGTH_PROPERTY_ATTRIBUTES) { + // Attributes are assumed to be the same, except for Array.prototype. + if (obj.class !== 'Array') { // Array.prototype is the only one with this class + throw new TypeError('non-default length attributes for unexpected object'); + } + } + + // For 'Function' classed objects, emit the native function stuff. + // Unfortunately this is more or less a copy of what we do for + // function properties now. This should be addressed if a rework + // on the init format is done. + + if (obj.class === 'Function') { + emitNatidx(obj.native); + + // Nargs. + if (propDefault(obj, 'varargs', false)) { + be.bits(1, 1); // flag: non-default nargs + be.bits(NARGS_VARARGS_MARKER, NARGS_BITS); // varargs + } else if (typeof obj.nargs === 'number' && obj.nargs !== length) { + be.bits(1, 1); // flag: non-default nargs + be.bits(obj.nargs, NARGS_BITS); + } else { + assert(typeof length === 'number'); + be.bits(0, 1); // flag: default nargs is OK + } + + // Function .name. + assert(propName); + assert(typeof propName.value === 'string'); + emitStridxOrString(propName.value); + + // All Function-classed global level objects are callable + // (have [[Call]]) but not all are constructable (have + // [[Construct]]). Flag that. + assert(obj.callable === true); + if (propDefault(obj, 'constructable', false)) { + be.bits(1, 1); // flag: constructable + } else { + be.bits(0, 1); // flag: not constructable + } + // DUK_HOBJECT_FLAG_SPECIAL_CALL is handled at runtime without init data. + + // Convert signed magic to 16-bit unsigned for encoding. + var magic = resolveMagic(propDefault(obj, 'magic', null), objIdToBidx) & 0xffff; + assert(magic >= 0 && magic <= 0xffff); + be.varuint(magic); + } +} + +// Generate init data for object properties. +function generateRamObjectInitDataForProps(meta, be, obj, stringToStridx, natfuncNameToNatidx, objIdToBidx, doubleByteOrder) { + var countNormal = 0; + var countFunction = 0; + + function emitBidx(bi_id) { + be.varuint(objIdToBidx[bi_id]); + } + function emitBidxOrNone(bi_id) { + if (typeof bi_id === 'string') { + be.varuint(objIdToBidx[bi_id] + 1); + } else { + be.varuint(0); + } + } + function emitStridx(strval) { + var stridx = stringToStridx[strval]; + be.varuint(stridx); + } + function emitStridxOrString(strval) { + var stridx = stringToStridx[strval]; + if (typeof stridx === 'number') { + be.varuint(stridx + 1); + } else { + be.varuint(0); + bitpack5BitBstr(be, strval, null); + } + } + function emitNatidx(nativeName) { + var natidx; + if (typeof nativeName === 'string') { + natidx = natfuncNameToNatidx[nativeName]; + } else { + natidx = 0; // 0 is NULL in the native functions table, denotes missing function. + } + be.varuint(natidx); + } + + var props = shallowCloneArray(obj.properties); // Clone so we can steal. + + // Internal prototype: not an actual property so not in property list. + emitBidxOrNone(obj.internal_prototype); + + // External prototype: encoded specially, steal from property list. + var propProto = stealProp(props, 'prototype'); + if (propProto) { + assert(typeof propProto.value === 'object' && propProto.value !== null && propProto.value.type === 'object'); + assert(propProto.attributes === ''); + emitBidxOrNone(propProto.value.id); + } else { + emitBidxOrNone(null); + } + + // External constructor: encoded specially, steal from property list. + var propConstructor = stealProp(props, 'constructor'); + if (propConstructor) { + assert(typeof propConstructor.value === 'object' && propConstructor.value !== null && propConstructor.value.type === 'object'); + assert(propConstructor.attributes === 'wc'); + emitBidxOrNone(propConstructor.value.id); + } else { + emitBidxOrNone(null); + } + + // Name: encoded specially for function objects, so steal and ignore here. + if (obj.class === 'Function') { + let propName = stealProp(props, 'name', { allowAccessor: false }); + assert(propName); + assert(typeof propName.value === 'string'); + assert(propName.attributes === 'c'); + } + + // length: encoded specially, so steal and ignore. + var propLength = stealProp(props, 'length', { allowAccessor: false }); + void propLength; + + // Date.prototype.toGMTString needs special handling and is handled + // directly in duk_hthread_builtins.c; so steal and ignore here. + if (obj.id === 'bi_date_prototype') { + let propToGmtString = stealProp(props, 'toGMTString'); + console.debug('stole .toGMTString property'); + } + + // Split properties into non-toplevel functions and other properties. + // This split is a bit arbitrary, but is used to reduce flag bits in + // the bit stream. + var values = []; + var functions = []; + props.forEach((prop) => { + if (typeof prop.value === 'object' && prop.value !== null && prop.value.type === 'object') { + var target = findObjectById(meta, prop.value.id); + assert(target); + if (typeof target.native === 'string' && // native function + typeof target.bidx === 'undefined') { // but not a top level built-in + functions.push(prop); + return; + } + } + values.push(prop); + }); + console.debug(obj.id + ': ' + values.length + ' values, ' + functions.length + ' functions'); + + // Encode 'values'. + be.varuint(values.length); + + values.forEach((prop) => { + var val = prop.value; + + countNormal++; + + // Key. + emitStridxOrString(prop.key); + + // Attributes. Attribute check doesn't check for accessor flag; that + // is now automatically set by C code when value is an accessor type. + // Accessors must not have 'writable', so they'll always have + // non-default attributes (less footprint than adding a different + // default). + var defaultAttrs = DEFAULT_DATA_PROPERTY_ATTRIBUTES; + var attrs = propDefault(prop, 'attributes', defaultAttrs); + attrs = attrs.replace('a', ''); // RAM bitstream doesn't encode the 'accessor' attribute. + if (attrs !== defaultAttrs) { + console.debug('non-default attributes for ' + prop.key + ': ' + attrs + ' vs ' + defaultAttrs); + be.bits(1, 1); // flag: have custom attributes + be.bits(encodePropertyFlags(attrs), PROP_FLAGS_BITS); + } else { + be.bits(0, 1); // flag: no custom attributes + } + + // Value. + if (val === void 0 || val === null) { + // RAM format doesn't support "null", use undefined. + be.bits(PROP_TYPE_UNDEFINED, PROP_TYPE_BITS); + } else if (typeof val === 'boolean') { + if (val) { + be.bits(PROP_TYPE_BOOLEAN_TRUE, PROP_TYPE_BITS); + } else { + be.bits(PROP_TYPE_BOOLEAN_FALSE, PROP_TYPE_BITS); + } + } else if (typeof val === 'number' || (typeof val === 'object' && val !== null && val.type === 'double')) { + // Avoid converting a manually specified NaN temporarily into + // a float to avoid risk of e.g. NaN being replaced by another. + if (typeof val === 'object') { + val = hexDecode(val.bytes); + } else { + let tmpAb = new ArrayBuffer(8); + let tmpDv = new DataView(tmpAb); + tmpDv.setFloat64(0, val); // big endian + val = new Uint8Array(tmpAb); + } + assert(val instanceof Uint8Array); + assert(val.length === 8); + + be.bits(PROP_TYPE_DOUBLE, PROP_TYPE_BITS); + + // Encoding of double must match target architecture byte order. + var indexList = ({ + big: [ 0, 1, 2, 3, 4, 5, 6, 7 ], + little: [ 7, 6, 5, 4, 3, 2, 1, 0 ], + mixed: [ 3, 2, 1, 0, 7, 6, 5, 4 ] + })[doubleByteOrder]; + assert(indexList); + + let dataU8 = new Uint8Array(8); + for (let i = 0; i < indexList.length; i++) { + dataU8[i] = val[indexList[i]]; + } + be.uint8array(dataU8); + } else if (typeof val === 'string') { + let stridx = stringToStridx[val]; + if (typeof stridx === 'number') { + // String value is in built-in string table -> encode + // using a string index. This saves some space, + // especially for the 'name' property of errors + // ('EvalError' etc). + be.bits(PROP_TYPE_STRIDX, PROP_TYPE_BITS); + emitStridx(val); + } else { + // Not in string table, bitpack string value as is. + be.bits(PROP_TYPE_STRING, PROP_TYPE_BITS); + bitpack5BitBstr(be, val); + } + } else if (typeof val === 'object' && val !== null) { + if (val.type === 'object') { + be.bits(PROP_TYPE_BUILTIN, PROP_TYPE_BITS); + emitBidx(val.id); + } else if (val.type === 'undefined') { + be.bits(PROP_TYPE_UNDEFINED, PROP_TYPE_BITS); + } else if (val.type === 'accessor') { + be.bits(PROP_TYPE_ACCESSOR, PROP_TYPE_BITS); + let getterNatfun, setterNatfun; + let getterMagic = 0, setterMagic = 0; + if (typeof val.getter_id === 'string') { + let getterFn = findObjectById(meta, val.getter_id); + getterNatfun = getterFn.native; + assert(getterFn.nargs === 0); + getterMagic = getterFn.magic; + } + if (typeof val.setter_id === 'string') { + let setterFn = findObjectById(meta, val.setter_id); + setterNatfun = setterFn.native; + assert(setterFn.nargs === 1); + setterMagic = setterFn.magic; + } + if (getterNatfun && setterNatfun) { + assert(getterMagic === setterMagic); + } + emitNatidx(getterNatfun); + emitNatidx(setterNatfun); + be.varuint(getterMagic); + } else if (val.type === 'lightfunc') { + console.log('RAM init data format doesn\'t support "lightfunc" now, value replaced with "undefined" for ' + prop.name); + be.bits(PROP_TYPE_UNDEFINED, PROP_TYPE_BITS); + } else { + throw new TypeError('unsupported value'); + } + } else { + throw new TypeError('unsupported value'); + } + }); + + // Encode 'functions'. + be.varuint(functions.length); + + functions.forEach((prop) => { + var val = prop.value; + var funObj = findObjectById(meta, val.id); + assert(funObj); + var propLen = findPropertyByKey(funObj, 'length'); + assert(propLen); + assert(typeof propLen.value === 'number'); + var length = propLen.value; + + countFunction++; + + // Key. + emitStridxOrString(prop.key); + + // Native function. + emitNatidx(funObj.native); + + // Length. + be.bits(length, LENGTH_PROP_BITS); + + // Nargs. + if (propDefault(funObj, 'varargs', false)) { + be.bits(1, 1); // flag: non-default nargs + be.bits(NARGS_VARARGS_MARKER, NARGS_BITS); + } else if (typeof funObj.nargs === 'number' && funObj.nargs !== length) { + be.bits(1, 1); // flag: non-default nargs + be.bits(funObj.nargs, NARGS_BITS); + } else { + be.bits(0, 1); // flag: default nargs OK + } + + // Magic. + var magic = resolveMagic(propDefault(funObj, 'magic', null), objIdToBidx) & 0xffff; + assert(magic >= 0 && magic <= 0xffff); + be.varuint(magic); + + // Property attributes. + var defaultAttrs = DEFAULT_FUNC_PROPERTY_ATTRIBUTES; + var attrs = propDefault(prop, 'attributes', defaultAttrs); + attrs = attrs.replace('a', ''); // RAM bitstream doesn't encode the 'accessor' attribute. + if (attrs !== defaultAttrs) { + console.debug('non-default attributes for ' + prop.key + ': ' + attrs + ' vs ' + defaultAttrs); + be.bits(1, 1); // flag: have custom attributes + be.bits(encodePropertyFlags(attrs), PROP_FLAGS_BITS); + } else { + be.bits(0, 1); // flag: no custom attributes + } + }); + + return { countNormal, countFunction }; +} + +// Generate init data for objects and their properties. +function generateRamObjectInitDataBitpacked(meta, nativeFuncs, natfuncNameToNatidx, doubleByteOrder) { + // RAM initialization is based on a specially filtered list of top + // level objects which includes objects with 'bidx' and objects + // which aren't handled as inline values in the init bitstream. + + var objList = meta.objects_ram_toplevel + var objIdToBidx = meta._objid_to_ramidx; + var stringToStridx = meta._plain_to_stridx; + + var be = new BitEncoder(); + var countBuiltins = 0; + var countNormalProps = 0; + var countFunctionProps = 0; + + objList.forEach((obj) => { + countBuiltins++; + generateRamObjectInitDataForObject(meta, be, obj, stringToStridx, natfuncNameToNatidx, objIdToBidx); + }); + objList.forEach((obj) => { + var { countNormal, countFunction } = generateRamObjectInitDataForProps(meta, be, obj, stringToStridx, natfuncNameToNatidx, objIdToBidx, doubleByteOrder); + countNormalProps += countNormal; + countFunctionProps += countFunction; + }); + + var data = be.getBytes(); + console.debug('RAM object init data: ' + be.getStatsString()); + + console.debug(countBuiltins + ' ram builtins, ' + countNormalProps + ' normal properties, ' + + countFunctionProps + ' function properties, ' + data.length + ' bytes of RAM object init data'); + + return { data }; +} +exports.generateRamObjectInitDataBitpacked = generateRamObjectInitDataBitpacked; + +function emitRamObjectNativeFuncDeclarations(gcHdr, ramNativeFuncs) { + ramNativeFuncs.forEach((fname) => { + // Visibility depends on whether the function is Duktape internal or user. + // Use a simple prefix check. + if (fname === null) { + // Zero index is special. + return; + } + assert(typeof fname === 'string'); + let visibility = (fname.startsWith('duk_') ? 'DUK_INTERNAL_DECL' : 'extern'); + gcHdr.emitLine(visibility + ' duk_ret_t ' + fname + '(duk_context *ctx);'); + }); +} +exports.emitRamObjectNativeFuncDeclarations = emitRamObjectNativeFuncDeclarations; + +function emitRamObjectNativeFuncArray(gcSrc, ramNativeFuncs) { + gcSrc.emitLine('DUK_INTERNAL const duk_c_function duk_bi_native_functions[' + ramNativeFuncs.length + '] = {'); + ramNativeFuncs.forEach((fname, idx) => { + // The function pointer cast here makes BCC complain about + // "initializer too complicated", so omit the cast. + //gcSrc.emitLine('\t(duk_c_function) ' + func + ','); + let comma = (idx < ramNativeFuncs.length - 1 ? ',' : ''); + if (fname === null) { + gcSrc.emitLine('\tNULL' + comma); + } else { + assert(typeof fname === 'string'); + gcSrc.emitLine('\t' + fname + comma); + } + }); + gcSrc.emitLine('};'); +} +exports.emitRamObjectNativeFuncArray = emitRamObjectNativeFuncArray; + +function emitRamObjectNativeFuncArrayDeclaration(gcSrc, ramNativeFuncs) { + gcSrc.emitLine('DUK_INTERNAL_DECL const duk_c_function duk_bi_native_functions[' + ramNativeFuncs.length + '];'); +} +exports.emitRamObjectNativeFuncArrayDeclaration = emitRamObjectNativeFuncArrayDeclaration; + +function emitRamObjectInitData(gcSrc, data) { + gcSrc.emitArray(data, { + tableName: 'duk_builtins_data', + typeName: 'duk_uint8_t', + useConst: true, + useCast: false, + visibility: 'DUK_INTERNAL' + }); +} +exports.emitRamObjectInitData = emitRamObjectInitData; + +function emitRamObjectInitDataDeclaration(gcHdr, data) { + gcHdr.emitLine('#if !defined(DUK_SINGLE_FILE)'); + gcHdr.emitLine('DUK_INTERNAL_DECL const duk_uint8_t duk_builtins_data[' + data.length + '];'); + gcHdr.emitLine('#endif /* !DUK_SINGLE_FILE */'); + gcHdr.emitDefine('DUK_BUILTINS_DATA_LENGTH', data.length); +} +exports.emitRamObjectInitDataDeclaration = emitRamObjectInitDataDeclaration; + +function emitRamStringHeader(gcHdr, data, strMaxLen) { + gcHdr.emitLine('#if !defined(DUK_SINGLE_FILE)'); + gcHdr.emitLine('DUK_INTERNAL_DECL const duk_uint8_t duk_strings_data[' + data.length + '];'); + gcHdr.emitLine('#endif /* !DUK_SINGLE_FILE */'); + gcHdr.emitDefine('DUK_STRDATA_MAX_STRLEN', strMaxLen); + gcHdr.emitDefine('DUK_STRDATA_DATA_LENGTH', data.length); +} +exports.emitRamStringHeader = emitRamStringHeader; + +function emitRamStringInitData(gcSrc, data) { + gcSrc.emitArray(data, { + tableName: 'duk_strings_data', + typeName: 'duk_uint8_t', + useConst: true, + useCast: false, + visibility: 'DUK_INTERNAL' + }); +} +exports.emitRamStringInitData = emitRamStringInitData; + +function emitRamObjectHeader(gcHdr, meta) { + var objList = meta.objects_bidx; + objList.forEach((obj, idx) => { + var tmp = obj.id.toUpperCase().split('_').slice(1).join('_'); // bi_foo_bar -> FOO_BAR + var defName = 'DUK_BIDX_' + tmp; + gcHdr.emitDefine(defName, idx); + }); + gcHdr.emitDefine('DUK_NUM_BUILTINS', objList.length); + gcHdr.emitDefine('DUK_NUM_BIDX_BUILTINS', objList.length); // Objects with 'bidx' + gcHdr.emitDefine('DUK_NUM_ALL_BUILTINS', meta.objects_ram_toplevel.length); // Objects with 'bidx' + temps needed in init. +} +exports.emitRamObjectHeader = emitRamObjectHeader; diff --git a/src-tools/lib/util/bstr.js b/src-tools/lib/util/bstr.js index 801eed6d..78361fde 100644 --- a/src-tools/lib/util/bstr.js +++ b/src-tools/lib/util/bstr.js @@ -82,3 +82,57 @@ function validateStringsAreBstrRecursive(doc) { visit(doc); } exports.validateStringsAreBstrRecursive = validateStringsAreBstrRecursive; + +// Convert all strings in a document to Uint8Array. +function recursiveBstrToUint8Array(doc) { + function f(x) { + if (typeof x === 'string') { + return bstrToUint8Array(x); + } else if (typeof x === 'object' && x !== null && Array.isArray(x)) { + let res = []; + for (let i = 0; i < x.length; i++) { + res.push(f(x[i])); + } + return res; + } else if (typeof x === 'object' && x !== null) { + let res = {}; + for (let k in x) { + // Python tooling encoded key too; this doesn't work well in JS. + //res[f(k)] = f(x[k]); + res[k] = f(x[k]); + } + return res; + } else { + return x; + } + } + return f(doc); +} +exports.recursiveBstrToUint8Array = recursiveBstrToUint8Array; + +// Convert all strings in an object to from Uint8Array to bstr. +function recursiveUint8ArrayToBstr(doc) { + function f(x) { + if (typeof x === 'object' && x !== null && x instanceof Uint8Array) { + return uint8ArrayToBstr(x); + } else if (typeof x === 'object' && x !== null && Array.isArray(x)) { + let res = []; + for (let i = 0; i < x.length; i++) { + res.push(f(x[i])); + } + return res; + } else if (typeof x === 'object' && x !== null) { + let res = {}; + for (let k in x) { + //res[f(k)] = f(x[k]); + res[k] = f(x[k]); + } + return res; + } else { + return x; + } + } + + return f(doc); +} +exports.recursiveUint8ArrayToBstr = recursiveUint8ArrayToBstr; diff --git a/src-tools/lib/util/clone.js b/src-tools/lib/util/clone.js index ad84efdb..8339d8d8 100644 --- a/src-tools/lib/util/clone.js +++ b/src-tools/lib/util/clone.js @@ -4,3 +4,12 @@ function jsonDeepClone(v) { return JSON.parse(JSON.stringify(v)); } exports.jsonDeepClone = jsonDeepClone; + +function shallowCloneArray(arr) { + var res = []; + for (let i = 0; i < arr.length; i++) { + res[i] = arr[i]; + } + return res; +} +exports.shallowCloneArray = shallowCloneArray; diff --git a/src-tools/lib/util/string_util.js b/src-tools/lib/util/string_util.js index 3949befa..aeadb3af 100644 --- a/src-tools/lib/util/string_util.js +++ b/src-tools/lib/util/string_util.js @@ -12,3 +12,21 @@ function normalizeNewlines(x) { return x.replace('\r\n', '\n'); } exports.normalizeNewlines = normalizeNewlines; + +// Check if string is an "array index" in ECMAScript terms. +function stringIsArridx(x) { + if (typeof x !== 'string') { + throw new TypeError('invalid argument'); + } + if (/-?[0-9]+/.test(x)) { + let ival = Math.floor(Number(x)); + if (String(ival) !== x) { + return false; + } + if (ival >= 0 && ival <= 0xfffffffe) { + return true; + } + } + return false; +} +exports.stringIsArridx = stringIsArridx;