From 209065e745b1b373c682208cedb035054b994c55 Mon Sep 17 00:00:00 2001 From: Sami Vaarala Date: Tue, 25 Nov 2014 15:08:17 +0200 Subject: [PATCH] Add ajduk example for compile time ext strings --- Makefile | 5 +- examples/alloc-hybrid/duk_alloc_hybrid.c | 7 + examples/cmdline/duk_cmdline_ajduk.c | 588 ++++++++++++++++++++--- util/duk_meta_to_strarray.py | 49 ++ 4 files changed, 575 insertions(+), 74 deletions(-) create mode 100644 util/duk_meta_to_strarray.py diff --git a/Makefile b/Makefile index 6492074d..9a371ff2 100644 --- a/Makefile +++ b/Makefile @@ -823,8 +823,9 @@ CCOPTS_AJDUK += -DDUK_OPT_HEAPPTR16 CCOPTS_AJDUK += '-DDUK_OPT_HEAPPTR_ENC16(p)=ajsheap_enc16(p)' CCOPTS_AJDUK += '-DDUK_OPT_HEAPPTR_DEC16(x)=ajsheap_dec16(x)' CCOPTS_AJDUK += -DDUK_OPT_EXTERNAL_STRINGS -CCOPTS_AJDUK += '-DDUK_OPT_EXTSTR_INTERN_CHECK(ptr,len)=ajsheap_ext_str_check((ptr),(len))' -CCOPTS_AJDUK += '-DDUK_OPT_DECLARE=extern uint8_t *ajsheap_ram; extern duk_uint16_t ajsheap_enc16(void *p); extern void *ajsheap_dec16(duk_uint16_t x); extern const void *ajsheap_ext_str_check(const void *ptr, duk_size_t len);' +CCOPTS_AJDUK += '-DDUK_OPT_EXTSTR_INTERN_CHECK(ptr,len)=ajsheap_extstr_check_1((ptr),(len))' +#CCOPTS_AJDUK += '-DDUK_OPT_EXTSTR_INTERN_CHECK(ptr,len)=ajsheap_extstr_check_2((ptr),(len))' +CCOPTS_AJDUK += '-DDUK_OPT_DECLARE=extern uint8_t *ajsheap_ram; extern duk_uint16_t ajsheap_enc16(void *p); extern void *ajsheap_dec16(duk_uint16_t x); extern const void *ajsheap_extstr_check_1(const void *ptr, duk_size_t len); extern const void *ajsheap_extstr_check_2(const void *ptr, duk_size_t len);' #CCOPTS_AJDUK += -DDUK_OPT_DEBUG -DDUK_OPT_DPRINT #CCOPTS_AJDUK += -DDUK_OPT_DEBUG -DDUK_OPT_DPRINT -DDUK_OPT_DDPRINT -DDUK_OPT_DDDPRINT diff --git a/examples/alloc-hybrid/duk_alloc_hybrid.c b/examples/alloc-hybrid/duk_alloc_hybrid.c index a50d5084..ca6afada 100644 --- a/examples/alloc-hybrid/duk_alloc_hybrid.c +++ b/examples/alloc-hybrid/duk_alloc_hybrid.c @@ -181,6 +181,9 @@ void *duk_alloc_hybrid(void *udata, duk_size_t size) { hdr->free = hdr->free->next; return new_ptr; } else { +#ifdef DUK_ALLOC_HYBRID_DEBUG + printf("alloc out of pool entries: %ld -> pool size %ld\n", (long) size, (long) hdr->size); +#endif break; } } @@ -283,4 +286,8 @@ void duk_free_hybrid(void *udata, void *ptr) { return; } } + +#ifdef DUK_ALLOC_HYBRID_DEBUG + printf("NEVER HERE\n"); +#endif } diff --git a/examples/cmdline/duk_cmdline_ajduk.c b/examples/cmdline/duk_cmdline_ajduk.c index 51ff0beb..a77e12c5 100644 --- a/examples/cmdline/duk_cmdline_ajduk.c +++ b/examples/cmdline/duk_cmdline_ajduk.c @@ -3,6 +3,32 @@ */ #ifdef DUK_CMDLINE_AJSHEAP + +#include +#include +#include +#include "ajs.h" +#include "ajs_heap.h" + +/* + * Helpers + */ + +static void safe_print_chars(const char *p, duk_size_t len) { + duk_size_t i; + + printf("\""); + for (i = 0; i < len; i++) { + unsigned char x = (unsigned char) p[i]; + if (x < 0x20 || x >= 0x7e || x == '"' || x == '\'' || x == '\\') { + printf("\\x%02x", (int) x); + } else { + printf("%c", (char) x); + } + } + printf("\""); +} + /* * Heap initialization when using AllJoyn.js pool allocator (without any * other AllJoyn.js integration). This serves as an example of how to @@ -19,12 +45,6 @@ * https://git.allseenalliance.org/cgit/core/alljoyn-js.git/tree/ajs.c */ -#include -#include -#include -#include "ajs.h" -#include "ajs_heap.h" - static const AJS_HeapConfig ajsheap_config[] = { { 8, 10, AJS_POOL_BORROW, 0 }, { 12, 10, AJS_POOL_BORROW, 0 }, @@ -50,10 +70,70 @@ static const AJS_HeapConfig ajsheap_config[] = { uint8_t *ajsheap_ram = NULL; -/* Example pointer compression functions. - * 'base' is chosen so that no non-NULL pointer results in a zero result - * which is reserved for NULL pointers. +void ajsheap_init(void) { + size_t heap_sz[1]; + uint8_t *heap_array[1]; + uint8_t num_pools, i; + AJ_Status ret; + + num_pools = (uint8_t) (sizeof(ajsheap_config) / sizeof(AJS_HeapConfig)); + heap_sz[0] = AJS_HeapRequired(ajsheap_config, /* heapConfig */ + num_pools, /* numPools */ + 0); /* heapNum */ + ajsheap_ram = (uint8_t *) malloc(heap_sz[0]); + if (!ajsheap_ram) { + fprintf(stderr, "Failed to allocate AJS heap\n"); + fflush(stderr); + exit(1); + } + heap_array[0] = ajsheap_ram; + + fprintf(stderr, "Allocated AJS heap of %ld bytes, pools:", (long) heap_sz[0]); + for (i = 0; i < num_pools; i++) { + fprintf(stderr, " (sz:%ld,num:%ld,brw:%ld,idx:%ld)", + (long) ajsheap_config[i].size, (long) ajsheap_config[i].entries, + (long) ajsheap_config[i].borrow, (long) ajsheap_config[i].heapIndex); + } + fprintf(stderr, "\n"); + fflush(stderr); + + ret = AJS_HeapInit((void **) heap_array, /* heap */ + (size_t *) heap_sz, /* heapSz */ + ajsheap_config, /* heapConfig */ + num_pools, /* numPools */ + 1); /* numHeaps */ + fprintf(stderr, "AJS_HeapInit() -> %ld\n", (long) ret); + fflush(stderr); +} + +/* AjsHeap.dump(), allows Ecmascript code to dump heap status at suitable + * points. */ +duk_ret_t ajsheap_dump_binding(duk_context *ctx) { + AJS_HeapDump(); + fflush(stdout); + return 0; +} + +void ajsheap_dump(void) { + AJS_HeapDump(); + fflush(stdout); +} + +void ajsheap_register(duk_context *ctx) { + duk_push_object(ctx); + duk_push_c_function(ctx, ajsheap_dump_binding, 0); + duk_put_prop_string(ctx, -2, "dump"); + duk_put_global_string(ctx, "AjsHeap"); +} + +/* + * Example pointer compression functions. + * + * 'base' is chosen so that no non-NULL pointer results in a zero result + * which is reserved for NULL pointers. + */ + duk_uint16_t ajsheap_enc16(void *p) { duk_uint32_t ret; char *base = (char *) ajsheap_ram - 4; @@ -88,31 +168,37 @@ void *ajsheap_dec16(duk_uint16_t x) { return ret; } -/* Simplified example of an external strings strategy where incoming strings - * are writted sequentially into a fixed flash memory area which is memory - * mapped. The example first scans if the string is already in the flash - * (which may happen if the same string is interned multiple times), then - * adds it to flash if there is space. +/* + * Simplified example of an external strings strategy where incoming strings + * are written sequentially into a fixed, memory mapped flash area. * - * This example is too slow to be used in a real world application: there - * should be e.g. a hash table to quickly check for strings that are already - * present in the string data (similarly to how string interning works in - * Duktape itself). + * The example first scans if the string is already in the flash (which may + * happen if the same string is interned multiple times), then adds it to + * flash if there is space. + * + * This example is too slow to be used in a real world application: there + * should be e.g. a hash table to quickly check for strings that are already + * present in the string data (similarly to how string interning works in + * Duktape itself). */ + static uint8_t ajsheap_strdata[65536]; static size_t ajsheap_strdata_used = 0; -const void *ajsheap_ext_str_check(const void *ptr, duk_size_t len) { +const void *ajsheap_extstr_check_1(const void *ptr, duk_size_t len) { uint8_t *p, *p_end; uint8_t initial; uint8_t *ret; size_t left; + (void) safe_print_chars; /* potentially unused */ + if (len <= 3) { /* It's not worth it to make very small strings external, as * they would take the same space anyway. Also avoids zero * length degenerate case. */ + return NULL; } /* @@ -132,7 +218,11 @@ const void *ajsheap_ext_str_check(const void *ptr, duk_size_t len) { p[len] == 0) { ret = p; #if 0 - printf("ajsheap_ext_str_check: ptr=%p, len=%ld -> existing %p (used=%ld)\n", (void *) ptr, (long) len, (void *) ret, (long) ajsheap_strdata_used); + printf("ajsheap_extstr_check_1: ptr=%p, len=%ld ", + (void *) ptr, (long) len); + safe_print_chars((const char *) ptr, len); + printf(" -> existing %p (used=%ld)\n", + (void *) ret, (long) ajsheap_strdata_used); #endif return ret; } @@ -145,7 +235,9 @@ const void *ajsheap_ext_str_check(const void *ptr, duk_size_t len) { if (ajsheap_strdata_used + len + 1 > sizeof(ajsheap_strdata)) { #if 0 - printf("ajsheap_ext_str_check: ptr=%p, len=%ld -> no space (used=%ld)\n", (void *) ptr, (long) len, (long) ajsheap_strdata_used); + printf("ajsheap_extstr_check_1: ptr=%p, len=%ld ", (void *) ptr, (long) len); + safe_print_chars((const char *) ptr, len); + printf(" -> no space (used=%ld)\n", (long) ajsheap_strdata_used); #endif return NULL; } @@ -161,67 +253,419 @@ const void *ajsheap_ext_str_check(const void *ptr, duk_size_t len) { ajsheap_strdata_used += len + 1; #if 0 - printf("ajsheap_ext_str_check: ptr=%p, len=%ld -> %p (used=%ld)\n", (void *) ptr, (long) len, (void *) ret, (long) ajsheap_strdata_used); + printf("ajsheap_extstr_check_1: ptr=%p, len=%ld -> ", (void *) ptr, (long) len); + safe_print_chars((const char *) ptr, len); + printf(" -> %p (used=%ld)\n", (void *) ret, (long) ajsheap_strdata_used); #endif return (const void *) ret; } -void ajsheap_init(void) { - size_t heap_sz[1]; - uint8_t *heap_array[1]; - uint8_t num_pools, i; - AJ_Status ret; +/* + * Simplified example of an external strings strategy where a set of strings + * is gathered during application compile time and baked into the application + * binary. + * + * Duktape built-in strings are available from duk_build_meta.json, see + * util/duk_meta_to_strarray.py. There may also be a lot of application + * specific strings, e.g. those used by application specific APIs. These + * must be gathered through some other means. + */ - num_pools = (uint8_t) (sizeof(ajsheap_config) / sizeof(AJS_HeapConfig)); - heap_sz[0] = AJS_HeapRequired(ajsheap_config, /* heapConfig */ - num_pools, /* numPools */ - 0); /* heapNum */ - ajsheap_ram = (uint8_t *) malloc(heap_sz[0]); - if (!ajsheap_ram) { - fprintf(stderr, "Failed to allocate AJS heap\n"); - fflush(stderr); - exit(1); - } - heap_array[0] = ajsheap_ram; +static const char *strdata_duk_builtin_strings[] = { + /* + * These strings are from util/duk_meta_to_strarray.py + */ - fprintf(stderr, "Allocated AJS heap of %ld bytes, pools:", (long) heap_sz[0]); - for (i = 0; i < num_pools; i++) { - fprintf(stderr, " (sz:%ld,num:%ld,brw:%ld,idx:%ld)", - (long) ajsheap_config[i].size, (long) ajsheap_config[i].entries, - (long) ajsheap_config[i].borrow, (long) ajsheap_config[i].heapIndex); - } - fprintf(stderr, "\n"); - fflush(stderr); + "Logger", + "Thread", + "Pointer", + "Buffer", + "DecEnv", + "ObjEnv", + "", + "global", + "Arguments", + "JSON", + "Math", + "Error", + "RegExp", + "Date", + "Number", + "Boolean", + "String", + "Array", + "Function", + "Object", + "Null", + "Undefined", + "{_func:true}", + "{\x22" "_func\x22" ":true}", + "{\x22" "_ninf\x22" ":true}", + "{\x22" "_inf\x22" ":true}", + "{\x22" "_nan\x22" ":true}", + "{\x22" "_undef\x22" ":true}", + "toLogString", + "clog", + "l", + "n", + "fatal", + "error", + "warn", + "debug", + "trace", + "raw", + "fmt", + "current", + "resume", + "compact", + "jc", + "jx", + "base64", + "hex", + "dec", + "enc", + "fin", + "gc", + "act", + "info", + "version", + "env", + "modLoaded", + "modSearch", + "errThrow", + "errCreate", + "compile", + "\xff" "Regbase", + "\xff" "Thread", + "\xff" "Handler", + "\xff" "Finalizer", + "\xff" "Callee", + "\xff" "Map", + "\xff" "Args", + "\xff" "This", + "\xff" "Pc2line", + "\xff" "Source", + "\xff" "Varenv", + "\xff" "Lexenv", + "\xff" "Varmap", + "\xff" "Formals", + "\xff" "Bytecode", + "\xff" "Next", + "\xff" "Target", + "\xff" "Value", + "pointer", + "buffer", + "\xff" "Tracedata", + "lineNumber", + "fileName", + "pc", + "stack", + "ThrowTypeError", + "Duktape", + "id", + "require", + "__proto__", + "setPrototypeOf", + "ownKeys", + "enumerate", + "deleteProperty", + "has", + "Proxy", + "callee", + "Invalid Date", + "[...]", + "\x0a" "\x09", + " ", + ",", + "-0", + "+0", + "0", + "-Infinity", + "+Infinity", + "Infinity", + "object", + "string", + "number", + "boolean", + "undefined", + "stringify", + "tan", + "sqrt", + "sin", + "round", + "random", + "pow", + "min", + "max", + "log", + "floor", + "exp", + "cos", + "ceil", + "atan2", + "atan", + "asin", + "acos", + "abs", + "SQRT2", + "SQRT1_2", + "PI", + "LOG10E", + "LOG2E", + "LN2", + "LN10", + "E", + "message", + "name", + "input", + "index", + "(?:)", + "lastIndex", + "multiline", + "ignoreCase", + "source", + "test", + "exec", + "toGMTString", + "setYear", + "getYear", + "toJSON", + "toISOString", + "toUTCString", + "setUTCFullYear", + "setFullYear", + "setUTCMonth", + "setMonth", + "setUTCDate", + "setDate", + "setUTCHours", + "setHours", + "setUTCMinutes", + "setMinutes", + "setUTCSeconds", + "setSeconds", + "setUTCMilliseconds", + "setMilliseconds", + "setTime", + "getTimezoneOffset", + "getUTCMilliseconds", + "getMilliseconds", + "getUTCSeconds", + "getSeconds", + "getUTCMinutes", + "getMinutes", + "getUTCHours", + "getHours", + "getUTCDay", + "getDay", + "getUTCDate", + "getDate", + "getUTCMonth", + "getMonth", + "getUTCFullYear", + "getFullYear", + "getTime", + "toLocaleTimeString", + "toLocaleDateString", + "toTimeString", + "toDateString", + "now", + "UTC", + "parse", + "toPrecision", + "toExponential", + "toFixed", + "POSITIVE_INFINITY", + "NEGATIVE_INFINITY", + "NaN", + "MIN_VALUE", + "MAX_VALUE", + "substr", + "trim", + "toLocaleUpperCase", + "toUpperCase", + "toLocaleLowerCase", + "toLowerCase", + "substring", + "split", + "search", + "replace", + "match", + "localeCompare", + "charCodeAt", + "charAt", + "fromCharCode", + "reduceRight", + "reduce", + "filter", + "map", + "forEach", + "some", + "every", + "lastIndexOf", + "indexOf", + "unshift", + "splice", + "sort", + "slice", + "shift", + "reverse", + "push", + "pop", + "join", + "concat", + "isArray", + "arguments", + "caller", + "bind", + "call", + "apply", + "propertyIsEnumerable", + "isPrototypeOf", + "hasOwnProperty", + "valueOf", + "toLocaleString", + "toString", + "constructor", + "set", + "get", + "enumerable", + "configurable", + "writable", + "value", + "keys", + "isExtensible", + "isFrozen", + "isSealed", + "preventExtensions", + "freeze", + "seal", + "defineProperties", + "defineProperty", + "create", + "getOwnPropertyNames", + "getOwnPropertyDescriptor", + "getPrototypeOf", + "prototype", + "length", + "alert", + "print", + "unescape", + "escape", + "encodeURIComponent", + "encodeURI", + "decodeURIComponent", + "decodeURI", + "isFinite", + "isNaN", + "parseFloat", + "parseInt", + "eval", + "URIError", + "TypeError", + "SyntaxError", + "ReferenceError", + "RangeError", + "EvalError", + "break", + "case", + "catch", + "continue", + "debugger", + "default", + "delete", + "do", + "else", + "finally", + "for", + "function", + "if", + "in", + "instanceof", + "new", + "return", + "switch", + "this", + "throw", + "try", + "typeof", + "var", + "void", + "while", + "with", + "class", + "const", + "enum", + "export", + "extends", + "import", + "super", + "null", + "true", + "false", + "implements", + "interface", + "let", + "package", + "private", + "protected", + "public", + "static", + "yield", - ret = AJS_HeapInit((void **) heap_array, /* heap */ - (size_t *) heap_sz, /* heapSz */ - ajsheap_config, /* heapConfig */ - num_pools, /* numPools */ - 1); /* numHeaps */ - fprintf(stderr, "AJS_HeapInit() -> %ld\n", (long) ret); - fflush(stderr); -} + /* + * These strings are manually added, and would be gathered in some + * application specific manner. + */ -/* AjsHeap.dump(), allows Ecmascript code to dump heap status at suitable - * points. - */ -duk_ret_t ajsheap_dump_binding(duk_context *ctx) { - AJS_HeapDump(); - fflush(stdout); - return 0; -} + "foo", + "bar", + "quux", + "enableFrob", + "disableFrob" + /* ... */ +}; -void ajsheap_dump(void) { - AJS_HeapDump(); - fflush(stdout); -} +const void *ajsheap_extstr_check_2(const void *ptr, duk_size_t len) { + int i, n; -void ajsheap_register(duk_context *ctx) { - duk_push_object(ctx); - duk_push_c_function(ctx, ajsheap_dump_binding, 0); - duk_put_prop_string(ctx, -2, "dump"); - duk_put_global_string(ctx, "AjsHeap"); + (void) safe_print_chars; /* potentially unused */ + + /* Linear scan. An actual implementation would need some acceleration + * structure, e.g. select a sublist based on first character. + * + * NOTE: input string (behind 'ptr' with 'len' bytes) DOES NOT have a + * trailing NUL character. Any strings returned from this function + * MUST have a trailing NUL character. + */ + + n = (int) (sizeof(strdata_duk_builtin_strings) / sizeof(const char *)); + for (i = 0; i < n; i++) { + const char *str; + + str = strdata_duk_builtin_strings[i]; + if (strlen(str) == len && memcmp(ptr, (const void *) str, len) == 0) { +#if 0 + printf("ajsheap_extstr_check_2: ptr=%p, len=%ld ", + (void *) ptr, (long) len); + safe_print_chars((const char *) ptr, len); + printf(" -> constant string index %ld\n", (long) i); +#endif + return (void *) strdata_duk_builtin_strings[i]; + } + } + +#if 0 + printf("ajsheap_extstr_check_2: ptr=%p, len=%ld ", + (void *) ptr, (long) len); + safe_print_chars((const char *) ptr, len); + printf(" -> not found\n"); +#endif + return NULL; } -#else + +#else /* DUK_CMDLINE_AJSHEAP */ + int ajs_dummy = 0; /* to avoid empty source file */ + #endif /* DUK_CMDLINE_AJSHEAP */ diff --git a/util/duk_meta_to_strarray.py b/util/duk_meta_to_strarray.py new file mode 100644 index 00000000..aaa3e4a8 --- /dev/null +++ b/util/duk_meta_to_strarray.py @@ -0,0 +1,49 @@ +#!/usr/bin/python +# +# Create an array of C strings with Duktape built-in strings. +# Useful when using external strings. +# + +import os +import sys +import json + +def to_c_string(x): + res = '"' + term = False + for i, c in enumerate(x): + if term: + term = False + res += '" "' + + o = ord(c) + if o < 0x20 or o > 0x7e or c in '\'"\\': + # Terminate C string so that escape doesn't become + # ambiguous + res += '\\x%02x' % o + term = True + else: + res += c + res += '"' + return res + +def main(): + f = open(sys.argv[1], 'rb') + d = f.read() + f.close() + meta = json.loads(d) + + print('const char *duk_builtin_strings[] = {') + + strlist = meta['builtin_strings_base64'] + for i in xrange(len(strlist)): + s = strlist[i] + if i == len(strlist) - 1: + print(' %s' % to_c_string(s.decode('base64'))) + else: + print(' %s,' % to_c_string(s.decode('base64'))) + + print('};') + +if __name__ == '__main__': + main()