diff --git a/Makefile b/Makefile index 392c7adb..2d35c07d 100644 --- a/Makefile +++ b/Makefile @@ -525,20 +525,6 @@ apitest: apiprep @echo "### apitest" "$(NODE)" runtests/runtests.js $(RUNTESTSOPTS) --num-threads 1 --log-file=/tmp/duk-api-test.log tests/api/ -# Matrix tests. -.PHONY: matrix10 -matrix10: dist - cd dist; $(PYTHON) ../util/matrix_compile.py --count=10 -.PHONY: matrix100 -matrix100: dist - cd dist; $(PYTHON) ../util/matrix_compile.py --count=100 -.PHONY: matrix1000 -matrix1000: dist - cd dist; $(PYTHON) ../util/matrix_compile.py --count=1000 -.PHONY: matrix10000 -matrix10000: dist - cd dist; $(PYTHON) ../util/matrix_compile.py --count=10000 - # Dukweb.js test. .PHONY: dukwebtest dukwebtest: dukweb.js jquery-1.11.0.js diff --git a/util/matrix_compile.py b/util/matrix_compile.py deleted file mode 100644 index 2bf440bd..00000000 --- a/util/matrix_compile.py +++ /dev/null @@ -1,561 +0,0 @@ -#!/usr/bin/env python2 -# -# Compile test for a lot of option combinations -# - -# XXX: rewrite as nodejs and parallelize (large indices handling need bigint) - -import os -import sys -import time -import datetime -import json -import random -import optparse -import subprocess -import StringIO # no need for cStringIO - -# -# Test matrix helper: given a specification of combinations, count the -# total number of combinations and allow a specific combination to be -# fetched using an index. This avoids creating the combinations explicitly -# and also allows random sampling of the combination space (which can be -# very large). -# - -# Select one: Select([ 1, 2, 3 ]) -> [ 1 ], [ 2 ], [ 3 ] -class Select: - val = None - - def __init__(self, val): - self.val = val - -# Combine: Combine([ 1, 2 ], 'foo') -> [ 1 'foo' ], [ 2 'foo' ] -class Combine: - val = None - - def __init__(self, val): - self.val = val - -# Subset: Subset([ 'foo', 'bar' ]) -> Combine([ [ '', 'foo' ], [ '', 'bar' ] ]) -# -> [ '' '' ], [ 'foo' '' ], [ '' 'bar' ], [ 'foo' 'bar' ] -class Subset: - val = None - - def __init__(self, val): - self.val = val - -# Sequence: Sequence([ 'foo', 'bar', 'quux' ]) -> [ 'foo', 'bar', 'quux' ] -# Plain list is also interpreted as a Sequence. -class Sequence: - val = None - - def __init__(self, val): - self.val = val - -# Prepare a combination lookup structure. -def prepcomb(val): - if isinstance(val, (str, unicode)): - return { 'size': 1, 'value': val, 'type': 'terminal' } - if isinstance(val, Sequence): - return { 'size': 1, 'value': val.val, 'type': 'sequence' } - if isinstance(val, list): - # interpret as Sequence - return { 'size': 1, 'value': val, 'type': 'sequence' } - if isinstance(val, Select): - nodes = [] - size = 0 - for i in val.val: - node = prepcomb(i) - nodes.append(node) - size += node['size'] - return { 'size': size, 'value': nodes, 'type': 'select' } - if isinstance(val, Combine): - nodes = [] - size = 1 - for i in val.val: - node = prepcomb(i) - nodes.append(node) - size *= node['size'] - return { 'size': size, 'value': nodes, 'type': 'combine' } - if isinstance(val, Subset): - nodes = [] - size = 1 - for i in val.val: - node = prepcomb(i) - nodes.append(node) - size *= (node['size'] + 1) # value or not present - return { 'size': size, 'value': nodes, 'type': 'subset' } - raise Exception('invalid argument') - -# Return number of combinations for input lists. -def countcombinations(prepped): - return prepped['size'] - -# Return a combination for index, for index in [0,countcombinations(lists)[. -# This allows random selection of combinations using a PRNG. -def getcomb(prepped, index): - if prepped['type'] == 'terminal': - return [ prepped['value'] ], index - if prepped['type'] == 'sequence': - return prepped['value'], index - if prepped['type'] == 'select': - idx = index % prepped['size'] - index = index / prepped['size'] - - for i in prepped['value']: - if idx >= i['size']: - idx -= i['size'] - continue - ret, ign_index = getcomb(i, idx) - return ret, index - - raise Exception('should not be here') - if prepped['type'] == 'combine': - ret = [] - for i in prepped['value']: - idx = index % i['size'] - index = index / i['size'] - tmp, tmp_index = getcomb(i, idx) - ret.append(tmp) - return ret, index - if prepped['type'] == 'subset': - ret = [] - for i in prepped['value']: - idx = index % (i['size'] + 1) - index = index / (i['size'] + 1) - if idx == 0: - # no value - ret.append('') - else: - tmp, tmp_index = getcomb(i, idx - 1) - ret.append(tmp) - return ret, index - raise Exception('invalid prepped value') - -def flatten(v): - if isinstance(v, (str, unicode)): - return [ v ] - if isinstance(v, list): - ret = [] - for i in v: - ret += flatten(i) - return ret - raise Exception('invalid value: %s' % repr(v)) - - -def getcombination(val, index): - res, res_index = getcomb(val, index) - if res_index != 0: - sys.stderr.write('WARNING: index not consumed entirely, invalid index? (input index %d, output index %d)\n' % (index, res_index)) - - return res - -# Generate all combinations. -def getcombinations(val): - res = [] - for i in xrange(countcombinations(val)): - res.append(getcombination(val, i)) - return res - -# -# Test matrix -# - -def create_matrix(fn_duk): - # A lot of compiler versions are used, must install at least: - # - # gcc-4.6 - # gcc-4.7 - # gcc-4.8 - # gcc-4.6-multilib - # g++-4.6-multilib - # gcc-4.7-multilib - # g++-4.7-multilib - # gcc-4.8-multilib - # g++-4.8-multilib - # gcc-multilib - # g++-multilib - # llvm-gcc-4.6 - # llvm-gcc-4.7 - # llvm-3.4 - # clang - # - # The set of compilers tested is distribution specific and not ery - # stable, so you may need to edit the compilers manually. - - gcc_cmd_dialect_options = Select([ - # Some dialects and architectures are only available for newer g++ versions - Combine([ - # -m32 with older llvm causes self test failure (double union) - Select([ 'llvm-gcc' ]), - Select([ '-m64' ]), - Select([ - '', - '-std=c89', - '-std=c99', - [ '-std=c99', '-pedantic' ] - ]) - ]), - Combine([ - Select([ 'gcc', 'gcc-4.6' ]), - Select([ '-m64', '-m32' ]), - Select([ - '', - '-std=c89', - '-std=c99', - [ '-std=c99', '-pedantic' ] - ]) - ]), - Combine([ - Select([ 'gcc-4.7', 'gcc-4.8' ]), - Select([ '-m64', '-m32', '-mx32' ]), - Select([ - '', - '-std=c89', - '-std=c99', - [ '-std=c99', '-pedantic' ] - ]) - ]), - ]) - gxx_cmd_dialect_options = Select([ - # Some dialects and architectures are only available for newer g++ versions - Combine([ - Select([ 'llvm-g++' ]), - Select([ '-m64' ]), - Select([ - '', - '-std=c++98', - [ '-std=c++11', '-pedantic' ] - ]) - ]), - Combine([ - Select([ 'g++', 'g++-4.6' ]), - Select([ '-m64', '-m32' ]), - Select([ - '', - '-std=c++98', - ]) - ]), - Combine([ - Select([ 'g++-4.7', 'g++-4.8' ]), - Select([ '-m64', '-m32', '-mx32' ]), - Select([ - '', - '-std=c++98', - [ '-std=c++11', '-pedantic' ] - ]) - ]), - Combine([ - Select([ 'g++', 'g++-4.8' ]), - Select([ '-m64', '-m32', '-mx32' ]), - Select([ - '-std=c++1y', - '-std=gnu++1y' - ]) - ]), - ]) - gcc_gxx_debug_options = Select([ - '', - [ '-g', '-ggdb' ] - ]) - gcc_gxx_warning_options = Select([ - '', - #'-Wall', - [ '-Wall', '-Wextra' ] - #XXX: -Wfloat-equal - # [ '-Wall', '-Wextra', '-Werror' ] - ]) - gcc_gxx_optimization_options = Select([ - '-O0', - '-O1', - '-O2', - - # -O3 and -O4 produces spurious warnings on gcc 4.8.1, e.g. "error: assuming signed overflow does not occur when assuming that (X - c) > X is always false [-Werror=strict-overflow]" - # Not sure what causes these, but perhaps GCC converts signed comparisons into subtractions and then runs into: https://gcc.gnu.org/wiki/FAQ#signed_overflow - - [ '-O3', '-fno-strict-overflow' ], - #'-O3' - - [ '-O4', '-fno-strict-overflow' ], - #'-O4' - - '-Os' - ]) - clang_cmd_dialect_options = Select([ - Combine([ - 'clang', - Select([ '-m64', '-m32' ]), - Select([ - '', - '-std=c89', - '-std=c99', - [ '-std=c99', '-pedantic' ] - ]) - ]) - ]) - clang_debug_options = Select([ - '', - [ '-g', '-ggdb' ] - ]) - clang_warning_options = Select([ - '', - [ '-Wall', '-Wextra' ], - [ '-Wall', '-Wextra', '-Wcast-align' ] - #XXX: -Wfloat-equal - #[ '-Wall', '-Wextra', '-Werror' ] - ]) - clang_optimization_options = Select([ - '-O0', - '-O1', - '-O2', - '-O3', - #'-O4', - '-Os' - ]) - - # Feature options in suitable chunks that can be subsetted arbitrarily. - - duktape_options = Subset([ - Select([ '-DDUK_OPT_NO_REFERENCE_COUNTING', - '-DDUK_OPT_NO_MARK_AND_SWEEP', - '-DDUK_OPT_GC_TORTURE' ]), - '-DDUK_OPT_SHUFFLE_TORTURE', - '-DDUK_OPT_NO_VOLUNTARY_GC', - '-DDUK_OPT_NO_PACKED_TVAL', - Select([ '', '-DDUK_OPT_FORCE_ALIGN=4', '-DDUK_OPT_FORCE_ALIGN=8' ]), - '-DDUK_OPT_NO_TRACEBACKS', - '-DDUK_OPT_NO_VERBOSE_ERRORS', - '-DDUK_OPT_PARANOID_ERRORS', - '-DDUK_OPT_NO_MS_RESIZE_STRINGTABLE', - '-DDUK_OPT_NO_STRICT_DECL', - '-DDUK_OPT_NO_REGEXP_SUPPORT', - '-DDUK_OPT_NO_ES6_REGEXP_SYNTAX', - '-DDUK_OPT_NO_SOURCE_NONBMP', - '-DDUK_OPT_STRICT_UTF8_SOURCE', - '-DDUK_OPT_NO_SECTION_B', - '-DDUK_OPT_NO_JX', - '-DDUK_OPT_NO_JC', - '-DDUK_OPT_NO_NONSTD_ACCESSOR_KEY_ARGUMENT', - '-DDUK_OPT_NO_NONSTD_FUNC_STMT', - '-DDUK_OPT_NONSTD_FUNC_CALLER_PROPERTY', - '-DDUK_OPT_NONSTD_FUNC_SOURCE_PROPERTY', - '-DDUK_OPT_NO_NONSTD_ARRAY_SPLICE_DELCOUNT', - '-DDUK_OPT_NO_NONSTD_ARRAY_CONCAT_TRAILER', - '-DDUK_OPT_NO_NONSTD_ARRAY_MAP_TRAILER', - '-DDUK_OPT_NO_NONSTD_JSON_ESC_U2028_U2029', - '-DDUK_OPT_NO_BYTECODE_DUMP_SUPPORT', - '-DDUK_OPT_NO_ES6_OBJECT_PROTO_PROPERTY', - '-DDUK_OPT_NO_ES6_OBJECT_SETPROTOTYPEOF', - '-DDUK_OPT_NO_ES6_PROXY', - '-DDUK_OPT_NO_ZERO_BUFFER_DATA', - '-DDUK_OPT_LIGHTFUNC_BUILTINS', - '-DDUK_OPT_ASSERTIONS', - [ '-DDUK_OPT_DEBUG', '-DDUK_OPT_DEBUG_WRITE(level,file,line,func,msg)=do {fprintf(stderr, "%ld %s %ld %s %s\\n", (long) (level), (file), (long) (line), (func), (msg));} while(0)', '-DDUK_OPT_DPRINT', '-DDUK_OPT_DDDPRINT' ], - '-DDUK_OPT_SELF_TESTS', - [ '-DDUK_OPT_STRTAB_CHAIN', '-DDUK_OPT_STRTAB_CHAIN_SIZE=64' ], - - # DUK_OPT_DEBUGGER_SUPPORT depends on having pc2line and - # interrupt counter, so avoid invalid combinations. - Select([ - Subset([ '-DDUK_OPT_NO_PC2LINE', '-DDUK_OPT_INTERRUPT_COUNTER' ]), - [ '-DDUK_OPT_DEBUGGER_SUPPORT', '-DDUK_OPT_INTERRUPT_COUNTER' ] - ]), - '-DDUK_OPT_DEBUGGER_FWD_LOGGING', - '-DDUK_OPT_DEBUGGER_DUMPHEAP', - '-DDUK_OPT_DEBUGGER_INSPECT', - '-DDUK_OPT_NO_DEBUGGER_THROW_NOTIFY', - '-DDUK_OPT_DEBUGGER_PAUSE_UNCAUGHT', - '-DDUK_OPT_JSON_STRINGIFY_FASTPATH' - - # XXX: 16-bit options - ]) - - # XXX: DUK_USE_LEXER_SLIDING_WINDOW - - # The final command is compiler specific because e.g. include path - # and link option syntax could (in principle) differ between compilers. - - gcc_cmd_matrix = Combine([ - gcc_cmd_dialect_options, - gcc_gxx_debug_options, - gcc_gxx_warning_options, - gcc_gxx_optimization_options, - duktape_options, - [ '-DDUK_CMDLINE_PRINTALERT_SUPPORT', '-Isrc', '-Iextras/print-alert', 'src/duktape.c', 'extras/print-alert/duk_print_alert.c', 'examples/cmdline/duk_cmdline.c', '-o', fn_duk, '-lm' ] - ]) - - gxx_cmd_matrix = Combine([ - gxx_cmd_dialect_options, - gcc_gxx_debug_options, - gcc_gxx_warning_options, - gcc_gxx_optimization_options, - duktape_options, - [ '-DDUK_CMDLINE_PRINTALERT_SUPPORT', '-Isrc', '-Iextras/print-alert', 'src/duktape.c', 'extras/print-alert/duk_print_alert.c', 'examples/cmdline/duk_cmdline.c', '-o', fn_duk, '-lm' ] - ]) - - clang_cmd_matrix = Combine([ - clang_cmd_dialect_options, - clang_debug_options, - clang_warning_options, - clang_optimization_options, - duktape_options, - [ '-DDUK_CMDLINE_PRINTALERT_SUPPORT', '-Isrc', '-Iextras/print-alert', 'src/duktape.c', 'extras/print-alert/duk_print_alert.c', 'examples/cmdline/duk_cmdline.c', '-o', fn_duk, '-lm' ] - ]) - - matrix = Select([ gcc_cmd_matrix, gxx_cmd_matrix, clang_cmd_matrix ]) - return matrix - -# -# Main -# - -def check_unlink(filename): - if os.path.exists(filename): - os.unlink(filename) - -def main(): - # XXX: add option for testcase(s) to run? - # XXX: add valgrind support, restrict to -m64 compilation? - # XXX: proper tempfile usage and cleanup - - time_str = str(long(time.time() * 1000.0)) - - parser = optparse.OptionParser() - parser.add_option('--count', dest='count', default='1000') - parser.add_option('--seed', dest='seed', default='default_seed_' + time_str) - parser.add_option('--out-results-json', dest='out_results_json', default='/tmp/matrix_results%s.json' % time_str) - parser.add_option('--out-failed', dest='out_failed', default='/tmp/matrix_failed%s.txt' % time_str) - parser.add_option('--verbose', dest='verbose', default=False, action='store_true') - (opts, args) = parser.parse_args() - - fn_testjs = '/tmp/test%s.js' % time_str - fn_duk = '/tmp/duk%s' % time_str - - # Avoid any optional features (like JSON or RegExps) in the test. - # Don't make the test very long, as it executes very slowly when - # DUK_OPT_DDDPRINT and DUK_OPT_ASSERTIONS are enabled. - - f = open(fn_testjs, 'wb') - f.write(''' -// Fibonacci using try-catch, exercises setjmp/longjmp a lot -function fibthrow(n) { - var f1, f2; - if (n === 0) { throw 0; } - if (n === 1) { throw 1; } - try { fibthrow(n-1); } catch (e) { f1 = e; } - try { fibthrow(n-2); } catch (e) { f2 = e; } - throw f1 + f2; -} -print('Hello world'); -print(1 + 2); -print(Math.PI); // tests constant endianness -print(JSON.stringify({ foo: 'bar' })); -try { fibthrow(9); } catch (e) { print(e); } -''') - f.close() - expect = 'Hello world\n3\n3.141592653589793\n{"foo":"bar"}\n34\n' - - print('Using seed: ' + repr(opts.seed)) - random.seed(opts.seed) - matrix = create_matrix(fn_duk) - prepped = prepcomb(matrix) -# print(json.dumps(prepped, indent=4)) -# print(json.dumps(getcombinations(prepped), indent=4)) - numcombinations = countcombinations(prepped) - - # The number of combinations is large so do (pseudo) random - # testing over the matrix. Ideally we'd avoid re-testing the - # same combination twice, but with the matrix space in billions - # this doesn't need to be checked. - - res = [] - failed = [] - for i in xrange(long(opts.count)): - fail = False - idx = random.randrange(0, numcombinations) - cmd = getcombination(prepped, idx) - #cmd = getcombination(prepped, idx) - compile_command = flatten(cmd) - compile_command = [ elem for elem in compile_command if elem != '' ] # remove empty strings - - print('%d/%d (combination %d, count %d)' % (i + 1, long(opts.count), idx, numcombinations)) - #print('%d/%d (combination %d, count %d) %s' % (i + 1, long(opts.count), idx, numcombinations, repr(compile_command))) - if opts.verbose: - print(' '.join(compile_command)) - - check_unlink(fn_duk) - #print(repr(compile_command)) - compile_p = subprocess.Popen(compile_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - compile_stdout, compile_stderr = compile_p.communicate() - compile_exitcode = compile_p.returncode - - if compile_exitcode != 0: - fail = True - else: - if not os.path.exists(fn_duk): - print('*** WARNING: compile success but no %s ***' % fn_duk) - - run_command = [ fn_duk, fn_testjs ] - if fail: - run_stdout = None - run_stderr = None - run_exitcode = 1 - else: - run_p = subprocess.Popen(run_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - run_stdout, run_stderr = run_p.communicate() - run_exitcode = run_p.returncode - - if run_exitcode != 0: - fail = True - if run_stdout != expect: - fail = True - - if fail: - print('------------------------------------------------------------------------------') - print('*** FAILED: %s' % repr(compile_command)) - print(' '.join(compile_command)) - failed.append(' '.join(compile_command)) - - print('COMPILE STDOUT:') - print(compile_stdout) - print('COMPILE STDERR:') - print(compile_stderr) - print('RUN STDOUT:') - print(run_stdout) - print('RUN STDERR:') - print(run_stderr) - print('------------------------------------------------------------------------------') - - res.append({ - 'compile_command': compile_command, - 'compile_stdout': compile_stdout, - 'compile_stderr': compile_stderr, - 'compile_exitcode': compile_exitcode, - 'run_command': run_command, - 'run_stdout': run_stdout, - # Don't include debug output, it's huge with DUK_OPT_DDDPRINT - #'run_stderr': run_stderr, - 'run_exitcode': run_exitcode, - 'run_expect': expect, - 'success': not fail - }) - - sys.stdout.flush() - sys.stderr.flush() - - f = open(opts.out_results_json, 'wb') - f.write(json.dumps(res, indent=4, sort_keys=True)) - f.close() - - f = open(opts.out_failed, 'wb') - f.write('\n'.join(failed) + '\n') - f.close() - - check_unlink(fn_duk) - check_unlink(fn_testjs) - - # XXX: summary of success/failure/warnings (= stderr got anything) - -if __name__ == '__main__': - main()