Browse Source

Remove matrix_compile.py test

The matrix compile test relied on DUK_OPT_xxx and needs to be reworked
if it is added back.
pull/1255/head
Sami Vaarala 8 years ago
parent
commit
3948239834
  1. 14
      Makefile
  2. 561
      util/matrix_compile.py

14
Makefile

@ -525,20 +525,6 @@ apitest: apiprep
@echo "### apitest"
"$(NODE)" runtests/runtests.js $(RUNTESTSOPTS) --num-threads 1 --log-file=/tmp/duk-api-test.log tests/api/
# Matrix tests.
.PHONY: matrix10
matrix10: dist
cd dist; $(PYTHON) ../util/matrix_compile.py --count=10
.PHONY: matrix100
matrix100: dist
cd dist; $(PYTHON) ../util/matrix_compile.py --count=100
.PHONY: matrix1000
matrix1000: dist
cd dist; $(PYTHON) ../util/matrix_compile.py --count=1000
.PHONY: matrix10000
matrix10000: dist
cd dist; $(PYTHON) ../util/matrix_compile.py --count=10000
# Dukweb.js test.
.PHONY: dukwebtest
dukwebtest: dukweb.js jquery-1.11.0.js

561
util/matrix_compile.py

@ -1,561 +0,0 @@
#!/usr/bin/env python2
#
# Compile test for a lot of option combinations
#
# XXX: rewrite as nodejs and parallelize (large indices handling need bigint)
import os
import sys
import time
import datetime
import json
import random
import optparse
import subprocess
import StringIO # no need for cStringIO
#
# Test matrix helper: given a specification of combinations, count the
# total number of combinations and allow a specific combination to be
# fetched using an index. This avoids creating the combinations explicitly
# and also allows random sampling of the combination space (which can be
# very large).
#
# Select one: Select([ 1, 2, 3 ]) -> [ 1 ], [ 2 ], [ 3 ]
class Select:
val = None
def __init__(self, val):
self.val = val
# Combine: Combine([ 1, 2 ], 'foo') -> [ 1 'foo' ], [ 2 'foo' ]
class Combine:
val = None
def __init__(self, val):
self.val = val
# Subset: Subset([ 'foo', 'bar' ]) -> Combine([ [ '', 'foo' ], [ '', 'bar' ] ])
# -> [ '' '' ], [ 'foo' '' ], [ '' 'bar' ], [ 'foo' 'bar' ]
class Subset:
val = None
def __init__(self, val):
self.val = val
# Sequence: Sequence([ 'foo', 'bar', 'quux' ]) -> [ 'foo', 'bar', 'quux' ]
# Plain list is also interpreted as a Sequence.
class Sequence:
val = None
def __init__(self, val):
self.val = val
# Prepare a combination lookup structure.
def prepcomb(val):
if isinstance(val, (str, unicode)):
return { 'size': 1, 'value': val, 'type': 'terminal' }
if isinstance(val, Sequence):
return { 'size': 1, 'value': val.val, 'type': 'sequence' }
if isinstance(val, list):
# interpret as Sequence
return { 'size': 1, 'value': val, 'type': 'sequence' }
if isinstance(val, Select):
nodes = []
size = 0
for i in val.val:
node = prepcomb(i)
nodes.append(node)
size += node['size']
return { 'size': size, 'value': nodes, 'type': 'select' }
if isinstance(val, Combine):
nodes = []
size = 1
for i in val.val:
node = prepcomb(i)
nodes.append(node)
size *= node['size']
return { 'size': size, 'value': nodes, 'type': 'combine' }
if isinstance(val, Subset):
nodes = []
size = 1
for i in val.val:
node = prepcomb(i)
nodes.append(node)
size *= (node['size'] + 1) # value or not present
return { 'size': size, 'value': nodes, 'type': 'subset' }
raise Exception('invalid argument')
# Return number of combinations for input lists.
def countcombinations(prepped):
return prepped['size']
# Return a combination for index, for index in [0,countcombinations(lists)[.
# This allows random selection of combinations using a PRNG.
def getcomb(prepped, index):
if prepped['type'] == 'terminal':
return [ prepped['value'] ], index
if prepped['type'] == 'sequence':
return prepped['value'], index
if prepped['type'] == 'select':
idx = index % prepped['size']
index = index / prepped['size']
for i in prepped['value']:
if idx >= i['size']:
idx -= i['size']
continue
ret, ign_index = getcomb(i, idx)
return ret, index
raise Exception('should not be here')
if prepped['type'] == 'combine':
ret = []
for i in prepped['value']:
idx = index % i['size']
index = index / i['size']
tmp, tmp_index = getcomb(i, idx)
ret.append(tmp)
return ret, index
if prepped['type'] == 'subset':
ret = []
for i in prepped['value']:
idx = index % (i['size'] + 1)
index = index / (i['size'] + 1)
if idx == 0:
# no value
ret.append('')
else:
tmp, tmp_index = getcomb(i, idx - 1)
ret.append(tmp)
return ret, index
raise Exception('invalid prepped value')
def flatten(v):
if isinstance(v, (str, unicode)):
return [ v ]
if isinstance(v, list):
ret = []
for i in v:
ret += flatten(i)
return ret
raise Exception('invalid value: %s' % repr(v))
def getcombination(val, index):
res, res_index = getcomb(val, index)
if res_index != 0:
sys.stderr.write('WARNING: index not consumed entirely, invalid index? (input index %d, output index %d)\n' % (index, res_index))
return res
# Generate all combinations.
def getcombinations(val):
res = []
for i in xrange(countcombinations(val)):
res.append(getcombination(val, i))
return res
#
# Test matrix
#
def create_matrix(fn_duk):
# A lot of compiler versions are used, must install at least:
#
# gcc-4.6
# gcc-4.7
# gcc-4.8
# gcc-4.6-multilib
# g++-4.6-multilib
# gcc-4.7-multilib
# g++-4.7-multilib
# gcc-4.8-multilib
# g++-4.8-multilib
# gcc-multilib
# g++-multilib
# llvm-gcc-4.6
# llvm-gcc-4.7
# llvm-3.4
# clang
#
# The set of compilers tested is distribution specific and not ery
# stable, so you may need to edit the compilers manually.
gcc_cmd_dialect_options = Select([
# Some dialects and architectures are only available for newer g++ versions
Combine([
# -m32 with older llvm causes self test failure (double union)
Select([ 'llvm-gcc' ]),
Select([ '-m64' ]),
Select([
'',
'-std=c89',
'-std=c99',
[ '-std=c99', '-pedantic' ]
])
]),
Combine([
Select([ 'gcc', 'gcc-4.6' ]),
Select([ '-m64', '-m32' ]),
Select([
'',
'-std=c89',
'-std=c99',
[ '-std=c99', '-pedantic' ]
])
]),
Combine([
Select([ 'gcc-4.7', 'gcc-4.8' ]),
Select([ '-m64', '-m32', '-mx32' ]),
Select([
'',
'-std=c89',
'-std=c99',
[ '-std=c99', '-pedantic' ]
])
]),
])
gxx_cmd_dialect_options = Select([
# Some dialects and architectures are only available for newer g++ versions
Combine([
Select([ 'llvm-g++' ]),
Select([ '-m64' ]),
Select([
'',
'-std=c++98',
[ '-std=c++11', '-pedantic' ]
])
]),
Combine([
Select([ 'g++', 'g++-4.6' ]),
Select([ '-m64', '-m32' ]),
Select([
'',
'-std=c++98',
])
]),
Combine([
Select([ 'g++-4.7', 'g++-4.8' ]),
Select([ '-m64', '-m32', '-mx32' ]),
Select([
'',
'-std=c++98',
[ '-std=c++11', '-pedantic' ]
])
]),
Combine([
Select([ 'g++', 'g++-4.8' ]),
Select([ '-m64', '-m32', '-mx32' ]),
Select([
'-std=c++1y',
'-std=gnu++1y'
])
]),
])
gcc_gxx_debug_options = Select([
'',
[ '-g', '-ggdb' ]
])
gcc_gxx_warning_options = Select([
'',
#'-Wall',
[ '-Wall', '-Wextra' ]
#XXX: -Wfloat-equal
# [ '-Wall', '-Wextra', '-Werror' ]
])
gcc_gxx_optimization_options = Select([
'-O0',
'-O1',
'-O2',
# -O3 and -O4 produces spurious warnings on gcc 4.8.1, e.g. "error: assuming signed overflow does not occur when assuming that (X - c) > X is always false [-Werror=strict-overflow]"
# Not sure what causes these, but perhaps GCC converts signed comparisons into subtractions and then runs into: https://gcc.gnu.org/wiki/FAQ#signed_overflow
[ '-O3', '-fno-strict-overflow' ],
#'-O3'
[ '-O4', '-fno-strict-overflow' ],
#'-O4'
'-Os'
])
clang_cmd_dialect_options = Select([
Combine([
'clang',
Select([ '-m64', '-m32' ]),
Select([
'',
'-std=c89',
'-std=c99',
[ '-std=c99', '-pedantic' ]
])
])
])
clang_debug_options = Select([
'',
[ '-g', '-ggdb' ]
])
clang_warning_options = Select([
'',
[ '-Wall', '-Wextra' ],
[ '-Wall', '-Wextra', '-Wcast-align' ]
#XXX: -Wfloat-equal
#[ '-Wall', '-Wextra', '-Werror' ]
])
clang_optimization_options = Select([
'-O0',
'-O1',
'-O2',
'-O3',
#'-O4',
'-Os'
])
# Feature options in suitable chunks that can be subsetted arbitrarily.
duktape_options = Subset([
Select([ '-DDUK_OPT_NO_REFERENCE_COUNTING',
'-DDUK_OPT_NO_MARK_AND_SWEEP',
'-DDUK_OPT_GC_TORTURE' ]),
'-DDUK_OPT_SHUFFLE_TORTURE',
'-DDUK_OPT_NO_VOLUNTARY_GC',
'-DDUK_OPT_NO_PACKED_TVAL',
Select([ '', '-DDUK_OPT_FORCE_ALIGN=4', '-DDUK_OPT_FORCE_ALIGN=8' ]),
'-DDUK_OPT_NO_TRACEBACKS',
'-DDUK_OPT_NO_VERBOSE_ERRORS',
'-DDUK_OPT_PARANOID_ERRORS',
'-DDUK_OPT_NO_MS_RESIZE_STRINGTABLE',
'-DDUK_OPT_NO_STRICT_DECL',
'-DDUK_OPT_NO_REGEXP_SUPPORT',
'-DDUK_OPT_NO_ES6_REGEXP_SYNTAX',
'-DDUK_OPT_NO_SOURCE_NONBMP',
'-DDUK_OPT_STRICT_UTF8_SOURCE',
'-DDUK_OPT_NO_SECTION_B',
'-DDUK_OPT_NO_JX',
'-DDUK_OPT_NO_JC',
'-DDUK_OPT_NO_NONSTD_ACCESSOR_KEY_ARGUMENT',
'-DDUK_OPT_NO_NONSTD_FUNC_STMT',
'-DDUK_OPT_NONSTD_FUNC_CALLER_PROPERTY',
'-DDUK_OPT_NONSTD_FUNC_SOURCE_PROPERTY',
'-DDUK_OPT_NO_NONSTD_ARRAY_SPLICE_DELCOUNT',
'-DDUK_OPT_NO_NONSTD_ARRAY_CONCAT_TRAILER',
'-DDUK_OPT_NO_NONSTD_ARRAY_MAP_TRAILER',
'-DDUK_OPT_NO_NONSTD_JSON_ESC_U2028_U2029',
'-DDUK_OPT_NO_BYTECODE_DUMP_SUPPORT',
'-DDUK_OPT_NO_ES6_OBJECT_PROTO_PROPERTY',
'-DDUK_OPT_NO_ES6_OBJECT_SETPROTOTYPEOF',
'-DDUK_OPT_NO_ES6_PROXY',
'-DDUK_OPT_NO_ZERO_BUFFER_DATA',
'-DDUK_OPT_LIGHTFUNC_BUILTINS',
'-DDUK_OPT_ASSERTIONS',
[ '-DDUK_OPT_DEBUG', '-DDUK_OPT_DEBUG_WRITE(level,file,line,func,msg)=do {fprintf(stderr, "%ld %s %ld %s %s\\n", (long) (level), (file), (long) (line), (func), (msg));} while(0)', '-DDUK_OPT_DPRINT', '-DDUK_OPT_DDDPRINT' ],
'-DDUK_OPT_SELF_TESTS',
[ '-DDUK_OPT_STRTAB_CHAIN', '-DDUK_OPT_STRTAB_CHAIN_SIZE=64' ],
# DUK_OPT_DEBUGGER_SUPPORT depends on having pc2line and
# interrupt counter, so avoid invalid combinations.
Select([
Subset([ '-DDUK_OPT_NO_PC2LINE', '-DDUK_OPT_INTERRUPT_COUNTER' ]),
[ '-DDUK_OPT_DEBUGGER_SUPPORT', '-DDUK_OPT_INTERRUPT_COUNTER' ]
]),
'-DDUK_OPT_DEBUGGER_FWD_LOGGING',
'-DDUK_OPT_DEBUGGER_DUMPHEAP',
'-DDUK_OPT_DEBUGGER_INSPECT',
'-DDUK_OPT_NO_DEBUGGER_THROW_NOTIFY',
'-DDUK_OPT_DEBUGGER_PAUSE_UNCAUGHT',
'-DDUK_OPT_JSON_STRINGIFY_FASTPATH'
# XXX: 16-bit options
])
# XXX: DUK_USE_LEXER_SLIDING_WINDOW
# The final command is compiler specific because e.g. include path
# and link option syntax could (in principle) differ between compilers.
gcc_cmd_matrix = Combine([
gcc_cmd_dialect_options,
gcc_gxx_debug_options,
gcc_gxx_warning_options,
gcc_gxx_optimization_options,
duktape_options,
[ '-DDUK_CMDLINE_PRINTALERT_SUPPORT', '-Isrc', '-Iextras/print-alert', 'src/duktape.c', 'extras/print-alert/duk_print_alert.c', 'examples/cmdline/duk_cmdline.c', '-o', fn_duk, '-lm' ]
])
gxx_cmd_matrix = Combine([
gxx_cmd_dialect_options,
gcc_gxx_debug_options,
gcc_gxx_warning_options,
gcc_gxx_optimization_options,
duktape_options,
[ '-DDUK_CMDLINE_PRINTALERT_SUPPORT', '-Isrc', '-Iextras/print-alert', 'src/duktape.c', 'extras/print-alert/duk_print_alert.c', 'examples/cmdline/duk_cmdline.c', '-o', fn_duk, '-lm' ]
])
clang_cmd_matrix = Combine([
clang_cmd_dialect_options,
clang_debug_options,
clang_warning_options,
clang_optimization_options,
duktape_options,
[ '-DDUK_CMDLINE_PRINTALERT_SUPPORT', '-Isrc', '-Iextras/print-alert', 'src/duktape.c', 'extras/print-alert/duk_print_alert.c', 'examples/cmdline/duk_cmdline.c', '-o', fn_duk, '-lm' ]
])
matrix = Select([ gcc_cmd_matrix, gxx_cmd_matrix, clang_cmd_matrix ])
return matrix
#
# Main
#
def check_unlink(filename):
if os.path.exists(filename):
os.unlink(filename)
def main():
# XXX: add option for testcase(s) to run?
# XXX: add valgrind support, restrict to -m64 compilation?
# XXX: proper tempfile usage and cleanup
time_str = str(long(time.time() * 1000.0))
parser = optparse.OptionParser()
parser.add_option('--count', dest='count', default='1000')
parser.add_option('--seed', dest='seed', default='default_seed_' + time_str)
parser.add_option('--out-results-json', dest='out_results_json', default='/tmp/matrix_results%s.json' % time_str)
parser.add_option('--out-failed', dest='out_failed', default='/tmp/matrix_failed%s.txt' % time_str)
parser.add_option('--verbose', dest='verbose', default=False, action='store_true')
(opts, args) = parser.parse_args()
fn_testjs = '/tmp/test%s.js' % time_str
fn_duk = '/tmp/duk%s' % time_str
# Avoid any optional features (like JSON or RegExps) in the test.
# Don't make the test very long, as it executes very slowly when
# DUK_OPT_DDDPRINT and DUK_OPT_ASSERTIONS are enabled.
f = open(fn_testjs, 'wb')
f.write('''
// Fibonacci using try-catch, exercises setjmp/longjmp a lot
function fibthrow(n) {
var f1, f2;
if (n === 0) { throw 0; }
if (n === 1) { throw 1; }
try { fibthrow(n-1); } catch (e) { f1 = e; }
try { fibthrow(n-2); } catch (e) { f2 = e; }
throw f1 + f2;
}
print('Hello world');
print(1 + 2);
print(Math.PI); // tests constant endianness
print(JSON.stringify({ foo: 'bar' }));
try { fibthrow(9); } catch (e) { print(e); }
''')
f.close()
expect = 'Hello world\n3\n3.141592653589793\n{"foo":"bar"}\n34\n'
print('Using seed: ' + repr(opts.seed))
random.seed(opts.seed)
matrix = create_matrix(fn_duk)
prepped = prepcomb(matrix)
# print(json.dumps(prepped, indent=4))
# print(json.dumps(getcombinations(prepped), indent=4))
numcombinations = countcombinations(prepped)
# The number of combinations is large so do (pseudo) random
# testing over the matrix. Ideally we'd avoid re-testing the
# same combination twice, but with the matrix space in billions
# this doesn't need to be checked.
res = []
failed = []
for i in xrange(long(opts.count)):
fail = False
idx = random.randrange(0, numcombinations)
cmd = getcombination(prepped, idx)
#cmd = getcombination(prepped, idx)
compile_command = flatten(cmd)
compile_command = [ elem for elem in compile_command if elem != '' ] # remove empty strings
print('%d/%d (combination %d, count %d)' % (i + 1, long(opts.count), idx, numcombinations))
#print('%d/%d (combination %d, count %d) %s' % (i + 1, long(opts.count), idx, numcombinations, repr(compile_command)))
if opts.verbose:
print(' '.join(compile_command))
check_unlink(fn_duk)
#print(repr(compile_command))
compile_p = subprocess.Popen(compile_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
compile_stdout, compile_stderr = compile_p.communicate()
compile_exitcode = compile_p.returncode
if compile_exitcode != 0:
fail = True
else:
if not os.path.exists(fn_duk):
print('*** WARNING: compile success but no %s ***' % fn_duk)
run_command = [ fn_duk, fn_testjs ]
if fail:
run_stdout = None
run_stderr = None
run_exitcode = 1
else:
run_p = subprocess.Popen(run_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
run_stdout, run_stderr = run_p.communicate()
run_exitcode = run_p.returncode
if run_exitcode != 0:
fail = True
if run_stdout != expect:
fail = True
if fail:
print('------------------------------------------------------------------------------')
print('*** FAILED: %s' % repr(compile_command))
print(' '.join(compile_command))
failed.append(' '.join(compile_command))
print('COMPILE STDOUT:')
print(compile_stdout)
print('COMPILE STDERR:')
print(compile_stderr)
print('RUN STDOUT:')
print(run_stdout)
print('RUN STDERR:')
print(run_stderr)
print('------------------------------------------------------------------------------')
res.append({
'compile_command': compile_command,
'compile_stdout': compile_stdout,
'compile_stderr': compile_stderr,
'compile_exitcode': compile_exitcode,
'run_command': run_command,
'run_stdout': run_stdout,
# Don't include debug output, it's huge with DUK_OPT_DDDPRINT
#'run_stderr': run_stderr,
'run_exitcode': run_exitcode,
'run_expect': expect,
'success': not fail
})
sys.stdout.flush()
sys.stderr.flush()
f = open(opts.out_results_json, 'wb')
f.write(json.dumps(res, indent=4, sort_keys=True))
f.close()
f = open(opts.out_failed, 'wb')
f.write('\n'.join(failed) + '\n')
f.close()
check_unlink(fn_duk)
check_unlink(fn_testjs)
# XXX: summary of success/failure/warnings (= stderr got anything)
if __name__ == '__main__':
main()
Loading…
Cancel
Save