#!/usr/bin/env python2 # # Small helper for perftest runs. # import os import sys import json import time import optparse import subprocess class Alarm(Exception): pass def alarm_handler(signum, frame): raise Alarm def main(): parser = optparse.OptionParser() parser.add_option('--count', type='int', dest='count', default=3, help='Number of test runs') parser.add_option('--mode', dest='mode', default='min', help='Mode for choosing result: min, max, avg, all') parser.add_option('--sleep', type='float', dest='sleep', default=0.0, help='Fixed sleep value between runs') parser.add_option('--sleep-factor', type='float', dest='sleep_factor', default=0.0, help='Relative sleep value between runs, e.g. 2.0 means sleep twice as long as previous test run') parser.add_option('--rerun-limit', type='int', dest='rerun_limit', default=30, help='Run test only once if test run time exceeds this time limit') parser.add_option('--kill-timeout', type='int', dest='kill_timeout', default=None, help='Timeout for SIGKILLing the test') parser.add_option('--kill-wait', type='int', dest='kill_wait', default=3, help='Time to wait after SIGKILLing subprocess') parser.add_option('--verbose', action='store_true', dest='verbose', default=False, help='Verbose output') parser.add_option('--output', default=None, help='Output JSON file') (opts, args) = parser.parse_args() time_min = None time_max = None time_sum = 0.0 if opts.verbose: sys.stderr.write('Running:') sys.stderr.flush() doc = { 'count': opts.count, 'mode': opts.mode, 'sleep': opts.sleep, 'sleep_factor': opts.sleep_factor, 'rerun_limit': opts.rerun_limit, 'runs': [], 'args': args, 'time_list': [], 'time_min': None, 'time_max': None, 'time_avg': None } if opts.output is not None and os.path.exists(opts.output): os.unlink(opts.output) for i in xrange(opts.count): time.sleep(opts.sleep) cmd = [] cmd = cmd + args #print(repr(cmd)) if opts.kill_timeout is not None: # https://stackoverflow.com/questions/1191374/using-module-subprocess-with-timeout import signal signal.signal(signal.SIGALRM, alarm_handler) signal.alarm(opts.kill_timeout) killed = False retval = -1 time_start = time.time() try: p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() if opts.kill_timeout is not None: import signal signal.alarm(0) retval = p.wait() except Alarm: # XXX: Kill children? Not needed for duk executable. killed = True os.kill(p.pid, signal.SIGKILL) retval = p.wait() time.sleep(opts.kill_wait) time_end = time.time() run = { 'cmd': cmd, 'retval': retval, 'failed': False, 'killed': False, 'sigsegv': False, } doc['runs'].append(run) if killed: run['failed'] = True run['killed'] = True doc['failed'] = True doc['killed'] = True break elif retval == 139: run['failed'] = True run['sigsegv'] = True doc['failed'] = True doc['sigsegv'] = True break elif retval != 0: run['failed'] = True doc['failed'] = True break time_this = time_end - time_start #print(i, time_this) if time_min is None: time_min = time_this else: time_min = min(time_min, time_this) if time_max is None: time_max = time_this else: time_max = max(time_max, time_this) time_sum += time_this if opts.verbose: sys.stderr.write(' %f' % time_this) sys.stderr.flush() doc['time_list'].append(time_this) # Sleep time dependent on test time is useful for thermal throttling. time_sleep = opts.sleep_factor * time_this + opts.sleep run['time'] = time_this run['sleep_time'] = time_sleep time.sleep(time_sleep) # If run takes too long, there's no point in trying to get an accurate # estimate. if time_this >= opts.rerun_limit: doc['rerun_limit_reached'] = True break if opts.verbose: sys.stderr.write('\n') sys.stderr.flush() # /usr/bin/time has only two digits of resolution if doc.get('killed', False): print('kill') elif doc.get('sigsegv', False): print('segv') elif doc.get('failed', False): print('n/a') else: time_avg = time_sum / float(len(doc['time_list'])) doc['time_avg'] = time_avg doc['time_min'] = time_min doc['time_max'] = time_max if opts.mode == 'min': print('%.02f' % time_min) elif opts.mode == 'max': print('%.02f' % time_max) elif opts.mode == 'avg': print('%.02f' % time_avg) elif opts.mode == 'all': print('min=%.02f, max=%.02f, avg=%0.2f, count=%d: %r' % \ (time_min, time_max, time_avg, len(doc['time_list']), doc['time_list'])) else: print('invalid mode: %r' % opts.mode) if opts.output is not None: with open(opts.output, 'wb') as f: f.write(json.dumps(doc, indent=4) + '\n') sys.exit(0) if __name__ == '__main__': main()