Add src-tools support for basic perf test runs

src-tools/lib/command/run_tests.js

@@ -12,12 +12,18 @@ const defaultNumThreads = 8;
 const runTestsCommandSpec = createBareObject({
     description: 'Run Duktape testcases(s)',
     options: createBareObject({
+        'engine-bin': { type: 'path', default: void 0, description: 'Path to external ECMAScript engine executable, e.g. for comparison tests' },
         'source-directory': { type: 'path', default: void 0, description: 'Directory with raw input sources (defaulted based on script path)' },
         'num-threads': { short: 'j', type: 'number', default: estimateCoreCount() ?? defaultNumThreads, description: 'Number of threads to use for running tests' },
         'uglifyjs-bin': { type: 'path', default: void 0, description: 'Path to UglifyJS binary for minifying' },
         'test-log-file': { type: 'path', default: void 0, description: 'Path to ndjson test log file' },
         'test-hash-min': { type: 'number', default: void 0, description: 'Minimum value for testcase content hash' },
-        'test-hash-max': { type: 'number', default: void 0, description: 'Maximum value (inclusive) for testcase content hash' }
+        'test-hash-max': { type: 'number', default: void 0, description: 'Maximum value (inclusive) for testcase content hash' },
+        'run-count': { type: 'number', default: 1, description: 'Number of times to run each test, >1 useful for performance testing' },
+        'run-sleep': { type: 'boolean', default: false, value: true, description: 'Enable sleeping between runs, useful for performance testing' },
+        'sleep-multiplier': { type: 'number', default: 2.0, description: 'Sleep duration multiplier, multiply duration of run' },
+        'sleep-adder': { type: 'number', default: 1.0, description: 'Sleep duration adder, added to sleep duration multiplier result' },
+        'sleep-minimum': { type: 'number', default: 1.0, description: 'Sleep duration minimum' }
     })
 });
 exports.runTestsCommandSpec = runTestsCommandSpec;
@@ -49,14 +55,21 @@ async function runTestsCommand({ commandOpts, commandPositional }, autoDuktapeRo
                 require('fs').appendFileSync(testLogFile, JSON.stringify(doc) + '\n');
             }
         },
-        numThreads: commandOpts['num-threads'] ?? defaultNumThreads,
+        numThreads: +commandOpts['num-threads'] ?? defaultNumThreads,
+        //ignoreSkip: true,
         polyfillFilenames: [],
         uglifyJs2ExePath: commandOpts['uglifyjs-bin'],
         repoDirectory: autoDuktapeRoot,
         includeDirectory: pathJoin(autoDuktapeRoot, 'tests/ecmascript'),
         knownIssuesDirectory: pathJoin(autoDuktapeRoot, 'tests/knownissues'),
         testHashMin: commandOpts['test-hash-min'] ? +commandOpts['test-hash-min'] : void 0,
-        testHashMax: commandOpts['test-hash-max'] ? +commandOpts['test-hash-max'] : void 0
+        testHashMax: commandOpts['test-hash-max'] ? +commandOpts['test-hash-max'] : void 0,
+        runCount: +commandOpts['run-count'],
+        runSleep: commandOpts['run-sleep'],
+        sleepMultiplier: +commandOpts['sleep-multiplier'],
+        sleepAdder: +commandOpts['sleep-adder'],
+        sleepMinimum: +commandOpts['sleep-minimum'],
+        engineExePath: commandOpts['engine-bin']
     });
 
     var success = 0,

src-tools/lib/testing/duktape/analyze_result.js

@@ -32,7 +32,7 @@ function analyzeTestcaseResult({ execResult, testcaseMetadata, testcaseExpect, i
             if (known) {
                 logDebug('known issue:', known.filename);
                 res.success = true;
-                res.knownIssue = true;
+                res.knownIssue = known;
             } else {
                 res.success = false;
             }

src-tools/lib/testing/duktape/execute.js

@@ -3,8 +3,9 @@
 const { asyncExecStdoutUtf8 } = require('../../util/exec');
 const { getNowMillis } = require('../../util/time');
 const { assert } = require('../../util/assert');
-const { compileCTestcase } = require('./compile');
+const { sleep } = require('../../util/sleep');
 const { logDebug, logInfo } = require('../../util/logging');
+const { compileCTestcase } = require('./compile');
 
 async function executeEcmascriptTestcase({ dukCommandFilename, preparedFilename }) {
     var stdout;
@@ -28,6 +29,16 @@ async function executeEcmascriptTestcase({ dukCommandFilename, preparedFilename
     return { stdout, startTime, endTime, execError };
 }
 
+// Execute a C testcase (API tests).  This is much trickier than ECMAScript
+// testcases because we must also compile the code.  Two methods of execution
+// are supported:
+//
+//   1. Given configured Duktape sources, compile Duktape and testcase
+//      together for execution.
+//   2. Given configured Duktape sources and a precompiled libduktape*.so,
+//      compile testcase and link against the precompiled library.  This
+//      is much faster.
+
 async function executeCTestcase({ cExeFilename }) {
     var startTime;
     var endTime;
@@ -47,10 +58,34 @@ async function executeCTestcase({ cExeFilename }) {
     return { stdout, startTime, endTime };
 }
 
-async function executeTestcase({ testcaseType, dukCommandFilename, dukLibraryFilename, prepDirectory, preparedFilename, tempDirectory }) {
-    var count = 1;
+function computeStats(values) {
+    let average, minimum, maximum;
+    let variance, standardDeviation;
+
+    if (values.length > 0) {
+        let sum = 0;
+        for (let v of values) {
+            sum += v;
+        }
+        minimum = Math.min.apply(null, values);
+        maximum = Math.max.apply(null, values);
+        average = sum / values.length;
+
+        let varsum = 0;
+        for (let v of values) {
+            varsum += (v - average) ** 2;
+        }
+        variance = varsum / values.length;
+        standardDeviation = Math.sqrt(variance);
+    }
+
+    return { average, minimum, maximum, variance, standardDeviation };
+}
+
+async function executeTestcase({ testcaseType, testcaseName, dukCommandFilename, dukLibraryFilename, prepDirectory, preparedFilename, tempDirectory, runCount = 1, runSleep = false, sleepMultiplier = 2.0, sleepAdder = 1.0, sleepMinimum = 1.0 }) {
     var results = [];
     var durations = [];
+    var sleepTimes = [];
 
     // For C test cases, compile the test case once before the test execution loop.
     var cExeFilename;
@@ -65,7 +100,7 @@ async function executeTestcase({ testcaseType, dukCommandFilename, dukLibraryFil
     logDebug(cExeFilename);
 
     // Execute test one or more times.  Keep track of timing statistics.
-    for (let i = 0; i < count; i++) {
+    for (let i = 0; i < runCount; i++) {
         let execResult;
 
         if (testcaseType === 'ecmascript') {
@@ -79,18 +114,32 @@ async function executeTestcase({ testcaseType, dukCommandFilename, dukLibraryFil
             throw new TypeError('internal error');
         }
 
-        let duration = execResult.endTime - execResult.startTime;
-        let sleepTime = duration * 2.0;
+        let duration = (execResult.endTime - execResult.startTime) / 1000;
 
-        durations.push(execResult.endTime - execResult.startTime);
+        durations.push(duration);
         results.push(execResult);
+
+        if (runSleep) {
+            let sleepTime = Math.max(sleepMultiplier * duration + sleepAdder, sleepMinimum) * 1000;
+            logDebug('sleeping', sleepTime, 'milliseconds');
+            await sleep(sleepTime);
+            sleepTimes.push(sleepTime);
+        }
     }
 
-    return {
+    let { average, minimum, maximum, standardDeviation } = computeStats(durations);
+
+    let retval = {
         execResults: results,
-        execDurations: durations,
-        execResult: results[0]
+        execResult: results[0],
+
+        sleepTimes,
+
+        durations: { values: durations, average, minimum, maximum, standardDeviation },
+        duration: minimum // easy consumption
     };
+
+    return retval;
 }
 
 exports.executeTestcase = executeTestcase;

src-tools/lib/testing/duktape/run_test.js

@@ -16,7 +16,7 @@ const { compileDukCommandCached, compileDukLibraryCached } = require('./compile'
 const { executeTestcase } = require('./execute');
 const { analyzeTestcaseResult } = require('./analyze_result');
 
-async function runTestcase({ repoDirectory, testcaseFilename, knownIssues, ignoreExpect, ignoreSkip, polyfillFilenames, uglifyJsExePath, uglifyJs2ExePath, closureJarPath, dukCommandFilename, dukLibraryFilename, testRunState }) {
+async function runTestcase({ engineExePath, repoDirectory, testcaseFilename, knownIssues, ignoreExpect, ignoreSkip, polyfillFilenames, uglifyJsExePath, uglifyJs2ExePath, closureJarPath, dukCommandFilename, dukLibraryFilename, testRunState, runCount, runSleep, sleepMultiplier, sleepAdder, sleepMinimum, numThreads }) {
     var testcaseFilename = assert(testcaseFilename);
     var includeDirectory = pathJoin(repoDirectory, 'tests', 'ecmascript');
     var prepDirectory;
@@ -54,7 +54,7 @@ async function runTestcase({ repoDirectory, testcaseFilename, knownIssues, ignor
     var preparedFilename = pathJoin(tempDirectory, testcaseName);
     writeFileUtf8(preparedFilename, preparedSource);
 
-    // Initialize result object.
+    // Initialize result object.  Keys should all be snake_case.
     testResult = createBareObject({
         testcase_file: testcaseFilename,
         testcase_name: testcaseName,
@@ -74,24 +74,39 @@ async function runTestcase({ repoDirectory, testcaseFilename, knownIssues, ignor
     // Compile Duktape or libduktape if necessary.  Sources may need to be prepared
     // or compiled multiple times if test cases have forced Duktape options.  Compiled
     // binaries and libraries are cached.
-    if (!dukCommandFilename && testcaseType === 'ecmascript') {
+    if (engineExePath && testcaseType === 'ecmascript') {
+        dukCommandFilename = engineExePath;
+        prepDirectory = void 0; // not available, not needed for ECMAScript tests
+    } else if (!dukCommandFilename && testcaseType === 'ecmascript') {
         ({ dukCommandFilename, prepDirectory } = await compileDukCommandCached({ repoDirectory, tempDirectory, testcaseMetadata, testRunState }));
-    }
-    if (!dukLibraryFilename && testcaseType === 'c') {
+    } else if (!dukLibraryFilename && testcaseType === 'c') {
         ({ dukLibraryFilename, prepDirectory } = await compileDukLibraryCached({ repoDirectory, tempDirectory, testcaseMetadata, testRunState }));
     }
 
     // Execute testcase.
-    var runCount = 1;
-    var runSleep = true;
-    var { execResult, execDurations } = await executeTestcase({ testcaseType, testcaseName, dukCommandFilename, dukLibraryFilename, prepDirectory, preparedFilename, tempDirectory, runCount, runSleep });
+    var { execResult, durations, sleepTimes } = await executeTestcase({ testcaseType, testcaseName, dukCommandFilename, dukLibraryFilename, prepDirectory, preparedFilename, tempDirectory, runCount, runSleep, sleepMultiplier, sleepAdder, sleepMinimum });
 
     // Test result analysis.
     var { analysisResult } = analyzeTestcaseResult({ execResult, testcaseMetadata, testcaseExpect, ignoreExpect, knownIssues });
-    logDebug('testcase result:', JSON.stringify(analysisResult));
+    logDebug('testcase analysis result:', JSON.stringify(analysisResult));
+
     testResult.success = analysisResult.success;
-    testResult.knownIssue = analysisResult.knownIssue;
+    testResult.known_issue = analysisResult.knownIssue;
     testResult.duration = execResult.endTime - execResult.startTime;
+    testResult.durations = {
+        values: durations.values,
+        average: durations.average,
+        minimum: durations.minimum,
+        maximum: durations.maximum,
+        standard_deviation: durations.standardDeviation // Convert to snake_case for output.
+    };
+    testResult.sleep_times = sleepTimes;
+    testResult.run_count = runCount;
+    testResult.run_sleep = runSleep;
+    testResult.sleep_multiplier = sleepMultiplier;
+    testResult.sleep_adder = sleepAdder;
+    testResult.sleep_minimum = sleepMinimum;
+    testResult.num_threads = numThreads; // Not test related but good to know for e.g. performance tests.
 
     return testResult;
 }
@@ -205,7 +220,7 @@ function computeFileContentHashByte(fn) {
     return hash;
 }
 
-async function runMultipleTests({ repoDirectory, knownIssuesDirectory, filenames, progressCallback, logFileCallback, numThreads = 8, ignoreSkip, polyfillFilenames, uglifyJsExePath, uglifyJs2ExePath, closureJarPath, testHashMin, testHashMax }) {
+async function runMultipleTests({ engineExePath, repoDirectory, knownIssuesDirectory, filenames, progressCallback, logFileCallback, numThreads = 8, ignoreSkip, polyfillFilenames, uglifyJsExePath, uglifyJs2ExePath, closureJarPath, testHashMin, testHashMax, runCount = 1, runSleep = false, sleepMultiplier = 2.0, sleepAdder = 1.0, sleepMinimum = 1.0 }) {
     // For compile caching and other global run state.
     var testRunState = Object.create(null);
     testRunState.dukPrepareCount = 0;
@@ -226,6 +241,7 @@ async function runMultipleTests({ repoDirectory, knownIssuesDirectory, filenames
     var jobs = filenames.map((fn) => {
         return () => {
             return runTestcase({
+                engineExePath,
                 repoDirectory,
                 testcaseFilename: fn,
                 knownIssues,
@@ -234,7 +250,13 @@ async function runMultipleTests({ repoDirectory, knownIssuesDirectory, filenames
                 uglifyJsExePath,
                 uglifyJs2ExePath,
                 closureJarPath,
-                testRunState
+                testRunState,
+                runCount,
+                runSleep,
+                sleepMultiplier,
+                sleepAdder,
+                sleepMinimum,
+                numThreads
             });
         };
     });