Merge pull request #1720 from svaarala/benchmark-text-compress-json

Use lz-string to compress benchmarks.html raw data

Makefile

@@ -291,6 +291,7 @@ cleanall: clean
 	@rm -rf FlameGraph
 	@rm -rf dtrace4linux
 	@rm -rf flow
+	@rm -rf lz-string
 	@rm -rf 3883a2e9063b0a5f2705bdac3263577a03913c94.zip
 	@rm -rf es5-tests.zip
 	@rm -f v1.3.5.tar.gz
@@ -943,6 +944,9 @@ dtrace4linux:
 flow:
 	# https://github.com/facebook/flow
 	$(GIT) clone --depth 1 https://github.com/facebook/flow.git
+lz-string:
+	# https://github.com/pieroxy/lz-string.git
+	$(GIT) clone --depth 1 https://github.com/pieroxy/lz-string.git
 
 # Duktape binary releases are in a separate repo.
 duktape-releases:
@@ -991,7 +995,7 @@ dist-iso:	dist-src
 .PHONY: tidy-site
 tidy-site:
 	for i in website/*/*.html; do echo "*** Checking $$i"; tidy -q -e -xml $$i; done
-site: duktape-releases dukweb.js jquery-1.11.0.js
+site: duktape-releases dukweb.js jquery-1.11.0.js lz-string
 	rm -rf site
 	mkdir site
 	-cd duktape-releases/; git pull --rebase  # get binaries up-to-date, but allow errors for offline use

util/build_benchmarks_page.py

@@ -6,6 +6,10 @@ import re
 import json
 
 def main():
+    # Adapt manually.
+
+    duk = '/usr/local/bin/duk'
+    lzstring = '/home/duktape/duktape/lz-string/libs/lz-string.js'
     duktape_repo = '/home/duktape/duktape'
     duktape_testrunner_repo = '/home/duktape/duktape-testrunner'
     duktape_testclient_config = '/home/duktape/duktape-testclient-config.yaml'
@@ -26,10 +30,11 @@ def main():
     print('%d hashes found' % len(hashes))
 
     # Get any release tags matching the hashes for annotations.
+
     re_release_tag = re.compile('^v\d+\.\d+\.\d+$')
     annotations = []
     for x,h in enumerate(hashes):
-        os.system('git tag -l --points-at %s > /tmp/tmp-taglog.txt' % h)
+        os.system('cd %s && git tag -l --points-at %s > /tmp/tmp-taglog.txt' % (duktape_repo, h))
         with open('/tmp/tmp-taglog.txt', 'rb') as f:
             for line in f:
                 line = line.strip()
@@ -39,14 +44,15 @@ def main():
                 annotations.append({ 'x': x, 'tag': line })
     print(json.dumps(annotations, indent=4))
 
+    # Get test data for hashed, and pack it into a JSON object embedded
+    # into the page.
+
     req = { 'repo_full': 'svaarala/duktape', 'sha_list': hashes }
     with open('/tmp/tmp-request.json', 'wb') as f:
         f.write(json.dumps(req))
 
     os.system('cd %s && cd client-simple-node && nodejs client.js --request-uri /query-commit-simple --config %s --request-file /tmp/tmp-request.json --output-file /tmp/tmp-result.json' % (duktape_testrunner_repo, duktape_testclient_config))
 
-    with open(benchmarks_template, 'rb') as f:
-        page = f.read()
     with open('/tmp/tmp-result.json', 'rb') as f:
         data = json.loads(f.read())
 
@@ -54,11 +60,34 @@ def main():
         'commit_simples': data,
         'annotations': annotations
     }
-    doc_formatted = json.dumps(doc, indent=4)
+    with open('/tmp/tmp-graphdata.json', 'wb') as f:
+        f.write(json.dumps(doc))
+
+    # There's a lot of JSON data so use http://pieroxy.net/blog/pages/lz-string/index.html
+    # to compress it.  'duk' executable can be used to compress data.
+
+    with open('/tmp/tmp-script.js', 'wb') as f:
+        f.write('''
+var input = new TextDecoder().decode(readFile('/tmp/tmp-graphdata.json'));
+var compressed = LZString.compressToBase64(input);
+writeFile('/tmp/tmp-graphdata-compressed.txt', compressed);
+''')
+    os.system('%s %s /tmp/tmp-script.js' % (duk, lzstring))
+    with open('/tmp/tmp-graphdata-compressed.txt') as f:
+        graphdata = f.read()
+
+    # Embed the compressed data into the benchmarks.html template.
+
+    with open(benchmarks_template, 'rb') as f:
+        page = f.read()
+
     page = page.replace('<!-- @DATA@ -->', \
-                        'var rawGraphData = \n' + doc_formatted + '\n')
+                        'var rawGraphDataCompressed = "' + graphdata + '";')
+
     with open('/tmp/benchmarks.html', 'wb') as f:
         f.write(page)
 
+    # Done!
+
 if __name__ == '__main__':
     main()

website/benchmarks.html

@@ -4,6 +4,7 @@
 <meta charset="UTF-8">
 <title>Duktape benchmarks</title>
 <script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
+<script src="lz-string.js"></script>
 <style type="text/css">
 html, body {
 	background: #ffffff;
@@ -45,6 +46,8 @@ h1 {
 <script>
 <!-- Injected by build script. -->
 <!-- @DATA@ -->
+
+var rawGraphData = JSON.parse(LZString.decompressFromBase64(rawGraphDataCompressed));
 </script>
 
 <!--