From 75680a9557490b219d080a9c1d9431e3ed506b21 Mon Sep 17 00:00:00 2001 From: Sami Vaarala Date: Thu, 13 Nov 2014 05:39:01 +0200 Subject: [PATCH 1/5] First draft of SPDX license file creation --- util/create_spdx_license.py | 240 ++++++++++++++++++++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 util/create_spdx_license.py diff --git a/util/create_spdx_license.py b/util/create_spdx_license.py new file mode 100644 index 00000000..3e635c55 --- /dev/null +++ b/util/create_spdx_license.py @@ -0,0 +1,240 @@ +#!/usr/bin/python +# +# Helper to create an SPDX license file (http://spdx.org) +# +# This must be executed when the dist/ directory is otherwise complete, +# except for the SPDX license, so that the file lists and such contained +# in the SPDX license will be correct. +# +# The utility outputs RDF/XML to specified file: +# +# $ python create_spdx_license.py /tmp/license.xml +# +# Then, validate with SPDXViewer and SPDXTools: +# +# $ java -jar SPDXViewer.jar /tmp/license.xml +# $ java -jar java -jar spdx-tools-1.2.5-jar-with-dependencies.jar RdfToHtml /tmp/license.xml /tmp/license.html +# +# Finally, copy to dist: +# +# $ cp /tmp/license.xml dist/ +# +# The algorithm to compute a "verification code", implemented in this file, +# can be verified as follows: +# +# # build dist tar.xz, copy to /tmp/duktape-N.N.N.tar.xz +# $ cd /tmp +# $ tar xvfJ duktape-N.N.N.tar.xz +# $ rm duktape-N.N.N/license.xml # remove file excluded from verification code +# $ java -jar spdx-tools-1.2.5-jar-with-dependencies.jar GenerateVerificationCode /tmp/duktape-N.N.N/ +# +# Compare the resulting verification code manually with the one in license.xml. +# +# Resources: +# +# - http://wiki.spdx.org/view/Technical_Team/Best_Practices +# + +import os +import sys +import re +import datetime +import sha +import rdflib +from rdflib import URIRef, BNode, Literal, Namespace + +RDF = Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#') +RDFS = Namespace('http://www.w3.org/2000/01/rdf-schema#') +XSD = Namespace('http://www.w3.org/2001/XMLSchema#') +SPDX = Namespace('http://spdx.org/rdf/terms#') +DOAP = Namespace('http://usefulinc.com/ns/doap#') +DUKTAPE = Namespace('http://duktape.org/rdf/terms#') + +def checksumFile(g, filename): + f = open(filename, 'rb') + d = f.read() + f.close() + shasum = sha.sha(d).digest().encode('hex').lower() + + csum_node = BNode() + g.add((csum_node, RDF.type, SPDX.Checksum)) + g.add((csum_node, SPDX.algorithm, SPDX.checksumAlgorithm_sha1)) + g.add((csum_node, SPDX.checksumValue, Literal(shasum))) + + return csum_node + +def computePackageVerification(g, dirname, excluded): + # SPDX 1.2 Section 4.7 + # The SPDXTools command "GenerateVerificationCode" can be used to + # check the verification codes created. Note that you must manually + # remove "license.xml" from the unpacked dist directory before + # computing the verification code. + + verify_node = BNode() + + hashes = [] + for dirpath, dirnames, filenames in os.walk(dirname): + for fn in filenames: + full_fn = os.path.join(dirpath, fn) + f = open(full_fn, 'rb') + d = f.read() + f.close() + + if full_fn in excluded: + #print('excluded in verification: ' + full_fn) + continue + #print('included in verification: ' + full_fn) + + file_sha1 = sha.sha(d).digest().encode('hex').lower() + hashes.append(file_sha1) + + #print(repr(hashes)) + hashes.sort() + #print(repr(hashes)) + verify_code = sha.sha(''.join(hashes)).digest().encode('hex').lower() + + for fn in excluded: + g.add((verify_node, SPDX.packageVerificationCodeExcludedFile, Literal(fn))) + g.add((verify_node, SPDX.packageVerificationCodeValue, Literal(verify_code))) + + return verify_node + +def fileType(filename): + ign, ext = os.path.splitext(filename) + if ext in [ '.c', '.h', '.js' ]: + return SPDX.fileType_source + else: + return SPDX.fileType_other + +def getDuktapeVersion(): + f = open('./src/duktape.h') + re_ver = re.compile(r'^#define\s+DUK_VERSION\s+(\d+)L$') + for line in f: + line = line.strip() + m = re_ver.match(line) + if m is None: + continue + ver = int(m.group(1)) + return '%d.%d.%d' % ((ver / 10000) % 100, + (ver / 100) % 100, + ver % 100) + + raise Exception('could not figure out Duktape version') + +def main(): + outfile = sys.argv[1] + + if not os.path.exists('CONTRIBUTING.md') and os.path.exists('ecmascript-testcases'): + sys.stderr.write('Invalid CWD, must be in Duktape root with dist/ built') + sys.exit(1) + os.chdir('dist') + if not os.path.exists('Makefile.cmdline'): + sys.stderr.write('Invalid CWD, must be in Duktape root with dist/ built') + sys.exit(1) + + duktape_version = getDuktapeVersion() + duktape_pkgname = 'duktape-' + duktape_version + '.tar.xz' + now = datetime.datetime.utcnow() + now = datetime.datetime(now.year, now.month, now.day, now.hour, now.minute, now.second) + creation_date = Literal(now.isoformat() + 'Z', datatype=XSD.dateTime) + duktape_org = Literal('Organization: duktape.org') + mit_license = URIRef('http://spdx.org/licenses/MIT') + duktape_copyright = Literal('Copyright 2013-2014 Duktape authors (see AUTHORS.rst in the Duktape distributable)') + + g = rdflib.Graph() + + crea_node = BNode() + g.add((crea_node, RDF.type, SPDX.CreationInfo)) + g.add((crea_node, RDFS.comment, Literal(''))) + g.add((crea_node, SPDX.creator, duktape_org)) + g.add((crea_node, SPDX.created, creation_date)) + g.add((crea_node, SPDX.licenseListVersion, Literal('1.20'))) # http://spdx.org/licenses/ + + # 'name' should not include a version number (see best practices) + pkg_node = BNode() + g.add((pkg_node, RDF.type, SPDX.Package)) + g.add((pkg_node, SPDX.name, Literal('Duktape'))) + g.add((pkg_node, SPDX.versionInfo, Literal(duktape_version))) + g.add((pkg_node, SPDX.packageFileName, Literal(duktape_pkgname))) + g.add((pkg_node, SPDX.supplier, duktape_org)) + g.add((pkg_node, SPDX.originator, duktape_org)) + g.add((pkg_node, SPDX.downloadLocation, Literal('http://duktape.org/' + duktape_pkgname, datatype=XSD.anyURI))) + g.add((pkg_node, SPDX.homePage, Literal('http://duktape.org/', datatype=XSD.anyURI))) + verify_node = computePackageVerification(g, '.', [ './license.xml' ]) + g.add((pkg_node, SPDX.packageVerificationCode, verify_node)) + # SPDX.checksum: omitted because license is inside the package + g.add((pkg_node, SPDX.sourceInfo, Literal('Official duktape.org release built from GitHub repo https://github.com/svaarala/duktape.'))) + + # NOTE: MIT license alone is sufficient fornow, because Duktape, MurmurHash2 and + # CommonJS (though probably not even relevant for licensing) are all MIT. + g.add((pkg_node, SPDX.licenseConcluded, mit_license)) + g.add((pkg_node, SPDX.licenseInfoFromFiles, mit_license)) + g.add((pkg_node, SPDX.licenseDeclared, mit_license)) + g.add((pkg_node, SPDX.licenseComments, Literal('Duktape is copyrighted by its authors and licensed under the MIT license. MurmurHash2 is used internally, it is also under the MIT license. Duktape module loader is based on the CommonJS module loading specification (without sharing any code), CommonJS is under the MIT license.'))) + g.add((pkg_node, SPDX.copyrightText, duktape_copyright)) + g.add((pkg_node, SPDX.summary, Literal('Duktape Ecmascript interpreter'))) + g.add((pkg_node, SPDX.description, Literal('Duktape is an embeddable Javascript engine, with a focus on portability and compact footprint'))) + # hasFile properties added separately below + + #reviewed_node = BNode() + #g.add((reviewed_node, RDF.type, SPDX.Review)) + #g.add((reviewed_node, SPDX.reviewer, XXX)) + #g.add((reviewed_node, SPDX.reviewDate, XXX)) + #g.add((reviewed_node, RDFS.comment, '')) + + spdx_doc = BNode() + g.add((spdx_doc, RDF.type, SPDX.SpdxDocument)) + g.add((spdx_doc, SPDX.specVersion, Literal('SPDX-1.2'))) + g.add((spdx_doc, SPDX.dataLicense, URIRef('http://spdx.org/licenses/CC0-1.0'))) + g.add((spdx_doc, RDFS.comment, Literal('SPDX license for Duktape ' + duktape_version))) + g.add((spdx_doc, SPDX.creationInfo, crea_node)) + g.add((spdx_doc, SPDX.describesPackage, pkg_node)) + # SPDX.hasExtractedLicensingInfo + # SPDX.reviewed + # SPDX.referencesFile: added below + + for dirpath, dirnames, filenames in os.walk('.'): + for fn in filenames: + full_fn = os.path.join(dirpath, fn) + #print('# file: ' + full_fn) + + file_node = BNode() + g.add((file_node, RDF.type, SPDX.File)) + g.add((file_node, SPDX.fileName, Literal(full_fn))) + g.add((file_node, SPDX.fileType, fileType(full_fn))) + g.add((file_node, SPDX.checksum, checksumFile(g, full_fn))) + + # Here we assume that LICENSE.txt provides the actual "in file" + # licensing information, and everything else is implicitly under + # MIT license. + g.add((file_node, SPDX.licenseConcluded, mit_license)) + if full_fn == './LICENSE.txt': + g.add((file_node, SPDX.licenseInfoInFile, mit_license)) + else: + g.add((file_node, SPDX.licenseInfoInFile, URIRef(SPDX.none))) + + # SPDX.licenseComments + g.add((file_node, SPDX.copyrightText, duktape_copyright)) + # SPDX.noticeText + # SPDX.artifactOf + # SPDX.fileDependency + # SPDX.fileContributor + + # XXX: should referencesFile include all files? + g.add((spdx_doc, SPDX.referencesFile, file_node)) + + g.add((pkg_node, SPDX.hasFile, file_node)) + + # Serialize into RDF/XML directly. We could also serialize into + # N-Triples and use external tools (like 'rapper') to get cleaner, + # abbreviated output. + + #print('# Duktape SPDX license file (autogenerated)') + #print(g.serialize(format='turtle')) + #print(g.serialize(format='nt')) + f = open(outfile, 'wb') + f.write(g.serialize(format='rdf/xml')) + f.close() + +if __name__ == '__main__': + main() From 1027715bc81ac74aa407d84244b27165983f8f81 Mon Sep 17 00:00:00 2001 From: Sami Vaarala Date: Thu, 13 Nov 2014 05:40:05 +0200 Subject: [PATCH 2/5] Add python-rdflib to README deps It is needed to create the SPDX license file. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aaed2533..115beea9 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Getting started: developing Duktape If you intend to change Duktape internals, run test cases, etc: # Install required packages - $ sudo apt-get install nodejs npm perl ant openjdk-7-jdk libreadline6-dev libncurses-dev + $ sudo apt-get install nodejs npm perl ant openjdk-7-jdk libreadline6-dev libncurses-dev python-rdflib # Compile the command line tool ('duk') $ git clone https://github.com/svaarala/duktape.git From 2310542fafd48ecc9bd0cc49c3171be2b8386644 Mon Sep 17 00:00:00 2001 From: Sami Vaarala Date: Thu, 13 Nov 2014 05:40:20 +0200 Subject: [PATCH 3/5] Add SPDX license to dist --- util/make_dist.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/util/make_dist.sh b/util/make_dist.sh index 3bcafd55..e007e861 100644 --- a/util/make_dist.sh +++ b/util/make_dist.sh @@ -676,6 +676,8 @@ python util/combine_src.py $DISTSRCSEP $DISTSRCCOM/duktape.c \ echo "CLOC report on combined duktape.c source file" perl cloc-1.60.pl --quiet $DISTSRCCOM/duktape.c -# Clean up remaining files - +# Clean up temp files rm $DIST/*.tmp + +# Create SPDX license once all other files are in place (and cleaned) +python util/create_spdx_license.py `pwd`/dist/license.xml From 350cd024c0e2967d1c6830ceff47d59ca4fe1ebb Mon Sep 17 00:00:00 2001 From: Sami Vaarala Date: Thu, 13 Nov 2014 05:40:41 +0200 Subject: [PATCH 4/5] Release note: SPDX 1.2 release --- RELEASES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RELEASES.rst b/RELEASES.rst index 19090993..fb590ee9 100644 --- a/RELEASES.rst +++ b/RELEASES.rst @@ -654,6 +654,8 @@ Planned detecting memory safety issues on platforms where valgrind is not available +* Add an SPDX 1.2 license into the distributable + 1.2.0 (2015-XX-XX) ------------------ From fc68fc387620fa4cf6ee1209ece00d625f9d1214 Mon Sep 17 00:00:00 2001 From: Sami Vaarala Date: Thu, 13 Nov 2014 06:12:45 +0200 Subject: [PATCH 5/5] Use 'license.spdx' for SPDX license name Also add 'set -e' to make_dist.sh. --- util/create_spdx_license.py | 20 ++++++++++++-------- util/make_dist.sh | 34 ++++++++++++++++++---------------- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/util/create_spdx_license.py b/util/create_spdx_license.py index 3e635c55..acb4e059 100644 --- a/util/create_spdx_license.py +++ b/util/create_spdx_license.py @@ -8,16 +8,19 @@ # # The utility outputs RDF/XML to specified file: # -# $ python create_spdx_license.py /tmp/license.xml +# $ python create_spdx_license.py /tmp/license.spdx # # Then, validate with SPDXViewer and SPDXTools: # -# $ java -jar SPDXViewer.jar /tmp/license.xml -# $ java -jar java -jar spdx-tools-1.2.5-jar-with-dependencies.jar RdfToHtml /tmp/license.xml /tmp/license.html +# $ java -jar SPDXViewer.jar /tmp/license.spdx +# $ java -jar java -jar spdx-tools-1.2.5-jar-with-dependencies.jar RdfToHtml /tmp/license.spdx /tmp/license.html # # Finally, copy to dist: # -# $ cp /tmp/license.xml dist/ +# $ cp /tmp/license.spdx dist/license.spdx +# +# SPDX FAQ indicates there is no standard extension for an SPDX license file +# but '.spdx' is a common practice. # # The algorithm to compute a "verification code", implemented in this file, # can be verified as follows: @@ -25,13 +28,14 @@ # # build dist tar.xz, copy to /tmp/duktape-N.N.N.tar.xz # $ cd /tmp # $ tar xvfJ duktape-N.N.N.tar.xz -# $ rm duktape-N.N.N/license.xml # remove file excluded from verification code +# $ rm duktape-N.N.N/license.spdx # remove file excluded from verification code # $ java -jar spdx-tools-1.2.5-jar-with-dependencies.jar GenerateVerificationCode /tmp/duktape-N.N.N/ # -# Compare the resulting verification code manually with the one in license.xml. +# Compare the resulting verification code manually with the one in license.spdx. # # Resources: # +# - http://spdx.org/about-spdx/faqs # - http://wiki.spdx.org/view/Technical_Team/Best_Practices # @@ -67,7 +71,7 @@ def computePackageVerification(g, dirname, excluded): # SPDX 1.2 Section 4.7 # The SPDXTools command "GenerateVerificationCode" can be used to # check the verification codes created. Note that you must manually - # remove "license.xml" from the unpacked dist directory before + # remove "license.spdx" from the unpacked dist directory before # computing the verification code. verify_node = BNode() @@ -160,7 +164,7 @@ def main(): g.add((pkg_node, SPDX.originator, duktape_org)) g.add((pkg_node, SPDX.downloadLocation, Literal('http://duktape.org/' + duktape_pkgname, datatype=XSD.anyURI))) g.add((pkg_node, SPDX.homePage, Literal('http://duktape.org/', datatype=XSD.anyURI))) - verify_node = computePackageVerification(g, '.', [ './license.xml' ]) + verify_node = computePackageVerification(g, '.', [ './license.spdx' ]) g.add((pkg_node, SPDX.packageVerificationCode, verify_node)) # SPDX.checksum: omitted because license is inside the package g.add((pkg_node, SPDX.sourceInfo, Literal('Official duktape.org release built from GitHub repo https://github.com/svaarala/duktape.'))) diff --git a/util/make_dist.sh b/util/make_dist.sh index e007e861..8e1d26b0 100644 --- a/util/make_dist.sh +++ b/util/make_dist.sh @@ -19,6 +19,8 @@ # example Makefiles are packaged into the dist package. # +set -e # exit on errors + INITJS_MINIFY=closure while [ $# -gt 0 ]; do case "$1" in @@ -186,7 +188,7 @@ for i in \ duk_replacements.c \ duk_replacements.h \ ; do - cp src/$i $DISTSRCSEP/ || exit 1 + cp src/$i $DISTSRCSEP/ done for i in \ @@ -196,15 +198,15 @@ for i in \ object-assign.js \ performance-now.js \ ; do - cp polyfills/$i $DIST/polyfills/ || exit 1 + cp polyfills/$i $DIST/polyfills/ done -cp examples/README.rst $DIST/examples/ || exit 1 +cp examples/README.rst $DIST/examples/ for i in \ README.rst \ duk_cmdline.c \ ; do - cp examples/cmdline/$i $DIST/examples/cmdline/ || exit 1 + cp examples/cmdline/$i $DIST/examples/cmdline/ done for i in \ @@ -222,21 +224,21 @@ for i in \ server-socket-test.js \ client-socket-test.js \ ; do - cp examples/eventloop/$i $DIST/examples/eventloop/ || exit 1 + cp examples/eventloop/$i $DIST/examples/eventloop/ done for i in \ README.rst \ hello.c \ ; do - cp examples/hello/$i $DIST/examples/hello/ || exit 1 + cp examples/hello/$i $DIST/examples/hello/ done for i in \ README.rst \ eval.c \ ; do - cp examples/eval/$i $DIST/examples/eval/ || exit 1 + cp examples/eval/$i $DIST/examples/eval/ done for i in \ @@ -248,7 +250,7 @@ for i in \ primecheck.c \ uppercase.c \ ; do - cp examples/guide/$i $DIST/examples/guide/ || exit 1 + cp examples/guide/$i $DIST/examples/guide/ done for i in \ @@ -257,21 +259,21 @@ for i in \ hello.coffee \ mandel.coffee \ ; do - cp examples/coffee/$i $DIST/examples/coffee/ || exit 1 + cp examples/coffee/$i $DIST/examples/coffee/ done for i in \ README.rst \ jxpretty.c \ ; do - cp examples/jxpretty/$i $DIST/examples/jxpretty/ || exit 1 + cp examples/jxpretty/$i $DIST/examples/jxpretty/ done for i in \ README.rst \ sandbox.c \ ; do - cp examples/sandbox/$i $DIST/examples/sandbox/ || exit 1 + cp examples/sandbox/$i $DIST/examples/sandbox/ done for i in \ @@ -280,7 +282,7 @@ for i in \ duk_alloc_logging.h \ log2gnuplot.py \ ; do - cp examples/alloc-logging/$i $DIST/examples/alloc-logging/ || exit 1 + cp examples/alloc-logging/$i $DIST/examples/alloc-logging/ done for i in \ @@ -288,10 +290,10 @@ for i in \ duk_alloc_torture.c \ duk_alloc_torture.h \ ; do - cp examples/alloc-torture/$i $DIST/examples/alloc-torture/ || exit 1 + cp examples/alloc-torture/$i $DIST/examples/alloc-torture/ done -cp extras/README.rst $DIST/extras/ || exit +cp extras/README.rst $DIST/extras/ # XXX: copy extras for i in \ @@ -304,7 +306,7 @@ for i in \ Makefile.sandbox \ mandel.js \ ; do - cp dist-files/$i $DIST/ || exit 1 + cp dist-files/$i $DIST/ done cat dist-files/README.rst | sed \ @@ -680,4 +682,4 @@ perl cloc-1.60.pl --quiet $DISTSRCCOM/duktape.c rm $DIST/*.tmp # Create SPDX license once all other files are in place (and cleaned) -python util/create_spdx_license.py `pwd`/dist/license.xml +python util/create_spdx_license.py `pwd`/dist/license.spdx