From e7ba21911c26f94df1f658d04bfc18cc022a9188 Mon Sep 17 00:00:00 2001 From: Sami Vaarala Date: Fri, 26 Aug 2016 04:25:07 +0300 Subject: [PATCH 1/9] Move CaseConversion.java to misc --- {src => misc}/CaseConversion.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {src => misc}/CaseConversion.java (100%) diff --git a/src/CaseConversion.java b/misc/CaseConversion.java similarity index 100% rename from src/CaseConversion.java rename to misc/CaseConversion.java From cac8061b4c38b2695d55f20d2a055aebccccb3aa Mon Sep 17 00:00:00 2001 From: Sami Vaarala Date: Sat, 27 Aug 2016 00:01:27 +0300 Subject: [PATCH 2/9] Underscore test shim fixes --- util/underscore-test-shim.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/util/underscore-test-shim.js b/util/underscore-test-shim.js index 1decd611..b5f575e3 100644 --- a/util/underscore-test-shim.js +++ b/util/underscore-test-shim.js @@ -188,3 +188,13 @@ $.browser = { var jQuery = $; +// document.createElement() is used by some tests +document.createElement = function createElement(name) { + return { name: name }; // Just a dummy return value +}; + +// the Duktape require() function confuses underscore tests +delete require; + +// ... and the 'window' binding is expected +var window = new Function('return this;')(); From 98eaa8605e74e7fb4cce85a22f827378165ffed2 Mon Sep 17 00:00:00 2001 From: Sami Vaarala Date: Sat, 27 Aug 2016 04:37:18 +0300 Subject: [PATCH 3/9] Add 8-bit manually stripped Unicode files --- src/SpecialCasing-8bit.txt | 1 + src/UnicodeData-8bit.txt | 256 +++++++++++++++++++++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100644 src/SpecialCasing-8bit.txt create mode 100644 src/UnicodeData-8bit.txt diff --git a/src/SpecialCasing-8bit.txt b/src/SpecialCasing-8bit.txt new file mode 100644 index 00000000..9c14466e --- /dev/null +++ b/src/SpecialCasing-8bit.txt @@ -0,0 +1 @@ +00DF; 00DF; 0053 0073; 0053 0053; # LATIN SMALL LETTER SHARP S diff --git a/src/UnicodeData-8bit.txt b/src/UnicodeData-8bit.txt new file mode 100644 index 00000000..01448ff7 --- /dev/null +++ b/src/UnicodeData-8bit.txt @@ -0,0 +1,256 @@ +0000;;Cc;0;BN;;;;;N;NULL;;;; +0001;;Cc;0;BN;;;;;N;START OF HEADING;;;; +0002;;Cc;0;BN;;;;;N;START OF TEXT;;;; +0003;;Cc;0;BN;;;;;N;END OF TEXT;;;; +0004;;Cc;0;BN;;;;;N;END OF TRANSMISSION;;;; +0005;;Cc;0;BN;;;;;N;ENQUIRY;;;; +0006;;Cc;0;BN;;;;;N;ACKNOWLEDGE;;;; +0007;;Cc;0;BN;;;;;N;BELL;;;; +0008;;Cc;0;BN;;;;;N;BACKSPACE;;;; +0009;;Cc;0;S;;;;;N;CHARACTER TABULATION;;;; +000A;;Cc;0;B;;;;;N;LINE FEED (LF);;;; +000B;;Cc;0;S;;;;;N;LINE TABULATION;;;; +000C;;Cc;0;WS;;;;;N;FORM FEED (FF);;;; +000D;;Cc;0;B;;;;;N;CARRIAGE RETURN (CR);;;; +000E;;Cc;0;BN;;;;;N;SHIFT OUT;;;; +000F;;Cc;0;BN;;;;;N;SHIFT IN;;;; +0010;;Cc;0;BN;;;;;N;DATA LINK ESCAPE;;;; +0011;;Cc;0;BN;;;;;N;DEVICE CONTROL ONE;;;; +0012;;Cc;0;BN;;;;;N;DEVICE CONTROL TWO;;;; +0013;;Cc;0;BN;;;;;N;DEVICE CONTROL THREE;;;; +0014;;Cc;0;BN;;;;;N;DEVICE CONTROL FOUR;;;; +0015;;Cc;0;BN;;;;;N;NEGATIVE ACKNOWLEDGE;;;; +0016;;Cc;0;BN;;;;;N;SYNCHRONOUS IDLE;;;; +0017;;Cc;0;BN;;;;;N;END OF TRANSMISSION BLOCK;;;; +0018;;Cc;0;BN;;;;;N;CANCEL;;;; +0019;;Cc;0;BN;;;;;N;END OF MEDIUM;;;; +001A;;Cc;0;BN;;;;;N;SUBSTITUTE;;;; +001B;;Cc;0;BN;;;;;N;ESCAPE;;;; +001C;;Cc;0;B;;;;;N;INFORMATION SEPARATOR FOUR;;;; +001D;;Cc;0;B;;;;;N;INFORMATION SEPARATOR THREE;;;; +001E;;Cc;0;B;;;;;N;INFORMATION SEPARATOR TWO;;;; +001F;;Cc;0;S;;;;;N;INFORMATION SEPARATOR ONE;;;; +0020;SPACE;Zs;0;WS;;;;;N;;;;; +0021;EXCLAMATION MARK;Po;0;ON;;;;;N;;;;; +0022;QUOTATION MARK;Po;0;ON;;;;;N;;;;; +0023;NUMBER SIGN;Po;0;ET;;;;;N;;;;; +0024;DOLLAR SIGN;Sc;0;ET;;;;;N;;;;; +0025;PERCENT SIGN;Po;0;ET;;;;;N;;;;; +0026;AMPERSAND;Po;0;ON;;;;;N;;;;; +0027;APOSTROPHE;Po;0;ON;;;;;N;APOSTROPHE-QUOTE;;;; +0028;LEFT PARENTHESIS;Ps;0;ON;;;;;Y;OPENING PARENTHESIS;;;; +0029;RIGHT PARENTHESIS;Pe;0;ON;;;;;Y;CLOSING PARENTHESIS;;;; +002A;ASTERISK;Po;0;ON;;;;;N;;;;; +002B;PLUS SIGN;Sm;0;ES;;;;;N;;;;; +002C;COMMA;Po;0;CS;;;;;N;;;;; +002D;HYPHEN-MINUS;Pd;0;ES;;;;;N;;;;; +002E;FULL STOP;Po;0;CS;;;;;N;PERIOD;;;; +002F;SOLIDUS;Po;0;CS;;;;;N;SLASH;;;; +0030;DIGIT ZERO;Nd;0;EN;;0;0;0;N;;;;; +0031;DIGIT ONE;Nd;0;EN;;1;1;1;N;;;;; +0032;DIGIT TWO;Nd;0;EN;;2;2;2;N;;;;; +0033;DIGIT THREE;Nd;0;EN;;3;3;3;N;;;;; +0034;DIGIT FOUR;Nd;0;EN;;4;4;4;N;;;;; +0035;DIGIT FIVE;Nd;0;EN;;5;5;5;N;;;;; +0036;DIGIT SIX;Nd;0;EN;;6;6;6;N;;;;; +0037;DIGIT SEVEN;Nd;0;EN;;7;7;7;N;;;;; +0038;DIGIT EIGHT;Nd;0;EN;;8;8;8;N;;;;; +0039;DIGIT NINE;Nd;0;EN;;9;9;9;N;;;;; +003A;COLON;Po;0;CS;;;;;N;;;;; +003B;SEMICOLON;Po;0;ON;;;;;N;;;;; +003C;LESS-THAN SIGN;Sm;0;ON;;;;;Y;;;;; +003D;EQUALS SIGN;Sm;0;ON;;;;;N;;;;; +003E;GREATER-THAN SIGN;Sm;0;ON;;;;;Y;;;;; +003F;QUESTION MARK;Po;0;ON;;;;;N;;;;; +0040;COMMERCIAL AT;Po;0;ON;;;;;N;;;;; +0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061; +0042;LATIN CAPITAL LETTER B;Lu;0;L;;;;;N;;;;0062; +0043;LATIN CAPITAL LETTER C;Lu;0;L;;;;;N;;;;0063; +0044;LATIN CAPITAL LETTER D;Lu;0;L;;;;;N;;;;0064; +0045;LATIN CAPITAL LETTER E;Lu;0;L;;;;;N;;;;0065; +0046;LATIN CAPITAL LETTER F;Lu;0;L;;;;;N;;;;0066; +0047;LATIN CAPITAL LETTER G;Lu;0;L;;;;;N;;;;0067; +0048;LATIN CAPITAL LETTER H;Lu;0;L;;;;;N;;;;0068; +0049;LATIN CAPITAL LETTER I;Lu;0;L;;;;;N;;;;0069; +004A;LATIN CAPITAL LETTER J;Lu;0;L;;;;;N;;;;006A; +004B;LATIN CAPITAL LETTER K;Lu;0;L;;;;;N;;;;006B; +004C;LATIN CAPITAL LETTER L;Lu;0;L;;;;;N;;;;006C; +004D;LATIN CAPITAL LETTER M;Lu;0;L;;;;;N;;;;006D; +004E;LATIN CAPITAL LETTER N;Lu;0;L;;;;;N;;;;006E; +004F;LATIN CAPITAL LETTER O;Lu;0;L;;;;;N;;;;006F; +0050;LATIN CAPITAL LETTER P;Lu;0;L;;;;;N;;;;0070; +0051;LATIN CAPITAL LETTER Q;Lu;0;L;;;;;N;;;;0071; +0052;LATIN CAPITAL LETTER R;Lu;0;L;;;;;N;;;;0072; +0053;LATIN CAPITAL LETTER S;Lu;0;L;;;;;N;;;;0073; +0054;LATIN CAPITAL LETTER T;Lu;0;L;;;;;N;;;;0074; +0055;LATIN CAPITAL LETTER U;Lu;0;L;;;;;N;;;;0075; +0056;LATIN CAPITAL LETTER V;Lu;0;L;;;;;N;;;;0076; +0057;LATIN CAPITAL LETTER W;Lu;0;L;;;;;N;;;;0077; +0058;LATIN CAPITAL LETTER X;Lu;0;L;;;;;N;;;;0078; +0059;LATIN CAPITAL LETTER Y;Lu;0;L;;;;;N;;;;0079; +005A;LATIN CAPITAL LETTER Z;Lu;0;L;;;;;N;;;;007A; +005B;LEFT SQUARE BRACKET;Ps;0;ON;;;;;Y;OPENING SQUARE BRACKET;;;; +005C;REVERSE SOLIDUS;Po;0;ON;;;;;N;BACKSLASH;;;; +005D;RIGHT SQUARE BRACKET;Pe;0;ON;;;;;Y;CLOSING SQUARE BRACKET;;;; +005E;CIRCUMFLEX ACCENT;Sk;0;ON;;;;;N;SPACING CIRCUMFLEX;;;; +005F;LOW LINE;Pc;0;ON;;;;;N;SPACING UNDERSCORE;;;; +0060;GRAVE ACCENT;Sk;0;ON;;;;;N;SPACING GRAVE;;;; +0061;LATIN SMALL LETTER A;Ll;0;L;;;;;N;;;0041;;0041 +0062;LATIN SMALL LETTER B;Ll;0;L;;;;;N;;;0042;;0042 +0063;LATIN SMALL LETTER C;Ll;0;L;;;;;N;;;0043;;0043 +0064;LATIN SMALL LETTER D;Ll;0;L;;;;;N;;;0044;;0044 +0065;LATIN SMALL LETTER E;Ll;0;L;;;;;N;;;0045;;0045 +0066;LATIN SMALL LETTER F;Ll;0;L;;;;;N;;;0046;;0046 +0067;LATIN SMALL LETTER G;Ll;0;L;;;;;N;;;0047;;0047 +0068;LATIN SMALL LETTER H;Ll;0;L;;;;;N;;;0048;;0048 +0069;LATIN SMALL LETTER I;Ll;0;L;;;;;N;;;0049;;0049 +006A;LATIN SMALL LETTER J;Ll;0;L;;;;;N;;;004A;;004A +006B;LATIN SMALL LETTER K;Ll;0;L;;;;;N;;;004B;;004B +006C;LATIN SMALL LETTER L;Ll;0;L;;;;;N;;;004C;;004C +006D;LATIN SMALL LETTER M;Ll;0;L;;;;;N;;;004D;;004D +006E;LATIN SMALL LETTER N;Ll;0;L;;;;;N;;;004E;;004E +006F;LATIN SMALL LETTER O;Ll;0;L;;;;;N;;;004F;;004F +0070;LATIN SMALL LETTER P;Ll;0;L;;;;;N;;;0050;;0050 +0071;LATIN SMALL LETTER Q;Ll;0;L;;;;;N;;;0051;;0051 +0072;LATIN SMALL LETTER R;Ll;0;L;;;;;N;;;0052;;0052 +0073;LATIN SMALL LETTER S;Ll;0;L;;;;;N;;;0053;;0053 +0074;LATIN SMALL LETTER T;Ll;0;L;;;;;N;;;0054;;0054 +0075;LATIN SMALL LETTER U;Ll;0;L;;;;;N;;;0055;;0055 +0076;LATIN SMALL LETTER V;Ll;0;L;;;;;N;;;0056;;0056 +0077;LATIN SMALL LETTER W;Ll;0;L;;;;;N;;;0057;;0057 +0078;LATIN SMALL LETTER X;Ll;0;L;;;;;N;;;0058;;0058 +0079;LATIN SMALL LETTER Y;Ll;0;L;;;;;N;;;0059;;0059 +007A;LATIN SMALL LETTER Z;Ll;0;L;;;;;N;;;005A;;005A +007B;LEFT CURLY BRACKET;Ps;0;ON;;;;;Y;OPENING CURLY BRACKET;;;; +007C;VERTICAL LINE;Sm;0;ON;;;;;N;VERTICAL BAR;;;; +007D;RIGHT CURLY BRACKET;Pe;0;ON;;;;;Y;CLOSING CURLY BRACKET;;;; +007E;TILDE;Sm;0;ON;;;;;N;;;;; +007F;;Cc;0;BN;;;;;N;DELETE;;;; +0080;;Cc;0;BN;;;;;N;;;;; +0081;;Cc;0;BN;;;;;N;;;;; +0082;;Cc;0;BN;;;;;N;BREAK PERMITTED HERE;;;; +0083;;Cc;0;BN;;;;;N;NO BREAK HERE;;;; +0084;;Cc;0;BN;;;;;N;;;;; +0085;;Cc;0;B;;;;;N;NEXT LINE (NEL);;;; +0086;;Cc;0;BN;;;;;N;START OF SELECTED AREA;;;; +0087;;Cc;0;BN;;;;;N;END OF SELECTED AREA;;;; +0088;;Cc;0;BN;;;;;N;CHARACTER TABULATION SET;;;; +0089;;Cc;0;BN;;;;;N;CHARACTER TABULATION WITH JUSTIFICATION;;;; +008A;;Cc;0;BN;;;;;N;LINE TABULATION SET;;;; +008B;;Cc;0;BN;;;;;N;PARTIAL LINE FORWARD;;;; +008C;;Cc;0;BN;;;;;N;PARTIAL LINE BACKWARD;;;; +008D;;Cc;0;BN;;;;;N;REVERSE LINE FEED;;;; +008E;;Cc;0;BN;;;;;N;SINGLE SHIFT TWO;;;; +008F;;Cc;0;BN;;;;;N;SINGLE SHIFT THREE;;;; +0090;;Cc;0;BN;;;;;N;DEVICE CONTROL STRING;;;; +0091;;Cc;0;BN;;;;;N;PRIVATE USE ONE;;;; +0092;;Cc;0;BN;;;;;N;PRIVATE USE TWO;;;; +0093;;Cc;0;BN;;;;;N;SET TRANSMIT STATE;;;; +0094;;Cc;0;BN;;;;;N;CANCEL CHARACTER;;;; +0095;;Cc;0;BN;;;;;N;MESSAGE WAITING;;;; +0096;;Cc;0;BN;;;;;N;START OF GUARDED AREA;;;; +0097;;Cc;0;BN;;;;;N;END OF GUARDED AREA;;;; +0098;;Cc;0;BN;;;;;N;START OF STRING;;;; +0099;;Cc;0;BN;;;;;N;;;;; +009A;;Cc;0;BN;;;;;N;SINGLE CHARACTER INTRODUCER;;;; +009B;;Cc;0;BN;;;;;N;CONTROL SEQUENCE INTRODUCER;;;; +009C;;Cc;0;BN;;;;;N;STRING TERMINATOR;;;; +009D;;Cc;0;BN;;;;;N;OPERATING SYSTEM COMMAND;;;; +009E;;Cc;0;BN;;;;;N;PRIVACY MESSAGE;;;; +009F;;Cc;0;BN;;;;;N;APPLICATION PROGRAM COMMAND;;;; +00A0;NO-BREAK SPACE;Zs;0;CS; 0020;;;;N;NON-BREAKING SPACE;;;; +00A1;INVERTED EXCLAMATION MARK;Po;0;ON;;;;;N;;;;; +00A2;CENT SIGN;Sc;0;ET;;;;;N;;;;; +00A3;POUND SIGN;Sc;0;ET;;;;;N;;;;; +00A4;CURRENCY SIGN;Sc;0;ET;;;;;N;;;;; +00A5;YEN SIGN;Sc;0;ET;;;;;N;;;;; +00A6;BROKEN BAR;So;0;ON;;;;;N;BROKEN VERTICAL BAR;;;; +00A7;SECTION SIGN;Po;0;ON;;;;;N;;;;; +00A8;DIAERESIS;Sk;0;ON; 0020 0308;;;;N;SPACING DIAERESIS;;;; +00A9;COPYRIGHT SIGN;So;0;ON;;;;;N;;;;; +00AA;FEMININE ORDINAL INDICATOR;Lo;0;L; 0061;;;;N;;;;; +00AB;LEFT-POINTING DOUBLE ANGLE QUOTATION MARK;Pi;0;ON;;;;;Y;LEFT POINTING GUILLEMET;;;; +00AC;NOT SIGN;Sm;0;ON;;;;;N;;;;; +00AD;SOFT HYPHEN;Cf;0;BN;;;;;N;;;;; +00AE;REGISTERED SIGN;So;0;ON;;;;;N;REGISTERED TRADE MARK SIGN;;;; +00AF;MACRON;Sk;0;ON; 0020 0304;;;;N;SPACING MACRON;;;; +00B0;DEGREE SIGN;So;0;ET;;;;;N;;;;; +00B1;PLUS-MINUS SIGN;Sm;0;ET;;;;;N;PLUS-OR-MINUS SIGN;;;; +00B2;SUPERSCRIPT TWO;No;0;EN; 0032;;2;2;N;SUPERSCRIPT DIGIT TWO;;;; +00B3;SUPERSCRIPT THREE;No;0;EN; 0033;;3;3;N;SUPERSCRIPT DIGIT THREE;;;; +00B4;ACUTE ACCENT;Sk;0;ON; 0020 0301;;;;N;SPACING ACUTE;;;; +00B5;MICRO SIGN;Ll;0;L; 03BC;;;;N;;;039C;;039C +00B6;PILCROW SIGN;Po;0;ON;;;;;N;PARAGRAPH SIGN;;;; +00B7;MIDDLE DOT;Po;0;ON;;;;;N;;;;; +00B8;CEDILLA;Sk;0;ON; 0020 0327;;;;N;SPACING CEDILLA;;;; +00B9;SUPERSCRIPT ONE;No;0;EN; 0031;;1;1;N;SUPERSCRIPT DIGIT ONE;;;; +00BA;MASCULINE ORDINAL INDICATOR;Lo;0;L; 006F;;;;N;;;;; +00BB;RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK;Pf;0;ON;;;;;Y;RIGHT POINTING GUILLEMET;;;; +00BC;VULGAR FRACTION ONE QUARTER;No;0;ON; 0031 2044 0034;;;1/4;N;FRACTION ONE QUARTER;;;; +00BD;VULGAR FRACTION ONE HALF;No;0;ON; 0031 2044 0032;;;1/2;N;FRACTION ONE HALF;;;; +00BE;VULGAR FRACTION THREE QUARTERS;No;0;ON; 0033 2044 0034;;;3/4;N;FRACTION THREE QUARTERS;;;; +00BF;INVERTED QUESTION MARK;Po;0;ON;;;;;N;;;;; +00C0;LATIN CAPITAL LETTER A WITH GRAVE;Lu;0;L;0041 0300;;;;N;LATIN CAPITAL LETTER A GRAVE;;;00E0; +00C1;LATIN CAPITAL LETTER A WITH ACUTE;Lu;0;L;0041 0301;;;;N;LATIN CAPITAL LETTER A ACUTE;;;00E1; +00C2;LATIN CAPITAL LETTER A WITH CIRCUMFLEX;Lu;0;L;0041 0302;;;;N;LATIN CAPITAL LETTER A CIRCUMFLEX;;;00E2; +00C3;LATIN CAPITAL LETTER A WITH TILDE;Lu;0;L;0041 0303;;;;N;LATIN CAPITAL LETTER A TILDE;;;00E3; +00C4;LATIN CAPITAL LETTER A WITH DIAERESIS;Lu;0;L;0041 0308;;;;N;LATIN CAPITAL LETTER A DIAERESIS;;;00E4; +00C5;LATIN CAPITAL LETTER A WITH RING ABOVE;Lu;0;L;0041 030A;;;;N;LATIN CAPITAL LETTER A RING;;;00E5; +00C6;LATIN CAPITAL LETTER AE;Lu;0;L;;;;;N;LATIN CAPITAL LETTER A E;;;00E6; +00C7;LATIN CAPITAL LETTER C WITH CEDILLA;Lu;0;L;0043 0327;;;;N;LATIN CAPITAL LETTER C CEDILLA;;;00E7; +00C8;LATIN CAPITAL LETTER E WITH GRAVE;Lu;0;L;0045 0300;;;;N;LATIN CAPITAL LETTER E GRAVE;;;00E8; +00C9;LATIN CAPITAL LETTER E WITH ACUTE;Lu;0;L;0045 0301;;;;N;LATIN CAPITAL LETTER E ACUTE;;;00E9; +00CA;LATIN CAPITAL LETTER E WITH CIRCUMFLEX;Lu;0;L;0045 0302;;;;N;LATIN CAPITAL LETTER E CIRCUMFLEX;;;00EA; +00CB;LATIN CAPITAL LETTER E WITH DIAERESIS;Lu;0;L;0045 0308;;;;N;LATIN CAPITAL LETTER E DIAERESIS;;;00EB; +00CC;LATIN CAPITAL LETTER I WITH GRAVE;Lu;0;L;0049 0300;;;;N;LATIN CAPITAL LETTER I GRAVE;;;00EC; +00CD;LATIN CAPITAL LETTER I WITH ACUTE;Lu;0;L;0049 0301;;;;N;LATIN CAPITAL LETTER I ACUTE;;;00ED; +00CE;LATIN CAPITAL LETTER I WITH CIRCUMFLEX;Lu;0;L;0049 0302;;;;N;LATIN CAPITAL LETTER I CIRCUMFLEX;;;00EE; +00CF;LATIN CAPITAL LETTER I WITH DIAERESIS;Lu;0;L;0049 0308;;;;N;LATIN CAPITAL LETTER I DIAERESIS;;;00EF; +00D0;LATIN CAPITAL LETTER ETH;Lu;0;L;;;;;N;;;;00F0; +00D1;LATIN CAPITAL LETTER N WITH TILDE;Lu;0;L;004E 0303;;;;N;LATIN CAPITAL LETTER N TILDE;;;00F1; +00D2;LATIN CAPITAL LETTER O WITH GRAVE;Lu;0;L;004F 0300;;;;N;LATIN CAPITAL LETTER O GRAVE;;;00F2; +00D3;LATIN CAPITAL LETTER O WITH ACUTE;Lu;0;L;004F 0301;;;;N;LATIN CAPITAL LETTER O ACUTE;;;00F3; +00D4;LATIN CAPITAL LETTER O WITH CIRCUMFLEX;Lu;0;L;004F 0302;;;;N;LATIN CAPITAL LETTER O CIRCUMFLEX;;;00F4; +00D5;LATIN CAPITAL LETTER O WITH TILDE;Lu;0;L;004F 0303;;;;N;LATIN CAPITAL LETTER O TILDE;;;00F5; +00D6;LATIN CAPITAL LETTER O WITH DIAERESIS;Lu;0;L;004F 0308;;;;N;LATIN CAPITAL LETTER O DIAERESIS;;;00F6; +00D7;MULTIPLICATION SIGN;Sm;0;ON;;;;;N;;;;; +00D8;LATIN CAPITAL LETTER O WITH STROKE;Lu;0;L;;;;;N;LATIN CAPITAL LETTER O SLASH;;;00F8; +00D9;LATIN CAPITAL LETTER U WITH GRAVE;Lu;0;L;0055 0300;;;;N;LATIN CAPITAL LETTER U GRAVE;;;00F9; +00DA;LATIN CAPITAL LETTER U WITH ACUTE;Lu;0;L;0055 0301;;;;N;LATIN CAPITAL LETTER U ACUTE;;;00FA; +00DB;LATIN CAPITAL LETTER U WITH CIRCUMFLEX;Lu;0;L;0055 0302;;;;N;LATIN CAPITAL LETTER U CIRCUMFLEX;;;00FB; +00DC;LATIN CAPITAL LETTER U WITH DIAERESIS;Lu;0;L;0055 0308;;;;N;LATIN CAPITAL LETTER U DIAERESIS;;;00FC; +00DD;LATIN CAPITAL LETTER Y WITH ACUTE;Lu;0;L;0059 0301;;;;N;LATIN CAPITAL LETTER Y ACUTE;;;00FD; +00DE;LATIN CAPITAL LETTER THORN;Lu;0;L;;;;;N;;;;00FE; +00DF;LATIN SMALL LETTER SHARP S;Ll;0;L;;;;;N;;;;; +00E0;LATIN SMALL LETTER A WITH GRAVE;Ll;0;L;0061 0300;;;;N;LATIN SMALL LETTER A GRAVE;;00C0;;00C0 +00E1;LATIN SMALL LETTER A WITH ACUTE;Ll;0;L;0061 0301;;;;N;LATIN SMALL LETTER A ACUTE;;00C1;;00C1 +00E2;LATIN SMALL LETTER A WITH CIRCUMFLEX;Ll;0;L;0061 0302;;;;N;LATIN SMALL LETTER A CIRCUMFLEX;;00C2;;00C2 +00E3;LATIN SMALL LETTER A WITH TILDE;Ll;0;L;0061 0303;;;;N;LATIN SMALL LETTER A TILDE;;00C3;;00C3 +00E4;LATIN SMALL LETTER A WITH DIAERESIS;Ll;0;L;0061 0308;;;;N;LATIN SMALL LETTER A DIAERESIS;;00C4;;00C4 +00E5;LATIN SMALL LETTER A WITH RING ABOVE;Ll;0;L;0061 030A;;;;N;LATIN SMALL LETTER A RING;;00C5;;00C5 +00E6;LATIN SMALL LETTER AE;Ll;0;L;;;;;N;LATIN SMALL LETTER A E;;00C6;;00C6 +00E7;LATIN SMALL LETTER C WITH CEDILLA;Ll;0;L;0063 0327;;;;N;LATIN SMALL LETTER C CEDILLA;;00C7;;00C7 +00E8;LATIN SMALL LETTER E WITH GRAVE;Ll;0;L;0065 0300;;;;N;LATIN SMALL LETTER E GRAVE;;00C8;;00C8 +00E9;LATIN SMALL LETTER E WITH ACUTE;Ll;0;L;0065 0301;;;;N;LATIN SMALL LETTER E ACUTE;;00C9;;00C9 +00EA;LATIN SMALL LETTER E WITH CIRCUMFLEX;Ll;0;L;0065 0302;;;;N;LATIN SMALL LETTER E CIRCUMFLEX;;00CA;;00CA +00EB;LATIN SMALL LETTER E WITH DIAERESIS;Ll;0;L;0065 0308;;;;N;LATIN SMALL LETTER E DIAERESIS;;00CB;;00CB +00EC;LATIN SMALL LETTER I WITH GRAVE;Ll;0;L;0069 0300;;;;N;LATIN SMALL LETTER I GRAVE;;00CC;;00CC +00ED;LATIN SMALL LETTER I WITH ACUTE;Ll;0;L;0069 0301;;;;N;LATIN SMALL LETTER I ACUTE;;00CD;;00CD +00EE;LATIN SMALL LETTER I WITH CIRCUMFLEX;Ll;0;L;0069 0302;;;;N;LATIN SMALL LETTER I CIRCUMFLEX;;00CE;;00CE +00EF;LATIN SMALL LETTER I WITH DIAERESIS;Ll;0;L;0069 0308;;;;N;LATIN SMALL LETTER I DIAERESIS;;00CF;;00CF +00F0;LATIN SMALL LETTER ETH;Ll;0;L;;;;;N;;;00D0;;00D0 +00F1;LATIN SMALL LETTER N WITH TILDE;Ll;0;L;006E 0303;;;;N;LATIN SMALL LETTER N TILDE;;00D1;;00D1 +00F2;LATIN SMALL LETTER O WITH GRAVE;Ll;0;L;006F 0300;;;;N;LATIN SMALL LETTER O GRAVE;;00D2;;00D2 +00F3;LATIN SMALL LETTER O WITH ACUTE;Ll;0;L;006F 0301;;;;N;LATIN SMALL LETTER O ACUTE;;00D3;;00D3 +00F4;LATIN SMALL LETTER O WITH CIRCUMFLEX;Ll;0;L;006F 0302;;;;N;LATIN SMALL LETTER O CIRCUMFLEX;;00D4;;00D4 +00F5;LATIN SMALL LETTER O WITH TILDE;Ll;0;L;006F 0303;;;;N;LATIN SMALL LETTER O TILDE;;00D5;;00D5 +00F6;LATIN SMALL LETTER O WITH DIAERESIS;Ll;0;L;006F 0308;;;;N;LATIN SMALL LETTER O DIAERESIS;;00D6;;00D6 +00F7;DIVISION SIGN;Sm;0;ON;;;;;N;;;;; +00F8;LATIN SMALL LETTER O WITH STROKE;Ll;0;L;;;;;N;LATIN SMALL LETTER O SLASH;;00D8;;00D8 +00F9;LATIN SMALL LETTER U WITH GRAVE;Ll;0;L;0075 0300;;;;N;LATIN SMALL LETTER U GRAVE;;00D9;;00D9 +00FA;LATIN SMALL LETTER U WITH ACUTE;Ll;0;L;0075 0301;;;;N;LATIN SMALL LETTER U ACUTE;;00DA;;00DA +00FB;LATIN SMALL LETTER U WITH CIRCUMFLEX;Ll;0;L;0075 0302;;;;N;LATIN SMALL LETTER U CIRCUMFLEX;;00DB;;00DB +00FC;LATIN SMALL LETTER U WITH DIAERESIS;Ll;0;L;0075 0308;;;;N;LATIN SMALL LETTER U DIAERESIS;;00DC;;00DC +00FD;LATIN SMALL LETTER Y WITH ACUTE;Ll;0;L;0079 0301;;;;N;LATIN SMALL LETTER Y ACUTE;;00DD;;00DD +00FE;LATIN SMALL LETTER THORN;Ll;0;L;;;;;N;;;00DE;;00DE +00FF;LATIN SMALL LETTER Y WITH DIAERESIS;Ll;0;L;0079 0308;;;;N;LATIN SMALL LETTER Y DIAERESIS;;0178;;0178 From 5691f5e344bdf85f5ac7d502ea3ba0999b85a42d Mon Sep 17 00:00:00 2001 From: Sami Vaarala Date: Fri, 26 Aug 2016 03:58:11 +0300 Subject: [PATCH 4/9] Python tool cleanups and improvements Various fixes, verbosity reduction, etc. --- Makefile | 6 +- .../client-simple-node/run_commit_test.py | 1649 +++++++++-------- tools/combine_src.py | 25 +- tools/extract_caseconv.py | 28 +- tools/extract_chars.py | 13 +- tools/genbuildparams.py | 44 - tools/genbuiltins.py | 34 +- tools/genconfig.py | 69 +- tools/prepare_sources.py | 112 +- util/bluebird-test-shim.js | 2 +- util/check_align.c | 2 +- util/check_code_policy.py | 2 +- util/example_rombuild.sh | 2 +- util/example_user_builtins1.yaml | 31 +- util/fastint_reps.py | 2 +- util/format_perftest.py | 1 + util/gendoubleconsts.py | 9 +- util/genhashsizes.py | 1 + util/gennumdigits.py | 1 + util/make_dist.py | 1072 ++++++----- util/prep_test.py | 2 +- util/underscore_test.sh | 1 + 22 files changed, 1628 insertions(+), 1480 deletions(-) delete mode 100644 tools/genbuildparams.py diff --git a/Makefile b/Makefile index abd7f1f2..ef25cb68 100644 --- a/Makefile +++ b/Makefile @@ -342,7 +342,7 @@ clean: @rm -f duktape-*.tar.* @rm -f duktape-*.iso @rm -f doc/*.html - @rm -f src/*.pyc + @rm -f src/*.pyc tools/*.pyc util/*.pyc @rm -rf massif.out.* ms_print.tmp.* @rm -rf cachegrind.out.* @rm -rf callgrind.out.* @@ -488,7 +488,7 @@ endif .PHONY: duksizes duksizes: duk.raw - $(PYTHON) src/genexesizereport.py $< > /tmp/duk_sizes.html + $(PYTHON) tools/genexesizereport.py $< > /tmp/duk_sizes.html .PHONY: issuecount issuecount: @@ -1154,7 +1154,7 @@ codepolicycheck: --check-mixed-indent \ --check-tab-indent \ --dump-vim-commands \ - src/*.py tools/*.py util/*.py debugger/*/*.py examples/*/*.py + src/*.py tools/*.py util/*.py debugger/*/*.py examples/*/*.py testrunner/*/*.py @$(PYTHON) util/check_code_policy.py \ $(CODEPOLICYOPTS) \ --check-debug-log-calls \ diff --git a/testrunner/client-simple-node/run_commit_test.py b/testrunner/client-simple-node/run_commit_test.py index 72961bc9..bc79389b 100644 --- a/testrunner/client-simple-node/run_commit_test.py +++ b/testrunner/client-simple-node/run_commit_test.py @@ -39,7 +39,7 @@ import md5 # Whitelisted repos, limit to main repo for now. repo_whitelist = [ - 'svaarala/duktape' + 'svaarala/duktape' ] # Strict reponame filter. @@ -73,551 +73,556 @@ assert(repo_snapshot_dir is not None) # def newenv(**kw): - ret = {} - for k in os.environ.keys(): - ret[k] = str(os.environ[k]) + ret = {} + for k in os.environ.keys(): + ret[k] = str(os.environ[k]) - for k in kw.keys(): - ret[k] = str(kw[k]) + for k in kw.keys(): + ret[k] = str(kw[k]) - #print('Final environment: %r' % ret) - return ret + #print('Final environment: %r' % ret) + return ret def execute(cmd, env=None, catch=False, input='', dump_stdout=True, dump_stderr=True): - print(' - ' + repr(cmd)) - - success = True - - def dump(x): - if isinstance(x, unicode): - x = x.encode('utf-8') - if len(x) == 0 or x[-1] != '\n': - x = x + '\n' - sys.stdout.write(x) - - proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) - ret = proc.communicate(input=input) - if ret[0] != '' and dump_stdout: - dump(ret[0]) - if ret[1] != '' and dump_stderr: - dump(ret[1]) - if proc.returncode != 0: - if catch: - success = False - else: - raise Exception('command failed: %r' % cmd) - - return { - 'returncode': proc.returncode, - 'stdout': ret[0], - 'stderr': ret[1], - 'success': success - } + print(' - ' + repr(cmd)) + + success = True + + def dump(x): + if isinstance(x, unicode): + x = x.encode('utf-8') + if len(x) == 0 or x[-1] != '\n': + x = x + '\n' + sys.stdout.write(x) + + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + ret = proc.communicate(input=input) + if ret[0] != '' and dump_stdout: + dump(ret[0]) + if ret[1] != '' and dump_stderr: + dump(ret[1]) + if proc.returncode != 0: + if catch: + success = False + else: + raise Exception('command failed: %r' % cmd) + + return { + 'returncode': proc.returncode, + 'stdout': ret[0], + 'stderr': ret[1], + 'success': success + } def unpack_targz(fn): - print('Extracting %s to %s' % (fn, os.getcwd())) - t = tarfile.open(fn) - t.extractall() - t.close() + print('Extracting %s to %s' % (fn, os.getcwd())) + t = tarfile.open(fn) + t.extractall() + t.close() def unpack_zip(fn): - print('Extracting %s to %s' % (fn, os.getcwd())) - z = zipfile.ZipFile(fn, 'r') - z.extractall() - z.close() + print('Extracting %s to %s' % (fn, os.getcwd())) + z = zipfile.ZipFile(fn, 'r') + z.extractall() + z.close() def get_binary_size(fn): - # Pattern works for Linux and OS X. - res = execute([ 'size', fn ]) - m = re.compile(r'.*?^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\d+).*?', re.MULTILINE | re.DOTALL).match(res['stdout']) - if m is None: - raise Exception('cannot figure out size for binary %r' % fn) - return { - 'text': int(m.group(1)), - 'data': int(m.group(2)), - 'bss': int(m.group(3)), - 'total': int(m.group(4)) - } + # Pattern works for Linux and OS X. + res = execute([ 'size', fn ]) + m = re.compile(r'.*?^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\d+).*?', re.MULTILINE | re.DOTALL).match(res['stdout']) + if m is None: + raise Exception('cannot figure out size for binary %r' % fn) + return { + 'text': int(m.group(1)), + 'data': int(m.group(2)), + 'bss': int(m.group(3)), + 'total': int(m.group(4)) + } def format_size_diff(newsz, oldsz): - return '%d %d %d (%d %d %d): %d' % ( - newsz['text'], newsz['data'], newsz['bss'], - oldsz['text'], oldsz['data'], oldsz['bss'], - newsz['total'] - oldsz['total'] - ) + return '%d %d %d (%d %d %d): %d' % ( + newsz['text'], newsz['data'], newsz['bss'], + oldsz['text'], oldsz['data'], oldsz['bss'], + newsz['total'] - oldsz['total'] + ) output_description = None def set_output_description(desc): - global output_description - output_description = desc + global output_description + output_description = desc # # Test context handlers # def genconfig_dist_src(genconfig_opts): - cwd = os.getcwd() - execute([ - 'python2', os.path.join(cwd, 'tools', 'genconfig.py'), - '--metadata', os.path.join(cwd, 'config'), - '--output', os.path.join(cwd, 'dist', 'src', 'duk_config.h'), - '--support-feature-options' - ] + genconfig_opts + [ - 'duk-config-header' - ]) + cwd = os.getcwd() + execute([ + 'python2', os.path.join(cwd, 'tools', 'genconfig.py'), + '--metadata', os.path.join(cwd, 'config'), + '--output', os.path.join(cwd, 'dist', 'src', 'duk_config.h'), + '--support-feature-options' + ] + genconfig_opts + [ + 'duk-config-header' + ]) def context_codepolicycheck(): - return execute([ 'make', 'codepolicycheck' ], env=newenv(TRAVIS=1), catch=True)['success'] + return execute([ 'make', 'codepolicycheck' ], env=newenv(TRAVIS=1), catch=True)['success'] def context_helper_x64_qecmatest(env=None, genconfig_opts=[], valgrind=False): - cwd = os.getcwd() - execute([ 'make', 'dist' ]) - genconfig_dist_src(genconfig_opts) - execute([ 'make', 'duk', 'runtestsdeps' ]) - opts = [] - if valgrind: - opts.append('--valgrind') - return execute([ - 'node', - os.path.join(cwd, 'runtests', 'runtests.js'), - '--prep-test-path', os.path.join(cwd, 'util', 'prep_test.py'), - '--minify-uglifyjs2', os.path.join(cwd, 'UglifyJS2', 'bin', 'uglifyjs'), - '--util-include-path', os.path.join(cwd, 'tests', 'ecmascript'), - '--known-issues', os.path.join(cwd, 'doc', 'testcase-known-issues.yaml'), - '--run-duk', '--cmd-duk', os.path.join(cwd, 'duk'), - '--num-threads', '1', - '--log-file', os.path.join(cwd, 'test.out') - ] + opts + [ - os.path.join(cwd, 'tests', 'ecmascript') - ], env=env, catch=True)['success'] + cwd = os.getcwd() + execute([ 'make', 'dist' ]) + genconfig_dist_src(genconfig_opts) + execute([ 'make', 'duk', 'runtestsdeps' ]) + opts = [] + if valgrind: + opts.append('--valgrind') + return execute([ + 'node', + os.path.join(cwd, 'runtests', 'runtests.js'), + '--prep-test-path', os.path.join(cwd, 'util', 'prep_test.py'), + '--minify-uglifyjs2', os.path.join(cwd, 'UglifyJS2', 'bin', 'uglifyjs'), + '--util-include-path', os.path.join(cwd, 'tests', 'ecmascript'), + '--known-issues', os.path.join(cwd, 'doc', 'testcase-known-issues.yaml'), + '--run-duk', '--cmd-duk', os.path.join(cwd, 'duk'), + '--num-threads', '1', + '--log-file', os.path.join(cwd, 'test.out') + ] + opts + [ + os.path.join(cwd, 'tests', 'ecmascript') + ], env=env, catch=True)['success'] def context_linux_x64_qecmatest(): - return context_helper_x64_qecmatest(env=newenv()) + return context_helper_x64_qecmatest(env=newenv()) def context_linux_x64_qecmatest_assert(): - return context_helper_x64_qecmatest(env=newenv(), genconfig_opts=[ '-DDUK_USE_ASSERTIONS' ]) + return context_helper_x64_qecmatest(env=newenv(), genconfig_opts=[ '-DDUK_USE_ASSERTIONS' ]) def context_linux_x64_qecmatest_valgrind(): - return context_helper_x64_qecmatest(env=newenv(), valgrind=True) + return context_helper_x64_qecmatest(env=newenv(), valgrind=True) def context_helper_x64_apitest(env=None, genconfig_opts=[], valgrind=False): - cwd = os.getcwd() - execute([ 'make', 'dist' ]) - genconfig_dist_src(genconfig_opts) - execute([ 'make', 'apiprep' ]) - opts = [] - if valgrind: - opts.append('--valgrind') - return execute([ - 'node', - os.path.join(cwd, 'runtests', 'runtests.js'), - '--prep-test-path', os.path.join(cwd, 'util', 'prep_test.py'), - '--minify-uglifyjs2', os.path.join(cwd, 'UglifyJS2', 'bin', 'uglifyjs'), - '--util-include-path', os.path.join(cwd, 'tests', 'ecmascript'), - '--known-issues', os.path.join(cwd, 'doc', 'testcase-known-issues.yaml'), - '--run-duk', '--cmd-duk', os.path.join(cwd, 'duk'), - '--num-threads', '1', - '--log-file', os.path.join(cwd, 'test.out'), - os.path.join(cwd, 'tests', 'api') - ] + opts + [ - ], env=env, catch=True)['success'] + cwd = os.getcwd() + execute([ 'make', 'dist' ]) + genconfig_dist_src(genconfig_opts) + execute([ 'make', 'apiprep' ]) + opts = [] + if valgrind: + opts.append('--valgrind') + return execute([ + 'node', + os.path.join(cwd, 'runtests', 'runtests.js'), + '--prep-test-path', os.path.join(cwd, 'util', 'prep_test.py'), + '--minify-uglifyjs2', os.path.join(cwd, 'UglifyJS2', 'bin', 'uglifyjs'), + '--util-include-path', os.path.join(cwd, 'tests', 'ecmascript'), + '--known-issues', os.path.join(cwd, 'doc', 'testcase-known-issues.yaml'), + '--run-duk', '--cmd-duk', os.path.join(cwd, 'duk'), + '--num-threads', '1', + '--log-file', os.path.join(cwd, 'test.out'), + os.path.join(cwd, 'tests', 'api') + ] + opts + [ + ], env=env, catch=True)['success'] def context_linux_x64_apitest(): - return context_helper_x64_apitest(env=newenv()) + return context_helper_x64_apitest(env=newenv()) def context_linux_x64_apitest_assert(): - return context_helper_x64_apitest(env=newenv(), genconfig_opts=[ '-DDUK_USE_ASSERTIONS' ]) + return context_helper_x64_apitest(env=newenv(), genconfig_opts=[ '-DDUK_USE_ASSERTIONS' ]) def context_linux_x64_apitest_valgrind(): - return context_helper_x64_apitest(env=newenv(), valgrind=True) + return context_helper_x64_apitest(env=newenv(), valgrind=True) def context_linux_x64_v8_bench_pass(): - cwd = os.getcwd() + cwd = os.getcwd() - print('NOTE: This performance test is executed as a functional') - print('test because it also stress GC etc; the benchmark score') - print('is meaningless unless executed on dedicated hardware.') - print('') + print('NOTE: This performance test is executed as a functional') + print('test because it also stress GC etc; the benchmark score') + print('is meaningless unless executed on dedicated hardware.') + print('') - unpack_targz(os.path.join(repo_snapshot_dir, 'google-v8-benchmark-v7.tar.gz')) - execute([ 'make', 'duk' ]) - os.chdir(os.path.join(cwd, 'tests', 'google-v8-benchmark-v7')) - execute([ 'make', 'combined.js' ]) - os.chdir(cwd) - execute([ os.path.join(cwd, 'duk'), os.path.join('tests', 'google-v8-benchmark-v7', 'combined.js') ]) - return True + unpack_targz(os.path.join(repo_snapshot_dir, 'google-v8-benchmark-v7.tar.gz')) + execute([ 'make', 'duk' ]) + os.chdir(os.path.join(cwd, 'tests', 'google-v8-benchmark-v7')) + execute([ 'make', 'combined.js' ]) + os.chdir(cwd) + execute([ os.path.join(cwd, 'duk'), os.path.join('tests', 'google-v8-benchmark-v7', 'combined.js') ]) + return True def context_linux_x64_duk_clang(): - cwd = os.getcwd() - execute([ 'make', 'duk-clang' ]) - res = execute([ - os.path.join(cwd, 'duk-clang'), - '-e', 'print("hello world!");' - ]) - return res['stdout'] == 'hello world!\n' + cwd = os.getcwd() + execute([ 'make', 'duk-clang' ]) + res = execute([ + os.path.join(cwd, 'duk-clang'), + '-e', 'print("hello world!");' + ]) + return res['stdout'] == 'hello world!\n' def context_linux_x64_duk_gxx(): - cwd = os.getcwd() - execute([ 'make', 'duk-g++' ]) - res = execute([ - os.path.join(cwd, 'duk-g++'), - '-e', 'print("hello world!");' - ]) - return res['stdout'] == 'hello world!\n' + cwd = os.getcwd() + execute([ 'make', 'duk-g++' ]) + res = execute([ + os.path.join(cwd, 'duk-g++'), + '-e', 'print("hello world!");' + ]) + return res['stdout'] == 'hello world!\n' def context_helper_get_binary_size_diff(compfn): - cwd = os.getcwd() - execute([ 'git', 'clean', '-f' ]) - execute([ 'git', 'reset', '--hard' ]) - compfn() - newsz = get_binary_size(os.path.join(cwd, 'duk')) - execute([ 'git', 'clean', '-f' ]) - execute([ 'git', 'reset', '--hard' ]) - execute([ 'git', 'checkout', 'master' ]) - execute([ 'git', 'clean', '-f' ]) - execute([ 'git', 'reset', '--hard' ]) - execute([ 'make', 'clean' ]) - compfn() - oldsz = get_binary_size(os.path.join(cwd, 'duk')) - set_output_description(format_size_diff(newsz, oldsz)) - return True + cwd = os.getcwd() + execute([ 'git', 'clean', '-f' ]) + execute([ 'git', 'reset', '--hard' ]) + compfn() + newsz = get_binary_size(os.path.join(cwd, 'duk')) + execute([ 'git', 'clean', '-f' ]) + execute([ 'git', 'reset', '--hard' ]) + execute([ 'git', 'checkout', 'master' ]) + execute([ 'git', 'clean', '-f' ]) + execute([ 'git', 'reset', '--hard' ]) + execute([ 'make', 'clean' ]) + compfn() + oldsz = get_binary_size(os.path.join(cwd, 'duk')) + set_output_description(format_size_diff(newsz, oldsz)) + return True def context_linux_x64_gcc_defsize_makeduk(): - cwd = os.getcwd() - def comp(): - execute([ 'make', 'duk' ]) - return context_helper_get_binary_size_diff(comp) + cwd = os.getcwd() + def comp(): + execute([ 'make', 'duk' ]) + return context_helper_get_binary_size_diff(comp) def context_linux_x64_gcc_defsize_fltoetc(): - cwd = os.getcwd() - def comp(): - execute([ 'make', 'dist' ]) - execute([ - 'gcc', '-oduk', - '-Os', '-fomit-frame-pointer', - '-flto', '-fno-asynchronous-unwind-tables', - '-ffunction-sections', '-Wl,--gc-sections', - '-I' + os.path.join('dist', 'src'), - '-I' + os.path.join('dist', 'examples', 'cmdline'), - os.path.join(cwd, 'dist', 'src', 'duktape.c'), - os.path.join(cwd, 'dist', 'examples', 'cmdline', 'duk_cmdline.c'), - '-lm' - ]) - return context_helper_get_binary_size_diff(comp) + cwd = os.getcwd() + def comp(): + execute([ 'make', 'dist' ]) + execute([ + 'gcc', '-oduk', + '-Os', '-fomit-frame-pointer', + '-flto', '-fno-asynchronous-unwind-tables', + '-ffunction-sections', '-Wl,--gc-sections', + '-I' + os.path.join('dist', 'src'), + '-I' + os.path.join('dist', 'examples', 'cmdline'), + os.path.join(cwd, 'dist', 'src', 'duktape.c'), + os.path.join(cwd, 'dist', 'examples', 'cmdline', 'duk_cmdline.c'), + '-lm' + ]) + return context_helper_get_binary_size_diff(comp) def context_helper_minsize_fltoetc(archopt, strip): - cwd = os.getcwd() - def comp(): - execute([ 'make', 'dist' ]) - cmd = [ - 'python2', os.path.join(cwd, 'tools', 'genconfig.py'), - '--metadata', os.path.join(cwd, 'config'), - '--output', os.path.join(cwd, 'dist', 'src', 'duk_config.h'), - '--option-file', os.path.join(cwd, 'config', 'examples', 'low_memory.yaml') - ] - if strip: - cmd += [ - '--option-file', os.path.join(cwd, 'config', 'examples', 'low_memory_strip.yaml') - ] - cmd += [ - 'duk-config-header' - ] - execute(cmd) - execute([ - 'gcc', '-oduk', archopt, - '-Os', '-fomit-frame-pointer', - '-flto', '-fno-asynchronous-unwind-tables', - '-ffunction-sections', '-Wl,--gc-sections', - '-I' + os.path.join('dist', 'src'), - '-I' + os.path.join('dist', 'examples', 'cmdline'), - os.path.join(cwd, 'dist', 'src', 'duktape.c'), - os.path.join(cwd, 'dist', 'examples', 'cmdline', 'duk_cmdline.c'), - '-lm' - ]) - res = execute([ - os.path.join(cwd, 'duk') - ], input='1+2, "hello world!"') - return 'hello world' in res['stdout'] - return context_helper_get_binary_size_diff(comp) + cwd = os.getcwd() + def comp(): + execute([ 'make', 'dist' ]) + + execute([ 'rm', '-rf', os.path.join(cwd, 'dist', 'src') ]) + execute([ 'rm', '-rf', os.path.join(cwd, 'dist', 'src-noline') ]) + execute([ 'rm', '-rf', os.path.join(cwd, 'dist', 'src-separate') ]) + + cmd = [ + 'python2', os.path.join(cwd, 'dist', 'tools', 'prepare_sources.py'), + '--source-directory', os.path.join(cwd, 'dist', 'src-input'), + '--output-directory', os.path.join(cwd, 'dist'), + '--config-metadata', os.path.join(cwd, 'dist', 'config', 'genconfig_metadata.tar.gz'), + '--option-file', os.path.join(cwd, 'config', 'examples', 'low_memory.yaml') + ] + if strip: + cmd += [ + '--option-file', os.path.join(cwd, 'config', 'examples', 'low_memory_strip.yaml'), + '--unicode-data', os.path.join(cwd, 'dist', 'src-input', 'UnicodeData-8bit.txt'), + '--special-casing', os.path.join(cwd, 'dist', 'src-input', 'SpecialCasing-8bit.txt') + ] + execute(cmd) + execute([ + 'gcc', '-oduk', archopt, + '-Os', '-fomit-frame-pointer', + '-flto', '-fno-asynchronous-unwind-tables', + '-ffunction-sections', '-Wl,--gc-sections', + '-I' + os.path.join('dist', 'src'), + '-I' + os.path.join('dist', 'examples', 'cmdline'), + os.path.join(cwd, 'dist', 'src', 'duktape.c'), + os.path.join(cwd, 'dist', 'examples', 'cmdline', 'duk_cmdline.c'), + '-lm' + ]) + res = execute([ + os.path.join(cwd, 'duk') + ], input='1+2, "hello world!"') + return 'hello world' in res['stdout'] + return context_helper_get_binary_size_diff(comp) def context_linux_x64_gcc_minsize_fltoetc(): - return context_helper_minsize_fltoetc('-m64', False) + return context_helper_minsize_fltoetc('-m64', False) def context_linux_x86_gcc_minsize_fltoetc(): - return context_helper_minsize_fltoetc('-m32', False) + return context_helper_minsize_fltoetc('-m32', False) def context_linux_x32_gcc_minsize_fltoetc(): - return context_helper_minsize_fltoetc('-mx32', False) + return context_helper_minsize_fltoetc('-mx32', False) def context_linux_x64_gcc_stripsize_fltoetc(): - return context_helper_minsize_fltoetc('-m64', True) + return context_helper_minsize_fltoetc('-m64', True) def context_linux_x86_gcc_stripsize_fltoetc(): - return context_helper_minsize_fltoetc('-m32', True) + return context_helper_minsize_fltoetc('-m32', True) def context_linux_x32_gcc_stripsize_fltoetc(): - return context_helper_minsize_fltoetc('-mx32', True) + return context_helper_minsize_fltoetc('-mx32', True) def context_linux_x64_cpp_exceptions(): - # For now rather simple: compile, run, and grep for my_class - # destruction prints. There are only 3 without C++ exceptions - # and 15 with them. - - cwd = os.getcwd() - - execute([ 'make', 'dist' ]) - execute([ - 'g++', '-oduk-cpp-exc', - '-DDUK_OPT_CPP_EXCEPTIONS', - '-I' + os.path.join(cwd, 'dist', 'src'), - os.path.join(cwd, 'dist', 'src', 'duktape.c'), - os.path.join(cwd, 'examples', 'cpp-exceptions', 'cpp_exceptions.cpp'), - '-lm' - ]) - - res = execute([ - os.path.join(cwd, 'duk-cpp-exc') - ]) - count = 0 - for line in res['stdout'].split('\n'): - if 'my_class instance destroyed' in line: - count += 1 - print('Destruct count: %d' % count) - - if count >= 15: - print('C++ exceptions seem to be working') - return True - else: - print('C++ exceptions don\'t seem to be working') - return False + # For now rather simple: compile, run, and grep for my_class + # destruction prints. There are only 3 without C++ exceptions + # and 15 with them. + + cwd = os.getcwd() + + execute([ 'make', 'dist' ]) + execute([ + 'g++', '-oduk-cpp-exc', + '-DDUK_OPT_CPP_EXCEPTIONS', + '-I' + os.path.join(cwd, 'dist', 'src'), + os.path.join(cwd, 'dist', 'src', 'duktape.c'), + os.path.join(cwd, 'examples', 'cpp-exceptions', 'cpp_exceptions.cpp'), + '-lm' + ]) + + res = execute([ + os.path.join(cwd, 'duk-cpp-exc') + ]) + count = 0 + for line in res['stdout'].split('\n'): + if 'my_class instance destroyed' in line: + count += 1 + print('Destruct count: %d' % count) + + if count >= 15: + print('C++ exceptions seem to be working') + return True + else: + print('C++ exceptions don\'t seem to be working') + return False def context_linux_x86_ajduk(): - cwd = os.getcwd() - execute([ 'make', 'ajduk' ]) - res = execute([ - os.path.join(cwd, 'ajduk'), - '-e', 'print("hello world!");' - ]) - return 'hello world!\n' in res['stdout'] + cwd = os.getcwd() + execute([ 'make', 'ajduk' ]) + res = execute([ + os.path.join(cwd, 'ajduk'), + '-e', 'print("hello world!");' + ]) + return 'hello world!\n' in res['stdout'] def context_linux_x86_ajduk_rombuild(): - cwd = os.getcwd() + cwd = os.getcwd() - execute([ - 'bash', - os.path.join(cwd, 'util', 'example_rombuild.sh'), - ]) + execute([ + 'bash', + os.path.join(cwd, 'util', 'example_rombuild.sh'), + ]) - got_hello = False - got_startrek = False + got_hello = False + got_startrek = False - res = execute([ - os.path.join(cwd, 'ajduk'), - '-e', 'print("hello world!");' - ]) - got_hello = ('hello world!\n' in res['stdout']) # ajduk stdout has pool dumps etc - print('Got hello: %r' % got_hello) + res = execute([ + os.path.join(cwd, 'ajduk'), + '-e', 'print("hello world!");' + ]) + got_hello = ('hello world!\n' in res['stdout']) # ajduk stdout has pool dumps etc + print('Got hello: %r' % got_hello) - res = execute([ - os.path.join(cwd, 'ajduk'), - '-e', 'print("StarTrek.ent:", StarTrek.ent);' - ]) - got_startrek = ('StarTrek.ent: true\n' in res['stdout']) - print('Got StarTrek: %r' % got_startrek) + res = execute([ + os.path.join(cwd, 'ajduk'), + '-e', 'print("StarTrek.ent:", StarTrek.ent);' + ]) + got_startrek = ('StarTrek.ent: true\n' in res['stdout']) + print('Got StarTrek: %r' % got_startrek) - return got_hello and got_startrek + return got_hello and got_startrek def context_linux_x64_test262test(): - cwd = os.getcwd() - - execute([ 'make', 'duk' ]) - - # Unpack separately, 'make clean' wipes this. - unpack_targz(os.path.join(repo_snapshot_dir, 'test262-es5-tests.tar.gz')) - unpack_zip(os.path.join(cwd, 'es5-tests.zip')) - - os.chdir(os.path.join(cwd, 'test262-es5-tests')) - res = execute([ - 'python2', - os.path.join(cwd, 'test262-es5-tests', 'tools', 'packaging', 'test262.py'), - '--command', os.path.join(cwd, 'duk') + ' {{path}}' - ], dump_stdout=False, dump_stderr=True) - test262_log = res['stdout'] - os.chdir(cwd) - - res = execute([ - 'python2', - os.path.join(cwd, 'util', 'filter_test262_log.py'), - os.path.join(cwd, 'doc', 'test262-known-issues.yaml') - ], input=test262_log) - - # Test result plumbing a bit awkward but works for now. - # Known and diagnosed issues are considered a "pass" for - # Github status. - return 'TEST262 SUCCESS\n' in res['stdout'] + cwd = os.getcwd() + + execute([ 'make', 'duk' ]) + + # Unpack separately, 'make clean' wipes this. + unpack_targz(os.path.join(repo_snapshot_dir, 'test262-es5-tests.tar.gz')) + unpack_zip(os.path.join(cwd, 'es5-tests.zip')) + + os.chdir(os.path.join(cwd, 'test262-es5-tests')) + res = execute([ + 'python2', + os.path.join(cwd, 'test262-es5-tests', 'tools', 'packaging', 'test262.py'), + '--command', os.path.join(cwd, 'duk') + ' {{path}}' + ], dump_stdout=False, dump_stderr=True) + test262_log = res['stdout'] + os.chdir(cwd) + + res = execute([ + 'python2', + os.path.join(cwd, 'util', 'filter_test262_log.py'), + os.path.join(cwd, 'doc', 'test262-known-issues.yaml') + ], input=test262_log) + + # Test result plumbing a bit awkward but works for now. + # Known and diagnosed issues are considered a "pass" for + # Github status. + return 'TEST262 SUCCESS\n' in res['stdout'] def context_linux_x64_duk_dddprint(): - cwd = os.getcwd() - - execute([ 'make', 'dist' ]) - os.chdir(os.path.join(cwd, 'dist')) - - res = execute([ - 'gcc', '-oduk', - '-DDUK_OPT_ASSERTIONS', '-DDUK_OPT_SELF_TESTS', - '-DDUK_OPT_DEBUG', '-DDUK_OPT_DPRINT', '-DDUK_OPT_DDPRINT', '-DDUK_OPT_DDDPRINT', - '-DDUK_OPT_DEBUG_WRITE(level,file,line,func,msg)=do {fprintf(stderr, "%ld %s:%ld (%s): %s\\n", (long) (level), (file), (long) (line), (func), (msg));} while(0)', - '-DDUK_CMDLINE_PRINTALERT_SUPPORT', - '-I' + os.path.join(cwd, 'dist', 'src'), - '-I' + os.path.join(cwd, 'dist', 'extras', 'print-alert'), - os.path.join(cwd, 'dist', 'src', 'duktape.c'), - os.path.join(cwd, 'dist', 'examples', 'cmdline', 'duk_cmdline.c'), - os.path.join(cwd, 'dist', 'extras', 'print-alert', 'duk_print_alert.c'), - '-lm' - ], catch=True) - if not res['success']: - print('Compilation failed.') - return False - - res = execute([ - os.path.join(cwd, 'dist', 'duk'), - '-e', 'print("Hello world!");' - ], dump_stderr=False) - - return 'Hello world!\n' in res['stdout'] + cwd = os.getcwd() + + execute([ 'make', 'dist' ]) + os.chdir(os.path.join(cwd, 'dist')) + + res = execute([ + 'gcc', '-oduk', + '-DDUK_OPT_ASSERTIONS', '-DDUK_OPT_SELF_TESTS', + '-DDUK_OPT_DEBUG', '-DDUK_OPT_DPRINT', '-DDUK_OPT_DDPRINT', '-DDUK_OPT_DDDPRINT', + '-DDUK_OPT_DEBUG_WRITE(level,file,line,func,msg)=do {fprintf(stderr, "%ld %s:%ld (%s): %s\\n", (long) (level), (file), (long) (line), (func), (msg));} while(0)', + '-DDUK_CMDLINE_PRINTALERT_SUPPORT', + '-I' + os.path.join(cwd, 'dist', 'src'), + '-I' + os.path.join(cwd, 'dist', 'extras', 'print-alert'), + os.path.join(cwd, 'dist', 'src', 'duktape.c'), + os.path.join(cwd, 'dist', 'examples', 'cmdline', 'duk_cmdline.c'), + os.path.join(cwd, 'dist', 'extras', 'print-alert', 'duk_print_alert.c'), + '-lm' + ], catch=True) + if not res['success']: + print('Compilation failed.') + return False + + res = execute([ + os.path.join(cwd, 'dist', 'duk'), + '-e', 'print("Hello world!");' + ], dump_stderr=False) + + return 'Hello world!\n' in res['stdout'] def context_linux_x64_duk_separate_src(): - cwd = os.getcwd() - - execute([ 'make', 'dist' ]) - os.chdir(os.path.join(cwd, 'dist')) - - cfiles =[] - for fn in os.listdir(os.path.join(cwd, 'dist', 'src-separate')): - if fn[-2:] == '.c': - cfiles.append(os.path.join(cwd, 'dist', 'src-separate', fn)) - cfiles.append(os.path.join(cwd, 'dist', 'examples', 'cmdline', 'duk_cmdline.c')) - cfiles.append(os.path.join(cwd, 'dist', 'extras', 'print-alert', 'duk_print_alert.c')) - - execute([ - 'gcc', '-oduk', - '-DDUK_CMDLINE_PRINTALERT_SUPPORT', - '-I' + os.path.join(cwd, 'dist', 'src-separate'), - '-I' + os.path.join(cwd, 'dist', 'extras', 'print-alert') - ] + cfiles + [ - '-lm' - ]) - - res = execute([ - os.path.join(cwd, 'dist', 'duk'), - '-e', 'print("Hello world!");' - ]) - - return 'Hello world!\n' in res['stdout'] + cwd = os.getcwd() + + execute([ 'make', 'dist' ]) + os.chdir(os.path.join(cwd, 'dist')) + + cfiles =[] + for fn in os.listdir(os.path.join(cwd, 'dist', 'src-separate')): + if fn[-2:] == '.c': + cfiles.append(os.path.join(cwd, 'dist', 'src-separate', fn)) + cfiles.append(os.path.join(cwd, 'dist', 'examples', 'cmdline', 'duk_cmdline.c')) + cfiles.append(os.path.join(cwd, 'dist', 'extras', 'print-alert', 'duk_print_alert.c')) + + execute([ + 'gcc', '-oduk', + '-DDUK_CMDLINE_PRINTALERT_SUPPORT', + '-I' + os.path.join(cwd, 'dist', 'src-separate'), + '-I' + os.path.join(cwd, 'dist', 'extras', 'print-alert') + ] + cfiles + [ + '-lm' + ]) + + res = execute([ + os.path.join(cwd, 'dist', 'duk'), + '-e', 'print("Hello world!");' + ]) + + return 'Hello world!\n' in res['stdout'] def context_linux_x86_packed_tval(): - cwd = os.getcwd() - - execute([ 'make', 'dist' ]) - os.chdir(os.path.join(cwd, 'dist')) - - execute([ - 'gcc', '-oduk', '-m32', - '-DDUK_CMDLINE_PRINTALERT_SUPPORT', - '-I' + os.path.join(cwd, 'dist', 'src'), - '-I' + os.path.join(cwd, 'dist', 'extras', 'print-alert'), - os.path.join(cwd, 'dist', 'src', 'duktape.c'), - os.path.join(cwd, 'dist', 'examples', 'cmdline', 'duk_cmdline.c'), - os.path.join(cwd, 'dist', 'extras', 'print-alert', 'duk_print_alert.c'), - '-lm' - ]) - - # Size of a 3-element array is 25 + 3x16 = 73 on x64 and - # 13 + 3x8 = 37 on x86. - res = execute([ - os.path.join(cwd, 'dist', 'duk'), - '-e', - 'var arr = Duktape.compact([1,2,3]); ' + - 'print(Duktape.info(true)[1] >= 0xf000); ' + # packed internal tag - 'print(Duktape.info(arr)[4] <= 40)' # array size (1 element + .length property) - ]); - return res['stdout'] == 'true\ntrue\n' + cwd = os.getcwd() + + execute([ 'make', 'dist' ]) + os.chdir(os.path.join(cwd, 'dist')) + + execute([ + 'gcc', '-oduk', '-m32', + '-DDUK_CMDLINE_PRINTALERT_SUPPORT', + '-I' + os.path.join(cwd, 'dist', 'src'), + '-I' + os.path.join(cwd, 'dist', 'extras', 'print-alert'), + os.path.join(cwd, 'dist', 'src', 'duktape.c'), + os.path.join(cwd, 'dist', 'examples', 'cmdline', 'duk_cmdline.c'), + os.path.join(cwd, 'dist', 'extras', 'print-alert', 'duk_print_alert.c'), + '-lm' + ]) + + # Size of a 3-element array is 25 + 3x16 = 73 on x64 and + # 13 + 3x8 = 37 on x86. + res = execute([ + os.path.join(cwd, 'dist', 'duk'), + '-e', + 'var arr = Duktape.compact([1,2,3]); ' + + 'print(Duktape.info(true)[1] >= 0xf000); ' + # packed internal tag + 'print(Duktape.info(arr)[4] <= 40)' # array size (1 element + .length property) + ]); + return res['stdout'] == 'true\ntrue\n' def context_linux_x86_dist_genconfig(): - cwd = os.getcwd() - - execute([ 'make', 'dist' ]) - - os.chdir(os.path.join(cwd, 'dist')) - execute([ - 'python2', os.path.join(cwd, 'dist', 'tools', 'genconfig.py'), - '--metadata', os.path.join(cwd, 'dist', 'config', 'genconfig_metadata.tar.gz'), - '--output', os.path.join(cwd, 'dist', 'src', 'duk_config.h'), # overwrite default duk_config.h - '-DDUK_USE_FASTINT', '-UDUK_USE_JX', '-UDUK_USE_JC', - 'duk-config-header' - ]) - - os.chdir(os.path.join(cwd, 'dist')) - execute([ - 'gcc', '-oduk', - '-DDUK_CMDLINE_PRINTALERT_SUPPORT', - '-I' + os.path.join(cwd, 'dist', 'src'), - '-I' + os.path.join(cwd, 'dist', 'extras', 'print-alert'), - os.path.join(cwd, 'dist', 'src', 'duktape.c'), - os.path.join(cwd, 'dist', 'examples', 'cmdline', 'duk_cmdline.c'), - os.path.join(cwd, 'dist', 'extras', 'print-alert', 'duk_print_alert.c'), - '-lm' - ]) - - res = execute([ - os.path.join(cwd, 'dist', 'duk'), - '-e', 'try { print(Duktape.enc("jx", {})); } catch (e) { print("ERROR: " + e.name); }' - ]) - - return 'ERROR: TypeError\n' in res['stdout'] + cwd = os.getcwd() + + execute([ 'make', 'dist' ]) + + os.chdir(os.path.join(cwd, 'dist')) + execute([ + 'python2', os.path.join(cwd, 'dist', 'tools', 'genconfig.py'), + '--metadata', os.path.join(cwd, 'dist', 'config', 'genconfig_metadata.tar.gz'), + '--output', os.path.join(cwd, 'dist', 'src', 'duk_config.h'), # overwrite default duk_config.h + '-DDUK_USE_FASTINT', '-UDUK_USE_JX', '-UDUK_USE_JC', + 'duk-config-header' + ]) + + os.chdir(os.path.join(cwd, 'dist')) + execute([ + 'gcc', '-oduk', + '-DDUK_CMDLINE_PRINTALERT_SUPPORT', + '-I' + os.path.join(cwd, 'dist', 'src'), + '-I' + os.path.join(cwd, 'dist', 'extras', 'print-alert'), + os.path.join(cwd, 'dist', 'src', 'duktape.c'), + os.path.join(cwd, 'dist', 'examples', 'cmdline', 'duk_cmdline.c'), + os.path.join(cwd, 'dist', 'extras', 'print-alert', 'duk_print_alert.c'), + '-lm' + ]) + + res = execute([ + os.path.join(cwd, 'dist', 'duk'), + '-e', 'try { print(Duktape.enc("jx", {})); } catch (e) { print("ERROR: " + e.name); }' + ]) + + return 'ERROR: TypeError\n' in res['stdout'] def context_linux_x64_error_variants(): - # Test Duktape build using: - # (1) verbose and non-paranoid errors - # (2) verbose and paranoid errors - # (3) non-verbose errors - - cwd = os.getcwd() - - retval = True - - for params in [ - { 'genconfig_opts': [ '-DDUK_USE_VERBOSE_ERRORS', '-UDUK_USE_PARANOID_ERRORS' ], - 'binary_name': 'duk.verbose_nonparanoid' }, - { 'genconfig_opts': [ '-DDUK_USE_VERBOSE_ERRORS', '-DDUK_USE_PARANOID_ERRORS' ], - 'binary_name': 'duk.verbose_paranoid' }, - { 'genconfig_opts': [ '-UDUK_USE_VERBOSE_ERRORS', '-UDUK_USE_PARANOID_ERRORS' ], - 'binary_name': 'duk.nonverbose' }, - ]: - os.chdir(cwd) - execute([ 'make', 'clean', 'dist' ]) - os.chdir(os.path.join(cwd, 'dist')) - execute([ - 'python2', os.path.join(cwd, 'dist', 'tools', 'genconfig.py'), - '--metadata', os.path.join(cwd, 'dist', 'config', 'genconfig_metadata.tar.gz'), - '--output', os.path.join(cwd, 'dist', 'src', 'duk_config.h') # overwrite default duk_config.h - ] + params['genconfig_opts'] + [ - 'duk-config-header' - ]) - execute([ - 'gcc', '-o' + params['binary_name'], - '-DDUK_CMDLINE_PRINTALERT_SUPPORT', - '-I' + os.path.join(cwd, 'dist', 'src'), - '-I' + os.path.join(cwd, 'dist', 'extras', 'print-alert'), - os.path.join(cwd, 'dist', 'src', 'duktape.c'), - os.path.join(cwd, 'dist', 'examples', 'cmdline', 'duk_cmdline.c'), - os.path.join(cwd, 'dist', 'extras', 'print-alert', 'duk_print_alert.c'), - '-lm' - ]) - execute([ 'size', params['binary_name'] ]) - - with open('test.js', 'wb') as f: - f.write("""\ + # Test Duktape build using: + # (1) verbose and non-paranoid errors + # (2) verbose and paranoid errors + # (3) non-verbose errors + + cwd = os.getcwd() + + retval = True + + for params in [ + { 'genconfig_opts': [ '-DDUK_USE_VERBOSE_ERRORS', '-UDUK_USE_PARANOID_ERRORS' ], + 'binary_name': 'duk.verbose_nonparanoid' }, + { 'genconfig_opts': [ '-DDUK_USE_VERBOSE_ERRORS', '-DDUK_USE_PARANOID_ERRORS' ], + 'binary_name': 'duk.verbose_paranoid' }, + { 'genconfig_opts': [ '-UDUK_USE_VERBOSE_ERRORS', '-UDUK_USE_PARANOID_ERRORS' ], + 'binary_name': 'duk.nonverbose' }, + ]: + os.chdir(cwd) + execute([ 'make', 'clean', 'dist' ]) + os.chdir(os.path.join(cwd, 'dist')) + execute([ + 'python2', os.path.join(cwd, 'dist', 'tools', 'genconfig.py'), + '--metadata', os.path.join(cwd, 'dist', 'config', 'genconfig_metadata.tar.gz'), + '--output', os.path.join(cwd, 'dist', 'src', 'duk_config.h') # overwrite default duk_config.h + ] + params['genconfig_opts'] + [ + 'duk-config-header' + ]) + execute([ + 'gcc', '-o' + params['binary_name'], + '-DDUK_CMDLINE_PRINTALERT_SUPPORT', + '-I' + os.path.join(cwd, 'dist', 'src'), + '-I' + os.path.join(cwd, 'dist', 'extras', 'print-alert'), + os.path.join(cwd, 'dist', 'src', 'duktape.c'), + os.path.join(cwd, 'dist', 'examples', 'cmdline', 'duk_cmdline.c'), + os.path.join(cwd, 'dist', 'extras', 'print-alert', 'duk_print_alert.c'), + '-lm' + ]) + execute([ 'size', params['binary_name'] ]) + + with open('test.js', 'wb') as f: + f.write("""\ try { (undefined).foo = 123; } catch (e) { @@ -627,289 +632,298 @@ try { } """) - res = execute([ - os.path.join(cwd, 'dist', params['binary_name']), - 'test.js' - ]) - if 'ERRORNAME: TypeError\n' not in res['stdout']: - print('Cannot find error name in output') - retval = False + res = execute([ + os.path.join(cwd, 'dist', params['binary_name']), + 'test.js' + ]) + if 'ERRORNAME: TypeError\n' not in res['stdout']: + print('Cannot find error name in output') + retval = False - # For now, just check that the code compiles and error Type is - # correct. XXX: add check for error message too. + # For now, just check that the code compiles and error Type is + # correct. XXX: add check for error message too. - return retval + return retval def context_helper_hello_ram(archopt): - cwd = os.getcwd() - - def test(genconfig_opts): - os.chdir(cwd) - execute([ 'make', 'clean' ]) - execute([ - 'python2', os.path.join(cwd, 'util', 'make_dist.py'), - '--rom-support' - ]) - genconfig_dist_src(genconfig_opts) - execute([ - 'gcc', '-ohello', archopt, - '-Os', '-fomit-frame-pointer', - '-flto', '-fno-asynchronous-unwind-tables', - '-ffunction-sections', '-Wl,--gc-sections', - '-I' + os.path.join('dist', 'src'), - os.path.join(cwd, 'dist', 'src', 'duktape.c'), - os.path.join(cwd, 'dist', 'examples', 'hello', 'hello.c'), - '-lm' - ]) - execute([ - 'size', - os.path.join(cwd, 'hello') - ]) - execute([ - 'valgrind', '--tool=massif', - '--massif-out-file=' + os.path.join(cwd, 'massif.out'), - '--peak-inaccuracy=0.0', - os.path.join(cwd, 'hello') - ]) - res = execute([ - 'ms_print', - os.path.join(cwd, 'massif.out') - ], dump_stdout=False) - lines = res['stdout'].split('\n') - print('\n'.join(lines[0:50])) # print 50 first lines only - - # KB - #107.5^ : - # | @#::::@:: :::@::: : : - - kb = '???' - re_kb = re.compile(r'^([0-9\.]+)\^.*?$') - for line in lines[0:10]: - m = re_kb.match(line) - if m is not None: - kb = m.group(1) - print(' --> KB: ' + kb) - return kb - - print('--- Default') - print('') - kb_default = test([]) - - print('') - print('--- No bufferobject support') - print('') - kb_nobufobj = test([ - '-UDUK_USE_BUFFEROBJECT_SUPPORT' - ]) - - print('') - print('--- ROM built-ins, global object inherits from ROM global') - print('--- No other low memory options (fast paths, pointer compression, etc)') - print('') - kb_rom = test([ - '-DDUK_USE_ROM_OBJECTS', - '-DDUK_USE_ROM_STRINGS', - '-DDUK_USE_ROM_GLOBAL_INHERIT' - ]) - - set_output_description('%s %s %s (kB)' % (kb_default, kb_nobufobj, kb_rom)) - - return True + cwd = os.getcwd() + + def test(genconfig_opts): + os.chdir(cwd) + execute([ 'make', 'clean', 'dist' ]) + + execute([ 'rm', '-rf', os.path.join(cwd, 'dist', 'src') ]) + execute([ 'rm', '-rf', os.path.join(cwd, 'dist', 'src-noline') ]) + execute([ 'rm', '-rf', os.path.join(cwd, 'dist', 'src-separate') ]) + + cmd = [ + 'python2', os.path.join(cwd, 'dist', 'tools', 'prepare_sources.py'), + '--source-directory', os.path.join(cwd, 'dist', 'src-input'), + '--output-directory', os.path.join(cwd, 'dist'), + '--config-metadata', os.path.join(cwd, 'dist', 'config', 'genconfig_metadata.tar.gz'), + '--rom-support' + ] + genconfig_opts + print(repr(cmd)) + execute(cmd) + execute([ + 'gcc', '-ohello', archopt, + '-Os', '-fomit-frame-pointer', + '-flto', '-fno-asynchronous-unwind-tables', + '-ffunction-sections', '-Wl,--gc-sections', + '-I' + os.path.join('dist', 'src'), + os.path.join(cwd, 'dist', 'src', 'duktape.c'), + os.path.join(cwd, 'dist', 'examples', 'hello', 'hello.c'), + '-lm' + ]) + execute([ + 'size', + os.path.join(cwd, 'hello') + ]) + execute([ + 'valgrind', '--tool=massif', + '--massif-out-file=' + os.path.join(cwd, 'massif.out'), + '--peak-inaccuracy=0.0', + os.path.join(cwd, 'hello') + ]) + res = execute([ + 'ms_print', + os.path.join(cwd, 'massif.out') + ], dump_stdout=False) + lines = res['stdout'].split('\n') + print('\n'.join(lines[0:50])) # print 50 first lines only + + # KB + #107.5^ : + # | @#::::@:: :::@::: : : + + kb = '???' + re_kb = re.compile(r'^([0-9\.]+)\^.*?$') + for line in lines[0:10]: + m = re_kb.match(line) + if m is not None: + kb = m.group(1) + print(' --> KB: ' + kb) + return kb + + print('--- Default') + print('') + kb_default = test([]) + + print('') + print('--- No bufferobject support') + print('') + kb_nobufobj = test([ + '-UDUK_USE_BUFFEROBJECT_SUPPORT' + ]) + + print('') + print('--- ROM built-ins, global object inherits from ROM global') + print('--- No other low memory options (fast paths, pointer compression, etc)') + print('') + kb_rom = test([ + '-DDUK_USE_ROM_OBJECTS', + '-DDUK_USE_ROM_STRINGS', + '-DDUK_USE_ROM_GLOBAL_INHERIT' + ]) + + set_output_description('%s %s %s (kB)' % (kb_default, kb_nobufobj, kb_rom)) + + return True def context_linux_x64_hello_ram(): - return context_helper_hello_ram('-m64') + return context_helper_hello_ram('-m64') def context_linux_x86_hello_ram(): - return context_helper_hello_ram('-m32') + return context_helper_hello_ram('-m32') def context_linux_x32_hello_ram(): - return context_helper_hello_ram('-mx32') + return context_helper_hello_ram('-mx32') def mandel_test(archopt, genconfig_opts): - cwd = os.getcwd() - execute([ 'make', 'dist' ]) - execute([ - 'python2', os.path.join(cwd, 'tools', 'genconfig.py'), - '--metadata', os.path.join(cwd, 'config'), - '--output', os.path.join(cwd, 'dist', 'src', 'duk_config.h') - ] + genconfig_opts + [ - 'duk-config-header' - ]) - execute([ - 'gcc', '-oduk', archopt, - '-Os', '-fomit-frame-pointer', - '-flto', '-fno-asynchronous-unwind-tables', - '-ffunction-sections', '-Wl,--gc-sections', - '-DDUK_CMDLINE_PRINTALERT_SUPPORT', - '-I' + os.path.join('dist', 'src'), - '-I' + os.path.join(cwd, 'dist', 'extras', 'print-alert'), - '-I' + os.path.join('dist', 'examples', 'cmdline'), - os.path.join(cwd, 'dist', 'src', 'duktape.c'), - os.path.join(cwd, 'dist', 'examples', 'cmdline', 'duk_cmdline.c'), - os.path.join(cwd, 'dist', 'extras', 'print-alert', 'duk_print_alert.c'), - '-lm' - ]) - - execute([ 'size', os.path.join(cwd, 'duk') ]) - - res = execute([ - os.path.join(cwd, 'duk'), - '-e', 'print(Duktape.version); print(Duktape.env); print(Math.PI)' - ]) - - res = execute([ - os.path.join(cwd, 'duk'), - os.path.join('dist', 'mandel.js') - ]) - md5_stdout = md5.md5(res['stdout']).digest().encode('hex') - md5_expect = '627cd86f0a4255e018c564f86c6d0ab3' - print(md5_stdout) - print(md5_expect) - return md5_stdout == md5_expect + cwd = os.getcwd() + execute([ 'make', 'dist' ]) + execute([ + 'python2', os.path.join(cwd, 'tools', 'genconfig.py'), + '--metadata', os.path.join(cwd, 'config'), + '--output', os.path.join(cwd, 'dist', 'src', 'duk_config.h') + ] + genconfig_opts + [ + 'duk-config-header' + ]) + execute([ + 'gcc', '-oduk', archopt, + '-Os', '-fomit-frame-pointer', + '-flto', '-fno-asynchronous-unwind-tables', + '-ffunction-sections', '-Wl,--gc-sections', + '-DDUK_CMDLINE_PRINTALERT_SUPPORT', + '-I' + os.path.join('dist', 'src'), + '-I' + os.path.join(cwd, 'dist', 'extras', 'print-alert'), + '-I' + os.path.join('dist', 'examples', 'cmdline'), + os.path.join(cwd, 'dist', 'src', 'duktape.c'), + os.path.join(cwd, 'dist', 'examples', 'cmdline', 'duk_cmdline.c'), + os.path.join(cwd, 'dist', 'extras', 'print-alert', 'duk_print_alert.c'), + '-lm' + ]) + + execute([ 'size', os.path.join(cwd, 'duk') ]) + + res = execute([ + os.path.join(cwd, 'duk'), + '-e', 'print(Duktape.version); print(Duktape.env); print(Math.PI)' + ]) + + res = execute([ + os.path.join(cwd, 'duk'), + os.path.join('dist', 'mandel.js') + ]) + md5_stdout = md5.md5(res['stdout']).digest().encode('hex') + md5_expect = '627cd86f0a4255e018c564f86c6d0ab3' + print(md5_stdout) + print(md5_expect) + return md5_stdout == md5_expect def context_linux_regconst_variants(): - res = True - res = res and mandel_test('-m64', [ '-DDUK_USE_EXEC_REGCONST_OPTIMIZE' ]) - res = res and mandel_test('-m64', [ '-UDUK_USE_EXEC_REGCONST_OPTIMIZE' ]) - res = res and mandel_test('-m32', [ '-DDUK_USE_EXEC_REGCONST_OPTIMIZE' ]) - res = res and mandel_test('-m32', [ '-UDUK_USE_EXEC_REGCONST_OPTIMIZE' ]) - return res + res = True + res = res and mandel_test('-m64', [ '-DDUK_USE_EXEC_REGCONST_OPTIMIZE' ]) + res = res and mandel_test('-m64', [ '-UDUK_USE_EXEC_REGCONST_OPTIMIZE' ]) + res = res and mandel_test('-m32', [ '-DDUK_USE_EXEC_REGCONST_OPTIMIZE' ]) + res = res and mandel_test('-m32', [ '-UDUK_USE_EXEC_REGCONST_OPTIMIZE' ]) + return res def context_linux_tval_variants(): - # Cover most duk_tval.h cases, but only for little endian now. - res = True - for archopt in [ '-m64', '-m32' ]: - optsets = [] - optsets.append([ '-UDUK_USE_PACKED_TVAL', '-DDUK_USE_64BIT_OPS', '-DDUK_USE_FASTINT' ]) - optsets.append([ '-UDUK_USE_PACKED_TVAL', '-DDUK_USE_64BIT_OPS', '-UDUK_USE_FASTINT' ]) - optsets.append([ '-UDUK_USE_PACKED_TVAL', '-UDUK_USE_64BIT_OPS', '-UDUK_USE_FASTINT' ]) - if archopt == '-m32': - optsets.append([ '-DDUK_USE_PACKED_TVAL', '-DDUK_USE_64BIT_OPS', '-DDUK_USE_FASTINT' ]) - optsets.append([ '-DDUK_USE_PACKED_TVAL', '-DDUK_USE_64BIT_OPS', '-UDUK_USE_FASTINT' ]) - optsets.append([ '-DDUK_USE_PACKED_TVAL', '-UDUK_USE_64BIT_OPS', '-UDUK_USE_FASTINT' ]) - for optset in optsets: - res = res and mandel_test(archopt, optset) - return res + # Cover most duk_tval.h cases, but only for little endian now. + res = True + for archopt in [ '-m64', '-m32' ]: + optsets = [] + optsets.append([ '-UDUK_USE_PACKED_TVAL', '-DDUK_USE_64BIT_OPS', '-DDUK_USE_FASTINT' ]) + optsets.append([ '-UDUK_USE_PACKED_TVAL', '-DDUK_USE_64BIT_OPS', '-UDUK_USE_FASTINT' ]) + optsets.append([ '-UDUK_USE_PACKED_TVAL', '-UDUK_USE_64BIT_OPS', '-UDUK_USE_FASTINT' ]) + if archopt == '-m32': + optsets.append([ '-DDUK_USE_PACKED_TVAL', '-DDUK_USE_64BIT_OPS', '-DDUK_USE_FASTINT' ]) + optsets.append([ '-DDUK_USE_PACKED_TVAL', '-DDUK_USE_64BIT_OPS', '-UDUK_USE_FASTINT' ]) + optsets.append([ '-DDUK_USE_PACKED_TVAL', '-UDUK_USE_64BIT_OPS', '-UDUK_USE_FASTINT' ]) + for optset in optsets: + res = res and mandel_test(archopt, optset) + return res def context_linux_x64_minisphere(): - cwd = os.getcwd() + cwd = os.getcwd() - execute([ 'make', 'dist' ]) + execute([ 'make', 'dist' ]) - # Unpack minisphere snapshot and copy Duktape files over. - unpack_targz(os.path.join(repo_snapshot_dir, 'minisphere-20160516.tar.gz')) + # Unpack minisphere snapshot and copy Duktape files over. + unpack_targz(os.path.join(repo_snapshot_dir, 'minisphere-20160516.tar.gz')) - for i in [ 'duktape.c', 'duktape.h', 'duk_config.h' ]: - execute([ - 'cp', - os.path.join(cwd, 'dist', 'src', i), - os.path.join(cwd, 'minisphere', 'src', 'shared', i) - ]) + for i in [ 'duktape.c', 'duktape.h', 'duk_config.h' ]: + execute([ + 'cp', + os.path.join(cwd, 'dist', 'src', i), + os.path.join(cwd, 'minisphere', 'src', 'shared', i) + ]) - # sudo apt-get install liballegro5-dev libmng-dev - os.chdir(os.path.join(cwd, 'minisphere')) - return execute([ 'make' ], catch=True)['success'] + # sudo apt-get install liballegro5-dev libmng-dev + os.chdir(os.path.join(cwd, 'minisphere')) + return execute([ 'make' ], catch=True)['success'] def context_linux_x64_dukluv(): - cwd = os.getcwd() - - execute([ 'make', 'dist' ]) - - # Unpack dukluv snapshot and symlink dukluv/lib/duktape to dist. - unpack_targz(os.path.join(repo_snapshot_dir, 'dukluv-20160528.tar.gz')) - - execute([ - 'mv', - os.path.join(cwd, 'dukluv', 'lib', 'duktape'), - os.path.join(cwd, 'dukluv', 'lib', 'duktape-moved') - ]) - execute([ - 'ln', - '-s', - os.path.join(cwd, 'dist'), - os.path.join(cwd, 'dukluv', 'lib', 'duktape') - ]) - - os.chdir(os.path.join(cwd, 'dukluv')) - execute([ 'mkdir', 'build' ]) - os.chdir(os.path.join(cwd, 'dukluv', 'build')) - execute([ 'cmake', '..' ]) - res = execute([ 'make' ], catch=True) - if not res['success']: - print('Build failed!') - return False - - # Binary is in dukluv/build/dukluv. - - execute([ - os.path.join(cwd, 'dukluv', 'build', 'dukluv'), - os.path.join(cwd, 'dukluv', 'test-argv.js') - ]) - - return True + cwd = os.getcwd() + + execute([ 'make', 'dist' ]) + + # Unpack dukluv snapshot and symlink dukluv/lib/duktape to dist. + unpack_targz(os.path.join(repo_snapshot_dir, 'dukluv-20160528.tar.gz')) + + execute([ + 'mv', + os.path.join(cwd, 'dukluv', 'lib', 'duktape'), + os.path.join(cwd, 'dukluv', 'lib', 'duktape-moved') + ]) + execute([ + 'ln', + '-s', + os.path.join(cwd, 'dist'), + os.path.join(cwd, 'dukluv', 'lib', 'duktape') + ]) + + os.chdir(os.path.join(cwd, 'dukluv')) + execute([ 'mkdir', 'build' ]) + os.chdir(os.path.join(cwd, 'dukluv', 'build')) + execute([ 'cmake', '..' ]) + res = execute([ 'make' ], catch=True) + if not res['success']: + print('Build failed!') + return False + + # Binary is in dukluv/build/dukluv. + + execute([ + os.path.join(cwd, 'dukluv', 'build', 'dukluv'), + os.path.join(cwd, 'dukluv', 'test-argv.js') + ]) + + return True context_handlers = { - # Linux - - 'codepolicycheck': context_codepolicycheck, - 'linux-x64-qecmatest': context_linux_x64_qecmatest, - 'linux-x64-qecmatest-assert': context_linux_x64_qecmatest_assert, - 'linux-x64-qecmatest-valgrind': context_linux_x64_qecmatest_valgrind, - # XXX: torture options - 'linux-x64-apitest': context_linux_x64_apitest, - 'linux-x64-apitest-assert': context_linux_x64_apitest_assert, - 'linux-x64-apitest-valgrind': context_linux_x64_apitest_valgrind, - 'linux-x64-test262test': context_linux_x64_test262test, - # XXX: torture options - - # XXX: regfuzztest - # XXX: luajstest - # XXX: jsinterpretertest - # XXX: bluebirdtest - # XXX: emscripteninceptiontest - - 'linux-x64-duk-clang': context_linux_x64_duk_clang, - 'linux-x64-duk-gxx': context_linux_x64_duk_gxx, - - 'linux-x64-gcc-defsize-makeduk': context_linux_x64_gcc_defsize_makeduk, - 'linux-x64-gcc-defsize-fltoetc': context_linux_x64_gcc_defsize_fltoetc, - 'linux-x64-gcc-minsize-fltoetc': context_linux_x64_gcc_minsize_fltoetc, - 'linux-x86-gcc-minsize-fltoetc': context_linux_x86_gcc_minsize_fltoetc, - 'linux-x32-gcc-minsize-fltoetc': context_linux_x32_gcc_minsize_fltoetc, - 'linux-x64-gcc-stripsize-fltoetc': context_linux_x64_gcc_stripsize_fltoetc, - 'linux-x86-gcc-stripsize-fltoetc': context_linux_x86_gcc_stripsize_fltoetc, - 'linux-x32-gcc-stripsize-fltoetc': context_linux_x32_gcc_stripsize_fltoetc, - - 'linux-x64-cpp-exceptions': context_linux_x64_cpp_exceptions, - - 'linux-x86-ajduk': context_linux_x86_ajduk, - 'linux-x86-ajduk-rombuild': context_linux_x86_ajduk_rombuild, - - 'linux-x64-v8-bench-pass': context_linux_x64_v8_bench_pass, - - 'linux-x64-duk-dddprint': context_linux_x64_duk_dddprint, - 'linux-x64-duk-separate-src': context_linux_x64_duk_separate_src, - 'linux-x86-packed-tval': context_linux_x86_packed_tval, - 'linux-x86-dist-genconfig': context_linux_x86_dist_genconfig, - - 'linux-x64-error-variants': context_linux_x64_error_variants, - - 'linux-x64-hello-ram': context_linux_x64_hello_ram, - 'linux-x86-hello-ram': context_linux_x86_hello_ram, - 'linux-x32-hello-ram': context_linux_x32_hello_ram, - - 'linux-regconst-variants': context_linux_regconst_variants, - 'linux-tval-variants': context_linux_tval_variants, - - 'linux-x64-minisphere': context_linux_x64_minisphere, - 'linux-x64-dukluv': context_linux_x64_dukluv, - - # OS X: can currently share Linux handlers - - 'osx-x64-qecmatest': context_linux_x64_qecmatest, - 'osx-x64-duk-clang': context_linux_x64_duk_clang, - 'osx-x64-duk-gxx': context_linux_x64_duk_gxx, - 'osx-x64-gcc-minsize-makeduk': context_linux_x64_gcc_defsize_makeduk + # Linux + + 'codepolicycheck': context_codepolicycheck, + 'linux-x64-qecmatest': context_linux_x64_qecmatest, + 'linux-x64-qecmatest-assert': context_linux_x64_qecmatest_assert, + 'linux-x64-qecmatest-valgrind': context_linux_x64_qecmatest_valgrind, + # XXX: torture options + 'linux-x64-apitest': context_linux_x64_apitest, + 'linux-x64-apitest-assert': context_linux_x64_apitest_assert, + 'linux-x64-apitest-valgrind': context_linux_x64_apitest_valgrind, + 'linux-x64-test262test': context_linux_x64_test262test, + # XXX: torture options + + # XXX: regfuzztest + # XXX: luajstest + # XXX: jsinterpretertest + # XXX: bluebirdtest + # XXX: emscripteninceptiontest + + 'linux-x64-duk-clang': context_linux_x64_duk_clang, + 'linux-x64-duk-gxx': context_linux_x64_duk_gxx, + + 'linux-x64-gcc-defsize-makeduk': context_linux_x64_gcc_defsize_makeduk, + 'linux-x64-gcc-defsize-fltoetc': context_linux_x64_gcc_defsize_fltoetc, + 'linux-x64-gcc-minsize-fltoetc': context_linux_x64_gcc_minsize_fltoetc, + 'linux-x86-gcc-minsize-fltoetc': context_linux_x86_gcc_minsize_fltoetc, + 'linux-x32-gcc-minsize-fltoetc': context_linux_x32_gcc_minsize_fltoetc, + 'linux-x64-gcc-stripsize-fltoetc': context_linux_x64_gcc_stripsize_fltoetc, + 'linux-x86-gcc-stripsize-fltoetc': context_linux_x86_gcc_stripsize_fltoetc, + 'linux-x32-gcc-stripsize-fltoetc': context_linux_x32_gcc_stripsize_fltoetc, + + 'linux-x64-cpp-exceptions': context_linux_x64_cpp_exceptions, + + 'linux-x86-ajduk': context_linux_x86_ajduk, + 'linux-x86-ajduk-rombuild': context_linux_x86_ajduk_rombuild, + + 'linux-x64-v8-bench-pass': context_linux_x64_v8_bench_pass, + + 'linux-x64-duk-dddprint': context_linux_x64_duk_dddprint, + 'linux-x64-duk-separate-src': context_linux_x64_duk_separate_src, + 'linux-x86-packed-tval': context_linux_x86_packed_tval, + 'linux-x86-dist-genconfig': context_linux_x86_dist_genconfig, + + 'linux-x64-error-variants': context_linux_x64_error_variants, + + 'linux-x64-hello-ram': context_linux_x64_hello_ram, + 'linux-x86-hello-ram': context_linux_x86_hello_ram, + 'linux-x32-hello-ram': context_linux_x32_hello_ram, + + 'linux-regconst-variants': context_linux_regconst_variants, + 'linux-tval-variants': context_linux_tval_variants, + + 'linux-x64-minisphere': context_linux_x64_minisphere, + 'linux-x64-dukluv': context_linux_x64_dukluv, + + # OS X: can currently share Linux handlers + + 'osx-x64-qecmatest': context_linux_x64_qecmatest, + 'osx-x64-duk-clang': context_linux_x64_duk_clang, + 'osx-x64-duk-gxx': context_linux_x64_duk_gxx, + 'osx-x64-gcc-minsize-makeduk': context_linux_x64_gcc_defsize_makeduk } # @@ -917,106 +931,113 @@ context_handlers = { # def main(): - print('*** Running test %r on %s' % (context, datetime.datetime.utcnow().isoformat() + 'Z')) - print('') - print('repo_full_name: ' + repo_full_name) - print('repo_clone_url: ' + repo_clone_url) - print('commit_name: ' + commit_name) - print('context: ' + context); - - if not os.path.isdir(temp_dir): - raise Exception('missing or invalid temporary directory: %r' % temp_dir) - - m = re_reponame.match(repo_full_name) - if m is None: - raise Exception('invalid repo name: %r' % repo_full_name) - - if repo_full_name not in repo_whitelist: - raise Exception('repo name is not whitelisted: %r' % repo_full_name) - - # Replace full repo forward slashes with platform separator - repo_targz = apply(os.path.join, [ repo_snapshot_dir ] + (repo_full_name + '.tar.gz').split('/')) - if repo_targz[0:len(repo_snapshot_dir)] != repo_snapshot_dir: - raise Exception('internal error figuring out repo_targz: %r' % repo_targz) - - repo_dir = os.path.join(temp_dir, 'repo') - os.chdir(temp_dir) - os.mkdir(repo_dir) - - print('') - print('*** Unpack repos and helpers') - print('') - - os.chdir(repo_dir) - unpack_targz(repo_targz) - execute([ 'git', 'config', 'core.filemode', 'false' ]) # avoid perm issues on Windows - - for fn in [ - 'alljoyn-72930212134129ae0464df93c526a6d110cb82f7.tar.gz', - 'ajtcl-cf47440914f31553a0064f3dabbbf337921ea357.tar.gz', - 'closure-20160317.tar.gz', - 'uglifyjs2-20160317.tar.gz', - 'runtests-node-modules-20160320.tar.gz' - ]: - unpack_targz(os.path.join(repo_snapshot_dir, fn)) - - execute([ 'git', 'clean', '-f' ]) - execute([ 'git', 'reset', '--hard' ]) - execute([ 'git', 'checkout', 'master' ]) - execute([ 'git', 'pull', '--rebase' ]) - execute([ 'git', 'clean', '-f' ]) - execute([ 'git', 'reset', '--hard' ]) - execute([ 'git', 'checkout', commit_name ]) - execute([ 'git', 'describe', '--always', '--dirty' ]) - - fn = context_handlers.get(context) - if fn is None: - print('Unknown context %s, supported contexts:') - for ctx in sorted(context_handlers.keys()): - print(' ' + ctx) - raise Exception('context unknown: ' + context) - - print('') - print('*** Running test for context: ' + context) - print('') - - success = fn() - - print('') - print('*** Finished test for context: ' + context + ', success: ' + repr(success)) - print('') - - if output_description is not None: - print('TESTRUNNER_DESCRIPTION: ' + output_description) - - if success == True: - # Testcase successful - print('Test succeeded') - sys.exit(0) - elif success == False: - # Testcase failed, but no test script error (= don't rerun automatically) - print('Test failed') - sys.exit(1) - else: - raise Exception('context handler returned a non-boolean: %r' % success) + print('*** Running test %r on %s' % (context, datetime.datetime.utcnow().isoformat() + 'Z')) + print('') + print('repo_full_name: ' + repo_full_name) + print('repo_clone_url: ' + repo_clone_url) + print('commit_name: ' + commit_name) + print('context: ' + context); + + if not os.path.isdir(temp_dir): + raise Exception('missing or invalid temporary directory: %r' % temp_dir) + + m = re_reponame.match(repo_full_name) + if m is None: + raise Exception('invalid repo name: %r' % repo_full_name) + + if repo_full_name not in repo_whitelist: + raise Exception('repo name is not whitelisted: %r' % repo_full_name) + + # Replace full repo forward slashes with platform separator + repo_targz = apply(os.path.join, [ repo_snapshot_dir ] + (repo_full_name + '.tar.gz').split('/')) + if repo_targz[0:len(repo_snapshot_dir)] != repo_snapshot_dir: + raise Exception('internal error figuring out repo_targz: %r' % repo_targz) + + repo_dir = os.path.join(temp_dir, 'repo') + os.chdir(temp_dir) + os.mkdir(repo_dir) + + print('') + print('*** GCC and Clang versions') + print('') + + execute([ 'gcc', '-v' ], catch=True) + execute([ 'clang', '-v' ], catch=True) + + print('') + print('*** Unpack repos and helpers') + print('') + + os.chdir(repo_dir) + unpack_targz(repo_targz) + execute([ 'git', 'config', 'core.filemode', 'false' ]) # avoid perm issues on Windows + + for fn in [ + 'alljoyn-72930212134129ae0464df93c526a6d110cb82f7.tar.gz', + 'ajtcl-cf47440914f31553a0064f3dabbbf337921ea357.tar.gz', + 'closure-20160317.tar.gz', + 'uglifyjs2-20160317.tar.gz', + 'runtests-node-modules-20160320.tar.gz' + ]: + unpack_targz(os.path.join(repo_snapshot_dir, fn)) + + execute([ 'git', 'clean', '-f' ]) + execute([ 'git', 'reset', '--hard' ]) + execute([ 'git', 'checkout', 'master' ]) + execute([ 'git', 'pull', '--rebase' ]) + execute([ 'git', 'clean', '-f' ]) + execute([ 'git', 'reset', '--hard' ]) + execute([ 'git', 'checkout', commit_name ]) + execute([ 'git', 'describe', '--always', '--dirty' ]) + + fn = context_handlers.get(context) + if fn is None: + print('Unknown context %s, supported contexts:') + for ctx in sorted(context_handlers.keys()): + print(' ' + ctx) + raise Exception('context unknown: ' + context) + + print('') + print('*** Running test for context: ' + context) + print('') + + success = fn() + + print('') + print('*** Finished test for context: ' + context + ', success: ' + repr(success)) + print('') + + if output_description is not None: + print('TESTRUNNER_DESCRIPTION: ' + output_description) + + if success == True: + # Testcase successful + print('Test succeeded') + sys.exit(0) + elif success == False: + # Testcase failed, but no test script error (= don't rerun automatically) + print('Test failed') + sys.exit(1) + else: + raise Exception('context handler returned a non-boolean: %r' % success) if __name__ == '__main__': - start_time = time.time() - - try: - try: - main() - except SystemExit: - raise - except: - # Test script failed, automatic retry is useful - print('') - print('*** Test script failed') - print('') - traceback.print_exc() - print('TESTRUNNER_DESCRIPTION: Test script error') - sys.exit(2) - finally: - end_time = time.time() - print('') - print('Test took %.2f minutes' % ((end_time - start_time) / 60.0)) + start_time = time.time() + + try: + try: + main() + except SystemExit: + raise + except: + # Test script failed, automatic retry is useful + print('') + print('*** Test script failed') + print('') + traceback.print_exc() + print('TESTRUNNER_DESCRIPTION: Test script error') + sys.exit(2) + finally: + end_time = time.time() + print('') + print('Test took %.2f minutes' % ((end_time - start_time) / 60.0)) diff --git a/tools/combine_src.py b/tools/combine_src.py index ae7579e6..28b12598 100644 --- a/tools/combine_src.py +++ b/tools/combine_src.py @@ -5,7 +5,7 @@ # Overview of the process: # # * Parse user supplied C files. Add automatic #undefs at the end -# of each C file to avoid defined bleeding from one file to another. +# of each C file to avoid defines bleeding from one file to another. # # * Combine the C files in specified order. If sources have ordering # dependencies (depends on application), order may matter. @@ -14,11 +14,11 @@ # them either as "internal" (found in specified include path) or # "external". Internal includes, unless explicitly excluded, are # inlined into the result while extenal includes are left as is. -# Duplicate #include statements are replaced with a comment. +# Duplicate internal #include statements are replaced with a comment. # # At every step, source and header lines are represented with explicit # line objects which keep track of original filename and line. The -# output contains #line directives, if necessary, to ensure error +# output contains #line directives, if requested, to ensure error # throwing and other diagnostic info will work in a useful manner when # deployed. It's also possible to generate a combined source with no # #line directives. @@ -34,14 +34,14 @@ # there are structs/unions/typedefs with conflicting names, these # have to be resolved in the source files first. # -# * Because duplicate #include statements are suppressed, currently -# assumes #include statements are not conditional. +# * Because duplicate #include statements are suppressed, currently +# assumes #include statements are not conditional. # -# * A system header might be #include'd in multiple source files with -# different feature defines (like _BSD_SOURCE). Because the #include -# file will only appear once in the resulting source, the first -# occurrence wins. The result may not work correctly if the feature -# defines must actually be different between two or more source files. +# * A system header might be #include'd in multiple source files with +# different feature defines (like _BSD_SOURCE). Because the #include +# file will only appear once in the resulting source, the first +# occurrence wins. The result may not work correctly if the feature +# defines must actually be different between two or more source files. # import os @@ -234,7 +234,7 @@ def main(): assert(opts.output_source) assert(opts.output_metadata) - print('Read input files, add automatic #undefs') + # Read input files, add automatic #undefs sources = args files = [] for fn in sources: @@ -243,7 +243,6 @@ def main(): addAutomaticUndefs(res) files.append(res) - print('Create combined source file from %d source files' % len(files)) combined_source, metadata = \ createCombined(files, opts.prologue, opts.line_directives) with open(opts.output_source, 'wb') as f: @@ -251,7 +250,7 @@ def main(): with open(opts.output_metadata, 'wb') as f: f.write(json.dumps(metadata, indent=4)) - print('Wrote %d bytes to %s' % (len(combined_source), opts.output_source)) + print('Combined %d source files, %d bytes written to %s' % (len(files), len(combined_source), opts.output_source)) if __name__ == '__main__': main() diff --git a/tools/extract_caseconv.py b/tools/extract_caseconv.py index b7bc0c4a..f767cd4c 100644 --- a/tools/extract_caseconv.py +++ b/tools/extract_caseconv.py @@ -12,14 +12,16 @@ # and String.prototype.toLocaleLowerCase()), so they are best handled # in C anyway. # -# Case conversion rules for ASCII are also excluded as they are -# handled by C fast path. Rules for non-BMP characters (codepoints -# above U+FFFF) are omitted as they're not required for standard -# Ecmascript. +# Case conversion rules for ASCII are also excluded as they are handled +# by the C fast path. Rules for non-BMP characters (codepoints above +# U+FFFF) are omitted as they're not required for standard Ecmascript. # -import os, sys, math +import os +import sys +import math import optparse + import dukutil class UnicodeData: @@ -214,8 +216,8 @@ def generate_tables(convmap): # most reliable and simple way to scan ranges = [] # range mappings (2 or more consecutive mappings with a certain skip) - singles = [] # 1:1 character mappings - complex = [] # 1:n character mappings + singles = [] # 1:1 character mappings + multis = [] # 1:n character mappings # Ranges with skips @@ -267,25 +269,25 @@ def generate_tables(convmap): # print 'special399, skip %d: %d %d %d' % (skip, start_i, start_o, count) # print len(tmp.keys()) # print repr(tmp) - # XXX: need to put 12 remaining mappings back to convmap... + # XXX: need to put 12 remaining mappings back to convmap # 1:n conversions k = convmap.keys() k.sort() for i in k: - complex.append([i, convmap[i]]) # codepoint, string + multis.append([i, convmap[i]]) # codepoint, string del convmap[i] for t in singles: print repr(t) - for t in complex: + for t in multis: print repr(t) print 'range mappings: %d' % len(ranges) print 'single character mappings: %d' % len(singles) - print 'complex mappings (1:n): %d' % len(complex) + print 'complex mappings (1:n): %d' % len(multis) print 'remaining (should be zero): %d' % len(convmap.keys()) # XXX: opportunities for diff encoding skip=3 ranges? @@ -330,9 +332,9 @@ def generate_tables(convmap): be.bits(cp_i, 16) be.bits(cp_o, 16) - count = len(complex) + count = len(multis) be.bits(count, 7) - for t in complex: + for t in multis: cp_i, str_o = t[0], t[1] be.bits(cp_i, 16) be.bits(len(str_o), 2) diff --git a/tools/extract_chars.py b/tools/extract_chars.py index 77e061c7..861fc930 100644 --- a/tools/extract_chars.py +++ b/tools/extract_chars.py @@ -15,8 +15,11 @@ # supported in standard Ecmascript. # -import os, sys, math +import os +import sys +import math import optparse + import dukutil def read_unicode_data(unidata, catsinc, catsexc, filterfunc): @@ -318,19 +321,19 @@ def main(): return True print('read unicode data') - res = read_unicode_data(unidata, catsinc, catsexc, filter1) + uni_filtered = read_unicode_data(unidata, catsinc, catsexc, filter1) print('done reading unicode data') # Raw output #print('RAW OUTPUT:') #print('===========') - #print('\n'.join(res)) + #print('\n'.join(uni_filtered)) # Scan ranges #print('') #print('RANGES:') #print('=======') - ranges = scan_ranges(res) + ranges = scan_ranges(uni_filtered) #for i in ranges: # if i[0] == i[1]: # print('0x%04x' % i[0]) @@ -376,7 +379,7 @@ def main(): # Image (for illustrative purposes only) if opts.out_png is not None: - generate_png(res, opts.out_png) + generate_png(uni_filtered, opts.out_png) if __name__ == '__main__': main() diff --git a/tools/genbuildparams.py b/tools/genbuildparams.py deleted file mode 100644 index ebf5f502..00000000 --- a/tools/genbuildparams.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python2 -# -# Generate build parameter files based on build information. -# A C header is generated for C code, and a JSON file for -# build scripts etc which need to know the build config. -# - -import os -import sys -import json -import optparse - -import dukutil - -if __name__ == '__main__': - parser = optparse.OptionParser() - parser.add_option('--version', dest='version') - parser.add_option('--git-commit', dest='git_commit') - parser.add_option('--git-describe', dest='git_describe') - parser.add_option('--git-branch', dest='git_branch') - parser.add_option('--out-json', dest='out_json') - parser.add_option('--out-header', dest='out_header') - (opts, args) = parser.parse_args() - - t = { - 'version': opts.version, - 'git_commit': opts.git_commit, - 'git_describe': opts.git_describe, - 'git_branch': opts.git_branch, - } - - f = open(opts.out_json, 'wb') - f.write(dukutil.json_encode(t).encode('ascii')) - f.close() - - f = open(opts.out_header, 'wb') - f.write('#ifndef DUK_BUILDPARAMS_H_INCLUDED\n') - f.write('#define DUK_BUILDPARAMS_H_INCLUDED\n') - f.write('/* automatically generated by genbuildparams.py, do not edit */\n') - f.write('\n') - f.write('/* DUK_VERSION is defined in duktape.h */') - f.write('\n') - f.write('#endif /* DUK_BUILDPARAMS_H_INCLUDED */\n') - f.close() diff --git a/tools/genbuiltins.py b/tools/genbuiltins.py index 1477cf36..93e2d45d 100644 --- a/tools/genbuiltins.py +++ b/tools/genbuiltins.py @@ -559,6 +559,7 @@ def metadata_normalize_rom_property_attributes(meta): # Add a 'name' property for all top level functions; expected by RAM # initialization code. def metadata_normalize_ram_function_names(meta): + num_added = 0 for o in meta['objects']: if not o.get('callable', False): continue @@ -568,9 +569,13 @@ def metadata_normalize_ram_function_names(meta): name_prop = p break if name_prop is None: - print('Adding missing "name" property for top level function %s' % o['id']) + num_added += 1 + #print('Adding missing "name" property for function %s' % o['id']) o['properties'].append({ 'key': 'name', 'value': '', 'attributes': '' }) + if num_added > 0: + print('Added missing "name" property for %d functions' % num_added) + # Add a built-in objects list for RAM initialization. def metadata_add_ram_filtered_object_list(meta): # For RAM init data to support user objects, we need to prepare a @@ -752,8 +757,8 @@ def metadata_remove_orphan_objects(meta): _markId(v.get('getter_id')) _markId(v.get('setter_id')) - print('Mark reachable: reachable count initially %d, now %d' % \ - (reachable_count, len(reachable.keys()))) + #print('Mark reachable: reachable count initially %d, now %d' % \ + # (reachable_count, len(reachable.keys()))) if reachable_count == len(reachable.keys()): break @@ -769,7 +774,8 @@ def metadata_remove_orphan_objects(meta): num_deleted += 1 break - print('Deleted %d unreachable objects' % num_deleted) + if num_deleted > 0: + print('Deleted %d unreachable objects' % num_deleted) # Add C define names for builtin strings. These defines are added to all # strings, even when they won't get a stridx because the define names are @@ -969,7 +975,7 @@ def load_metadata(opts, rom=False, build_info=None): # Add Duktape.version and (Duktape.env for ROM case). for o in meta['objects']: if o['id'] == 'bi_duktape': - o['properties'].insert(0, { 'key': 'version', 'value': int(build_info['version']), 'attributes': '' }) + o['properties'].insert(0, { 'key': 'version', 'value': int(build_info['duk_version']), 'attributes': '' }) if rom: # Use a fixed (quite dummy for now) Duktape.env # when ROM builtins are in use. In the RAM case @@ -2811,7 +2817,10 @@ def emit_header_native_function_declarations(genc, meta): def main(): parser = optparse.OptionParser() - parser.add_option('--buildinfo', dest='buildinfo', help='Build info, JSON format') + parser.add_option('--git-commit', dest='git_commit', default=None, help='Git commit hash') + parser.add_option('--git-describe', dest='git_describe', default=None, help='Git describe') + parser.add_option('--git-branch', dest='git_branch', default=None, help='Git branch name') + parser.add_option('--duk-version', dest='duk_version', default=None, help='Duktape version (e.g. 10203)') parser.add_option('--used-stridx-metadata', dest='used_stridx_metadata', help='DUK_STRIDX_xxx used by source/headers, JSON format') parser.add_option('--strings-metadata', dest='strings_metadata', help='Built-in strings metadata file, YAML format') parser.add_option('--objects-metadata', dest='objects_metadata', help='Built-in objects metadata file, YAML format') @@ -2828,11 +2837,12 @@ def main(): # Options processing. - if opts.buildinfo is None: - raise Exception('missing buildinfo') - - with open(opts.buildinfo, 'rb') as f: - build_info = dukutil.json_decode(f.read().strip()) + build_info = { + 'git_commit': opts.git_commit, + 'git_branch': opts.git_branch, + 'git_describe': opts.git_describe, + 'duk_version': int(opts.duk_version), + } # Read in metadata files, normalizing and merging as necessary. @@ -2954,7 +2964,7 @@ def main(): # Write a JSON file with build metadata, e.g. built-in strings. - ver = long(build_info['version']) + ver = long(build_info['duk_version']) plain_strs = [] base64_strs = [] str_objs = [] diff --git a/tools/genconfig.py b/tools/genconfig.py index ba8a3dc7..2caa5f35 100644 --- a/tools/genconfig.py +++ b/tools/genconfig.py @@ -1398,7 +1398,7 @@ def generate_duk_config_header(opts, meta_dir): # Main # -def main(): +def add_genconfig_optparse_options(parser, direct=False): # Forced options from multiple sources are gathered into a shared list # so that the override order remains the same as on the command line. force_options_yaml = [] @@ -1436,19 +1436,14 @@ def main(): line = line[:-1] fixup_header_lines.append(line) - commands = [ - 'duk-config-header', - 'feature-documentation', - 'config-documentation' - ] - parser = optparse.OptionParser( - usage='Usage: %prog [options] COMMAND', - description='Generate a duk_config.h or config option documentation based on config metadata.', - epilog='COMMAND can be one of: ' + ', '.join(commands) + '.' - ) + if direct: + parser.add_option('--metadata', dest='config_metadata', default=None, help='metadata directory or metadata tar.gz file') + parser.add_option('--output', dest='output', default=None, help='output filename for C header or RST documentation file') + else: + # Different option name when called through prepare_sources.py, + # also no --output option. + parser.add_option('--config-metadata', dest='config_metadata', default=None, help='metadata directory or metadata tar.gz file') - parser.add_option('--metadata', dest='metadata', default=None, help='metadata directory or metadata tar.gz file') - parser.add_option('--output', dest='output', default=None, help='output filename for C header or RST documentation file') parser.add_option('--platform', dest='platform', default=None, help='platform (default is autodetect)') parser.add_option('--compiler', dest='compiler', default=None, help='compiler (default is autodetect)') parser.add_option('--architecture', dest='architecture', default=None, help='architecture (default is autodetec)') @@ -1471,26 +1466,46 @@ def main(): parser.add_option('--fixup-line', type='string', dest='fixup_header_lines', action='callback', callback=add_fixup_header_line, default=fixup_header_lines, help='C header fixup line to be appended to generated header (e.g. --fixup-line "#define DUK_USE_FASTINT")') parser.add_option('--sanity-warning', dest='sanity_strict', action='store_false', default=True, help='emit a warning instead of #error for option sanity check issues') parser.add_option('--use-cpp-warning', dest='use_cpp_warning', action='store_true', default=False, help='emit a (non-portable) #warning when appropriate') - parser.add_option('--git-commit', dest='git_commit', default=None, help='git commit hash to be included in header comments') - parser.add_option('--git-describe', dest='git_describe', default=None, help='git describe string to be included in header comments') - parser.add_option('--git-branch', dest='git_branch', default=None, help='git branch string to be included in header comments') + + if direct: + parser.add_option('--git-commit', dest='git_commit', default=None, help='git commit hash to be included in header comments') + parser.add_option('--git-describe', dest='git_describe', default=None, help='git describe string to be included in header comments') + parser.add_option('--git-branch', dest='git_branch', default=None, help='git branch string to be included in header comments') + +def parse_options(): + commands = [ + 'duk-config-header', + 'feature-documentation', + 'config-documentation' + ] + + parser = optparse.OptionParser( + usage='Usage: %prog [options] COMMAND', + description='Generate a duk_config.h or config option documentation based on config metadata.', + epilog='COMMAND can be one of: ' + ', '.join(commands) + '.' + ) + + add_genconfig_optparse_options(parser, direct=True) (opts, args) = parser.parse_args() - meta_dir = opts.metadata - if opts.metadata is None: + return opts, args + +def genconfig(opts, args): + meta_dir = opts.config_metadata + if opts.config_metadata is None: if os.path.isfile(os.path.join('.', 'genconfig_metadata.tar.gz')): - opts.metadata = 'genconfig_metadata.tar.gz' + opts.config_metadata = 'genconfig_metadata.tar.gz' elif os.path.isdir(os.path.join('.', 'config-options')): - opts.metadata = '.' + opts.config_metadata = '.' - if opts.metadata is not None and os.path.isdir(opts.metadata): - meta_dir = opts.metadata + if opts.config_metadata is not None and os.path.isdir(opts.config_metadata): + meta_dir = opts.config_metadata metadata_src_text = 'Using metadata directory: %r' % meta_dir - elif opts.metadata is not None and os.path.isfile(opts.metadata) and tarfile.is_tarfile(opts.metadata): + elif opts.config_metadata is not None and os.path.isfile(opts.config_metadata) and tarfile.is_tarfile(opts.config_metadata): meta_dir = get_auto_delete_tempdir() - tar = tarfile.open(name=opts.metadata, mode='r:*') + tar = tarfile.open(name=opts.config_metadata, mode='r:*') tar.extractall(path=meta_dir) - metadata_src_text = 'Using metadata tar file %r, unpacked to directory: %r' % (opts.metadata, meta_dir) + metadata_src_text = 'Using metadata tar file %r, unpacked to directory: %r' % (opts.config_metadata, meta_dir) else: raise Exception('metadata source must be a directory or a tar.gz file') @@ -1526,5 +1541,9 @@ def main(): else: raise Exception('invalid command: %r' % cmd) +def main(): + opts, args = parse_options() + genconfig(opts, args) + if __name__ == '__main__': main() diff --git a/tools/prepare_sources.py b/tools/prepare_sources.py index f8b674d7..c216bc2b 100644 --- a/tools/prepare_sources.py +++ b/tools/prepare_sources.py @@ -15,6 +15,8 @@ import json import yaml import subprocess +import genconfig + # Helpers def exec_get_stdout(cmd, input=None, default=None, print_stdout=False): @@ -214,29 +216,11 @@ def main(): parser.add_option('--user-builtin-metadata', dest='user_builtin_metadata', action='append', default=[], help='User strings and objects to add, YAML format (can be repeated for multiple overrides)') # Options forwarded to genconfig.py. - parser.add_option('--config-metadata', dest='config_metadata', default=None, help='metadata directory or metadata tar.gz file') - parser.add_option('--platform', dest='platform', default=None, help='platform (default is autodetect)') - parser.add_option('--compiler', dest='compiler', default=None, help='compiler (default is autodetect)') - parser.add_option('--architecture', dest='architecture', default=None, help='architecture (default is autodetec)') - parser.add_option('--c99-types-only', dest='c99_types_only', action='store_true', default=False, help='assume C99 types, no legacy type detection') - parser.add_option('--dll', dest='dll', action='store_true', default=False, help='dll build of Duktape, affects symbol visibility macros especially on Windows') - parser.add_option('--support-feature-options', dest='support_feature_options', action='store_true', default=False, help='support DUK_OPT_xxx feature options in duk_config.h') - parser.add_option('--emit-legacy-feature-check', dest='emit_legacy_feature_check', action='store_true', default=False, help='emit preprocessor checks to reject legacy feature options (DUK_OPT_xxx)') - parser.add_option('--emit-config-sanity-check', dest='emit_config_sanity_check', action='store_true', default=False, help='emit preprocessor checks for config option consistency (DUK_OPT_xxx)') - parser.add_option('--omit-removed-config-options', dest='omit_removed_config_options', action='store_true', default=False, help='omit removed config options from generated headers') - parser.add_option('--omit-deprecated-config-options', dest='omit_deprecated_config_options', action='store_true', default=False, help='omit deprecated config options from generated headers') - parser.add_option('--omit-unused-config-options', dest='omit_unused_config_options', action='store_true', default=False, help='omit unused config options from generated headers') - parser.add_option('--add-active-defines-macro', dest='add_active_defines_macro', action='store_true', default=False, help='add DUK_ACTIVE_DEFINES macro, for development only') - parser.add_option('--define', type='string', dest='force_options_yaml', action='callback', callback=add_force_option_define, default=force_options_yaml, help='force #define option using a C compiler like syntax, e.g. "--define DUK_USE_DEEP_C_STACK" or "--define DUK_USE_TRACEBACK_DEPTH=10"') - parser.add_option('-D', type='string', dest='force_options_yaml', action='callback', callback=add_force_option_define, default=force_options_yaml, help='synonym for --define, e.g. "-DDUK_USE_DEEP_C_STACK" or "-DDUK_USE_TRACEBACK_DEPTH=10"') - parser.add_option('--undefine', type='string', dest='force_options_yaml', action='callback', callback=add_force_option_undefine, default=force_options_yaml, help='force #undef option using a C compiler like syntax, e.g. "--undefine DUK_USE_DEEP_C_STACK"') - parser.add_option('-U', type='string', dest='force_options_yaml', action='callback', callback=add_force_option_undefine, default=force_options_yaml, help='synonym for --undefine, e.g. "-UDUK_USE_DEEP_C_STACK"') - parser.add_option('--option-yaml', type='string', dest='force_options_yaml', action='callback', callback=add_force_option_yaml, default=force_options_yaml, help='force option(s) using inline YAML (e.g. --option-yaml "DUK_USE_DEEP_C_STACK: true")') - parser.add_option('--option-file', type='string', dest='force_options_yaml', action='callback', callback=add_force_option_file, default=force_options_yaml, help='YAML file(s) providing config option overrides') - parser.add_option('--fixup-file', type='string', dest='fixup_header_lines', action='callback', callback=add_fixup_header_file, default=fixup_header_lines, help='C header snippet file(s) to be appended to generated header, useful for manual option fixups') - parser.add_option('--fixup-line', type='string', dest='fixup_header_lines', action='callback', callback=add_fixup_header_line, default=fixup_header_lines, help='C header fixup line to be appended to generated header (e.g. --fixup-line "#define DUK_USE_FASTINT")') - parser.add_option('--sanity-warning', dest='sanity_strict', action='store_false', default=True, help='emit a warning instead of #error for option sanity check issues') - parser.add_option('--use-cpp-warning', dest='use_cpp_warning', action='store_true', default=False, help='emit a (non-portable) #warning when appropriate') + genconfig.add_genconfig_optparse_options(parser) + + # Options for Unicode. + parser.add_option('--unicode-data', dest='unicode_data', default=None, help='Provide custom UnicodeData.txt') + parser.add_option('--special-casing', dest='special_casing', default=None, help='Provide custom SpecialCasing.txt') (opts, args) = parser.parse_args() @@ -290,6 +274,15 @@ def main(): git_describe_cstring = cstring(git_describe) git_branch_cstring = cstring(git_branch) + if opts.unicode_data is None: + unicode_data = os.path.join(srcdir, 'UnicodeData.txt') + else: + unicode_data = opts.unicode_data + if opts.special_casing is None: + special_casing = os.path.join(srcdir, 'SpecialCasing.txt') + else: + special_casing = opts.special_casing + print('Config-and-prepare for Duktape version %s, commit %s, describe %s, branch %s' % \ (duk_version_formatted, git_commit, git_describe, git_branch)) @@ -431,7 +424,8 @@ def main(): copy_and_cquote('AUTHORS.rst', os.path.join(outdir, 'AUTHORS.rst.tmp')) # Create a duk_config.h. - # XXX: might be easier to invoke genconfig directly + # XXX: might be easier to invoke genconfig directly, but there are a few + # options which currently conflict (output file, git commit info, etc). def forward_genconfig_options(): res = [] res += [ '--metadata', os.path.abspath(opts.config_metadata) ] # rename option, --config-metadata => --metadata @@ -459,10 +453,10 @@ def main(): res += [ '--omit-unused-config-options' ] if opts.add_active_defines_macro: res += [ '--add-active-defines-macro' ] - for i in force_options_yaml: + for i in opts.force_options_yaml: res += [ '--option-yaml', i ] - for i in fixup_header_lines: - res += [ '--fixup-linu', i ] + for i in opts.fixup_header_lines: + res += [ '--fixup-line', i ] if not opts.sanity_strict: res += [ '--sanity-warning' ] if opts.use_cpp_warning: @@ -478,7 +472,7 @@ def main(): cmd += [ 'duk-config-header' ] - print(repr(cmd)) + #print(repr(cmd)) exec_print_stdout(cmd) copy_file(os.path.join(outdir, 'duk_config.h.tmp'), os.path.join(outdir, 'src', 'duk_config.h')) @@ -516,18 +510,7 @@ def main(): # There are currently no profile specific variants of strings/builtins, but # this will probably change when functions are added/removed based on profile. - # XXX: nuke this util, it's pointless - exec_print_stdout([ - sys.executable, - os.path.join('tools', 'genbuildparams.py'), - '--version=' + str(duk_version), - '--git-commit=' + git_commit, - '--git-describe=' + git_describe, - '--git-branch=' + git_branch, - '--out-json=' + os.path.join(outdir, 'src-separate', 'buildparams.json.tmp'), - '--out-header=' + os.path.join(outdir, 'src-separate', 'duk_buildparams.h.tmp') - ]) - + # XXX: call as direct python res = exec_get_stdout([ sys.executable, os.path.join('tools', 'scan_used_stridx_bidx.py') @@ -538,31 +521,40 @@ def main(): with open(os.path.join(outdir, 'duk_used_stridx_bidx_defs.json.tmp'), 'wb') as f: f.write(res) - gb_opts = [] - gb_opts.append('--ram-support') # enable by default - if opts.rom_support: - # ROM string/object support is not enabled by default because - # it increases the generated duktape.c considerably. - print('Enabling --rom-support for genbuiltins.py') - gb_opts.append('--rom-support') - if opts.rom_auto_lightfunc: - print('Enabling --rom-auto-lightfunc for genbuiltins.py') - gb_opts.append('--rom-auto-lightfunc') - for fn in opts.user_builtin_metadata: - print('Forwarding --user-builtin-metadata %s' % fn) - gb_opts.append('--user-builtin-metadata') - gb_opts.append(fn) - exec_print_stdout([ + # XXX: call as direct python? does this need to work outside of prepare_sources.py? + cmd = [ sys.executable, os.path.join('tools', 'genbuiltins.py'), - '--buildinfo=' + os.path.join(outdir, 'src-separate', 'buildparams.json.tmp'), + ] + cmd += [ + '--git-commit', git_commit, + '--git-branch', git_branch, + '--git-describe', git_describe, + '--duk-version', str(duk_version) + ] + cmd += [ '--used-stridx-metadata=' + os.path.join(outdir, 'duk_used_stridx_bidx_defs.json.tmp'), '--strings-metadata=' + os.path.join(srcdir, 'strings.yaml'), '--objects-metadata=' + os.path.join(srcdir, 'builtins.yaml'), '--out-header=' + os.path.join(outdir, 'src-separate', 'duk_builtins.h'), '--out-source=' + os.path.join(outdir, 'src-separate', 'duk_builtins.c'), '--out-metadata-json=' + os.path.join(outdir, 'duk_build_meta.json') - ] + gb_opts) + ] + cmd.append('--ram-support') # enable by default + if opts.rom_support: + # ROM string/object support is not enabled by default because + # it increases the generated duktape.c considerably. + print('Enabling --rom-support for genbuiltins.py') + cmd.append('--rom-support') + if opts.rom_auto_lightfunc: + print('Enabling --rom-auto-lightfunc for genbuiltins.py') + cmd.append('--rom-auto-lightfunc') + for fn in opts.user_builtin_metadata: + print('Forwarding --user-builtin-metadata %s' % fn) + cmd.append('--user-builtin-metadata') + cmd.append(fn) + #print(repr(cmd)) + exec_print_stdout(cmd) # Autogenerated Unicode files # @@ -628,7 +620,7 @@ def main(): exec_print_stdout([ sys.executable, os.path.join('tools', 'prepare_unicode_data.py'), - os.path.join(srcdir, 'UnicodeData.txt'), + unicode_data, os.path.join(outdir, 'src-separate', 'UnicodeData-expanded.tmp') ]) @@ -654,7 +646,7 @@ def main(): os.path.join('tools', 'extract_caseconv.py'), '--command=caseconv_bitpacked', '--unicode-data=' + os.path.join(outdir, 'src-separate', 'UnicodeData-expanded.tmp'), - '--special-casing=' + os.path.join(srcdir, 'SpecialCasing.txt'), + '--special-casing=' + special_casing, '--out-source=' + os.path.join(outdir, 'src-separate', 'duk_unicode_caseconv.c.tmp'), '--out-header=' + os.path.join(outdir, 'src-separate', 'duk_unicode_caseconv.h.tmp'), '--table-name-lc=duk_unicode_caseconv_lc', @@ -669,7 +661,7 @@ def main(): os.path.join('tools', 'extract_caseconv.py'), '--command=re_canon_lookup', '--unicode-data=' + os.path.join(outdir, 'src-separate', 'UnicodeData-expanded.tmp'), - '--special-casing=' + os.path.join(srcdir, 'SpecialCasing.txt'), + '--special-casing=' + special_casing, '--out-source=' + os.path.join(outdir, 'src-separate', 'duk_unicode_re_canon_lookup.c.tmp'), '--out-header=' + os.path.join(outdir, 'src-separate', 'duk_unicode_re_canon_lookup.h.tmp'), '--table-name-re-canon-lookup=duk_unicode_re_canon_lookup' diff --git a/util/bluebird-test-shim.js b/util/bluebird-test-shim.js index 7a96ba01..34b86163 100644 --- a/util/bluebird-test-shim.js +++ b/util/bluebird-test-shim.js @@ -7,7 +7,7 @@ var window; (function () { var timers = []; - window = Function('return this')(); + window = Function('return this')(); // window <- global object window.setTimeout = function (fn, timeout) { timers.push(fn); diff --git a/util/check_align.c b/util/check_align.c index d2594699..c96e6dab 100644 --- a/util/check_align.c +++ b/util/check_align.c @@ -2,7 +2,7 @@ * Check for alignment requirements and endianness. * * Called from a shell script check_align.sh to execute one test at a time. - * Prohibited unaligned accesses cause a SIGBUS. + * Prohibited unaligned accesses usually cause a SIGBUS. */ #include diff --git a/util/check_code_policy.py b/util/check_code_policy.py index 97161e15..68989fb2 100644 --- a/util/check_code_policy.py +++ b/util/check_code_policy.py @@ -1,6 +1,6 @@ #!/usr/bin/env python2 # -# Check various C source code policy rules and issue warnings for offenders +# Check various source code policy rules and issue warnings for offenders. # # Usage: # diff --git a/util/example_rombuild.sh b/util/example_rombuild.sh index 723ba092..9e1bf579 100644 --- a/util/example_rombuild.sh +++ b/util/example_rombuild.sh @@ -10,7 +10,7 @@ make clean dist # Prepare-and-config sources manually to enable ROM support. User builtin # metadata can be provided through one or more YAML files (which are applied -# in sequence). +# in sequence). Duktape configuration can be given at the same time. rm -rf dist/src dist/src-noline dist/src-separate $PYTHON dist/tools/prepare_sources.py \ --source-directory dist/src-input \ diff --git a/util/example_user_builtins1.yaml b/util/example_user_builtins1.yaml index 74b445a2..b1cf068b 100644 --- a/util/example_user_builtins1.yaml +++ b/util/example_user_builtins1.yaml @@ -8,7 +8,7 @@ # # See examples below for details on how to use these. # -# Note that genbuiltins.py (and prepare_sources.py) accepts multiple user +# Note that genbuiltins.py and prepare_sources.py accept multiple user # built-in YAML files, so that you can manage your custom strings and # objects in individual YAML files for modularity. # @@ -48,6 +48,13 @@ add_forced_strings: - str: "9" # Non-ascii strings: encode into UTF-8, map bytes to U+0000...U+00FF. + # This encoding approach ensures byte purity which is important to allow + # representing arbitrary byte strings in Duktape internal representation. + # + # For example, to encode 'ä' (U+00E4) first UTF-8 encode the codepoint + # which yields the bytes C3 A4, and then Unicode escape the bytes to get + # '\u00c3\u00a4'. The direct Unicode escape '\u00e4' *will not work*. + # Another example: # # >>> u'säätö'.encode('utf-8').encode('hex') # '73c3a4c3a474c3b6' @@ -105,7 +112,8 @@ objects: # Example of adding an object. Properties may reference other objects using # the 'id' strings even if the object hasn't been introduced yet (and may be - # introduced by a different user builtins YAML file). + # introduced by a different user builtins YAML file). The ID references are + # in effect resolved only after all metadata has been loaded. # # Add a built-in object with plain property values. This just creates the # built-in object; for the object to be actually useful you must use e.g. @@ -147,7 +155,8 @@ objects: # # - The "null" type is supported for ROM init data but not for RAM # init data. Null values are replaced with "undefined" for RAM - # init data at the moment (with a build warning). + # init data at the moment (with a build warning). Same applies + # to lightfunc values at present. # # - One simple way of coming up with exact IEEE doubles is to use # Python: @@ -172,10 +181,10 @@ objects: value: true attributes: wec - key: "integerType" - value: 123 + value: 123 # Interpreted as an IEEE double attributes: wec - key: "floatType" - value: 123.4 + value: 123.4 # Interpreted as an IEEE double attributes: wec - key: "ieeeDoubleType" # Allows exact specification of IEEE doubles value: @@ -193,13 +202,13 @@ objects: # native: duk_bi_math_object_random # nargs: 0 # varargs: false - # length: 13 + # length: 11 # magic: 0 # attributes: wec # Object IDs are only resolved when metadata loading is complete, so it's - # OK to create reference loops or refer to objects defined later (even in - # a separate YAML file not yet loaded). + # OK to create reference loops or refer to objects defined later,(even in + # a separate YAML file not yet loaded. - id: bi_circular1 add: true @@ -279,7 +288,7 @@ objects: value: "" - id: bi_json - disable: true # disabled in metadata + disable: true # disabled in metadata, so no effect unless commented out replace: true class: Object @@ -300,8 +309,8 @@ objects: # custom built-ins and a target specific YAML file removes bindings not # needed for a certain target. # - # In this example we'd delete the StarTrek object. The global reference - # global.StarTrek would be deleted automatically. + # In this example we'd delete the StarTrek object. The dangling global + # reference global.StarTrek would be deleted automatically. - id: bi_star_trek disable: true # disabled in metadata diff --git a/util/fastint_reps.py b/util/fastint_reps.py index 19a075d9..b30edeaf 100644 --- a/util/fastint_reps.py +++ b/util/fastint_reps.py @@ -11,7 +11,7 @@ def isFastint(x): if math.floor(x) == x and \ x >= -(2**47) and \ x < (2**47) and \ - (x != 0 or math.copysign(1.0, x) == 1.0): + (x != 0 or math.copysign(1.0, x) == 1.0): # Negative zero is a bit tricky return True return False diff --git a/util/format_perftest.py b/util/format_perftest.py index 9c2befcc..544f4e8e 100644 --- a/util/format_perftest.py +++ b/util/format_perftest.py @@ -1,6 +1,7 @@ #!/usr/bin/env python2 # # Format a perftest text dump into a HTML table. +# import os import sys diff --git a/util/gendoubleconsts.py b/util/gendoubleconsts.py index ac04e1d9..0dcc594e 100644 --- a/util/gendoubleconsts.py +++ b/util/gendoubleconsts.py @@ -1,8 +1,9 @@ #!/usr/bin/env python2 - -# Double constants, see http://en.wikipedia.org/wiki/Double-precision_floating-point_format. -# Some double constants have been created with 'python-mpmath'. The constants are in binary -# so that the package is not needed for a normal build. +# +# Double constants, see http://en.wikipedia.org/wiki/Double-precision_floating-point_format. +# YAML builtins metadata expressed the constants in binary form (8 bytes of +# IEEE double data) to ensure bit exactness. +# import struct import mpmath diff --git a/util/genhashsizes.py b/util/genhashsizes.py index 2809a85e..bb2236cd 100644 --- a/util/genhashsizes.py +++ b/util/genhashsizes.py @@ -8,6 +8,7 @@ # # Also generates a set of probe steps which are relatively prime to every # hash size. +# import sys import math diff --git a/util/gennumdigits.py b/util/gennumdigits.py index b2b7284e..bff640b9 100644 --- a/util/gennumdigits.py +++ b/util/gennumdigits.py @@ -17,6 +17,7 @@ # values bounded. The exponent limit is relative to an integer # significand padded to the precision-related digit count (e.g. # 20 for decimal). +# import math diff --git a/util/make_dist.py b/util/make_dist.py index 20de5ae9..cbaad04e 100644 --- a/util/make_dist.py +++ b/util/make_dist.py @@ -3,21 +3,6 @@ # Create a distributable Duktape package into 'dist' directory. The contents # of this directory can then be packaged into a source distributable. # -# The distributed source files contain all profiles and variants in one. -# A developer should be able to use the distributed source as follows: -# -# 1. Add the Duktape source files to their project, whichever build -# tool they use (make, scons, etc) -# -# 2. Add the Duktape header files to their include path. -# -# 3. Optionally define some DUK_OPT_xxx feature options. -# -# 4. Compile their program (which uses Duktape API). -# -# In addition to sources, documentation, example programs, and some -# example Makefiles are packaged into the dist package. -# import os import sys @@ -28,7 +13,7 @@ import optparse import subprocess import tarfile -# Helpers +# Helpers. def exec_get_stdout(cmd, input=None, default=None, print_stdout=False): try: @@ -128,8 +113,9 @@ def glob_files(pattern): def cstring(x): return '"' + x + '"' # good enough for now -# DUK_VERSION is grepped from duk_api_public.h.in: it is needed for the -# public API and we want to avoid defining it in two places. +# Get Duktape version number as an integer. DUK_VERSION is grepped from +# duk_api_public.h.in: it is needed for the public API and we want to avoid +# defining it in multiple places. def get_duk_version(): r = re.compile(r'^#define\s+DUK_VERSION\s+(.*?)L?\s*$') with open(os.path.join('src', 'duk_api_public.h.in'), 'rb') as f: @@ -187,30 +173,35 @@ def create_dist_directories(dist): mkdir(os.path.join(dist, 'examples', 'dummy-date-provider')) mkdir(os.path.join(dist, 'examples', 'cpp-exceptions')) -# Path check (spot check a few files to ensure we're in Duktape repo root) +# Spot check a few files to ensure we're in Duktape repo root, as dist only +# works from there. + +def check_cwd_duktape_repo_root(): + if not (os.path.isfile(os.path.join('src', 'duk_api_public.h.in')) and \ + os.path.isfile(os.path.join('config', 'platforms.yaml'))): + sys.stderr.write('\n') + sys.stderr.write('*** Working directory must be Duktape repo checkout root!\n') + sys.stderr.write('\n') + raise Exception('Incorrect working directory') -if not (os.path.isfile(os.path.join('src', 'duk_api_public.h.in')) and \ - os.path.isfile(os.path.join('config', 'platforms.yaml'))): - sys.stderr.write('\n') - sys.stderr.write('*** Working directory must be Duktape repo checkout root!\n') - sys.stderr.write('\n') - raise Exception('Incorrect working directory') +# Option parsing. -# Option parsing +def parse_options(): + parser = optparse.OptionParser() + parser.add_option('--create-spdx', dest='create_spdx', action='store_true', default=False, help='Create SPDX license file') + parser.add_option('--git-commit', dest='git_commit', default=None, help='Force git commit hash') + parser.add_option('--git-describe', dest='git_describe', default=None, help='Force git describe') + parser.add_option('--git-branch', dest='git_branch', default=None, help='Force git branch name') + parser.add_option('--rom-support', dest='rom_support', action='store_true', help='Deprecated, use prepare_sources.py instead') + parser.add_option('--rom-auto-lightfunc', dest='rom_auto_lightfunc', action='store_true', default=False, help='Deprecated, use prepare_sources.py instead') + parser.add_option('--user-builtin-metadata', dest='user_builtin_metadata', action='append', default=[], help='Deprecated, use prepare_sources.py instead') + (opts, args) = parser.parse_args() -parser = optparse.OptionParser() -parser.add_option('--create-spdx', dest='create_spdx', action='store_true', default=False, help='Create SPDX license file') -parser.add_option('--git-commit', dest='git_commit', default=None, help='Force git commit hash') -parser.add_option('--git-describe', dest='git_describe', default=None, help='Force git describe') -parser.add_option('--git-branch', dest='git_branch', default=None, help='Force git branch name') -parser.add_option('--rom-support', dest='rom_support', action='store_true', help='Add support for ROM strings/objects (increases duktape.c size considerably)') -parser.add_option('--rom-auto-lightfunc', dest='rom_auto_lightfunc', action='store_true', default=False, help='Convert ROM built-in function properties into lightfuncs automatically whenever possible') -parser.add_option('--user-builtin-metadata', dest='user_builtin_metadata', action='append', default=[], help='User strings and objects to add, YAML format (can be repeated for multiple overrides)') -(opts, args) = parser.parse_args() + return opts, args -# Python module check and friendly errors +# Python module check and friendly errors. -def check_python_modules(): +def check_python_modules(opts): # make_dist.py doesn't need yaml but other dist utils will; check for it and # warn if it is missing. failed = False @@ -243,445 +234,586 @@ def check_python_modules(): sys.stderr.write('\n') raise Exception('Missing some required Python modules') -check_python_modules() - -# Figure out directories, git info, etc - -entry_pwd = os.getcwd() -dist = os.path.join(entry_pwd, 'dist') - -duk_version, duk_major, duk_minor, duk_patch, duk_version_formatted = get_duk_version() - -if opts.git_commit is not None: - git_commit = opts.git_commit -else: - git_commit = exec_get_stdout([ 'git', 'rev-parse', 'HEAD' ], default='external').strip() -if opts.git_describe is not None: - git_describe = opts.git_describe -else: - git_describe = exec_get_stdout([ 'git', 'describe', '--always', '--dirty' ], default='external').strip() -if opts.git_branch is not None: - git_branch = opts.git_branch -else: - git_branch = exec_get_stdout([ 'git', 'rev-parse', '--abbrev-ref', 'HEAD' ], default='external').strip() - -git_commit_cstring = cstring(git_commit) -git_describe_cstring = cstring(git_describe) -git_branch_cstring = cstring(git_branch) - -print('Dist for Duktape version %s, commit %s, describe %s, branch %s' % \ - (duk_version_formatted, git_commit, git_describe, git_branch)) - -print('Create dist directories and copy static files') - -# Create dist directory structure - -create_dist_directories(dist) - -# Copy most files directly - -os.chdir(entry_pwd) - -for fn in glob.glob(os.path.join('src', '*')): - copy_file(fn, os.path.join(dist, 'src-input', os.path.basename(fn))) - -os.chdir(os.path.join(entry_pwd, 'config')) -create_targz(os.path.join(dist, 'config', 'genconfig_metadata.tar.gz'), [ - 'tags.yaml', - 'platforms.yaml', - 'architectures.yaml', - 'compilers.yaml', - 'platforms', - 'architectures', - 'compilers', - 'feature-options', - 'config-options', - 'helper-snippets', - 'header-snippets', - 'other-defines', - 'examples' -]) -os.chdir(entry_pwd) - -copy_files([ - 'prepare_sources.py', - 'combine_src.py', - 'create_spdx_license.py', - 'duk_meta_to_strarray.py', - 'dukutil.py', - 'dump_bytecode.py', - 'extract_caseconv.py', - 'extract_chars.py', - 'extract_unique_options.py', - 'genbuildparams.py', - 'genbuiltins.py', - 'genconfig.py', - 'json2yaml.py', - 'merge_debug_meta.py', - 'prepare_unicode_data.py', - 'resolve_combined_lineno.py', - 'scan_strings.py', - 'scan_used_stridx_bidx.py', - 'yaml2json.py', -], 'tools', os.path.join(dist, 'tools')) - -# XXX: Copy genconfig.py also to config/genconfig.py for now. -copy_file(os.path.join(dist, 'tools', 'genconfig.py'), os.path.join(dist, 'config', 'genconfig.py')) - -copy_files([ - 'README.rst' -], 'config', os.path.join(dist, 'config')) - -copy_files([ - 'README.rst', - 'Makefile', - 'package.json', - 'duk_debug.js', - 'duk_debug_proxy.js', - 'duk_classnames.yaml', - 'duk_debugcommands.yaml', - 'duk_debugerrors.yaml', - 'duk_opcodes.yaml' -], 'debugger', os.path.join(dist, 'debugger')) -copy_files([ - 'index.html', - 'style.css', - 'webui.js' -], os.path.join('debugger', 'static'), os.path.join(dist, 'debugger', 'static')) - -copy_files([ - 'console-minimal.js', - 'object-prototype-definegetter.js', - 'object-prototype-definesetter.js', - 'object-assign.js', - 'performance-now.js', - 'duktape-isfastint.js', - 'duktape-error-setter-writable.js', - 'duktape-error-setter-nonwritable.js', - 'duktape-buffer.js' -], 'polyfills', os.path.join(dist, 'polyfills')) - -copy_files([ - 'README.rst' -], 'examples', os.path.join(dist, 'examples')) - -copy_files([ - 'README.rst', - 'duk_cmdline.c', - 'duk_cmdline_ajduk.c' -], os.path.join('examples', 'cmdline'), os.path.join(dist, 'examples', 'cmdline')) - -copy_files([ - 'README.rst', - 'c_eventloop.c', - 'c_eventloop.js', - 'ecma_eventloop.js', - 'main.c', - 'poll.c', - 'ncurses.c', - 'socket.c', - 'fileio.c', - 'curses-timers.js', - 'basic-test.js', - 'server-socket-test.js', - 'client-socket-test.js' -], os.path.join('examples', 'eventloop'), os.path.join(dist, 'examples', 'eventloop')) - -copy_files([ - 'README.rst', - 'hello.c' -], os.path.join('examples', 'hello'), os.path.join(dist, 'examples', 'hello')) - -copy_files([ - 'README.rst', - 'eval.c' -], os.path.join('examples', 'eval'), os.path.join(dist, 'examples', 'eval')) - -copy_files([ - 'README.rst', - 'fib.js', - 'process.js', - 'processlines.c', - 'prime.js', - 'primecheck.c', - 'uppercase.c' -], os.path.join('examples', 'guide'), os.path.join(dist, 'examples', 'guide')) - -copy_files([ - 'README.rst', - 'globals.coffee', - 'hello.coffee', - 'mandel.coffee' -], os.path.join('examples', 'coffee'), os.path.join(dist, 'examples', 'coffee')) - -copy_files([ - 'README.rst', - 'jxpretty.c' -], os.path.join('examples', 'jxpretty'), os.path.join(dist, 'examples', 'jxpretty')) - -copy_files([ - 'README.rst', - 'sandbox.c' -], os.path.join('examples', 'sandbox'), os.path.join(dist, 'examples', 'sandbox')) - -copy_files([ - 'README.rst', - 'duk_alloc_logging.c', - 'duk_alloc_logging.h', - 'log2gnuplot.py' -], os.path.join('examples', 'alloc-logging'), os.path.join(dist, 'examples', 'alloc-logging')) - -copy_files([ - 'README.rst', - 'duk_alloc_torture.c', - 'duk_alloc_torture.h' -], os.path.join('examples', 'alloc-torture'), os.path.join(dist, 'examples', 'alloc-torture')) - -copy_files([ - 'README.rst', - 'duk_alloc_hybrid.c', - 'duk_alloc_hybrid.h' -], os.path.join('examples', 'alloc-hybrid'), os.path.join(dist, 'examples', 'alloc-hybrid')) - -copy_files([ - 'README.rst', - 'duk_trans_socket_unix.c', - 'duk_trans_socket_windows.c', - 'duk_trans_socket.h' -], os.path.join('examples', 'debug-trans-socket'), os.path.join(dist, 'examples', 'debug-trans-socket')) - -copy_files([ - 'README.rst', - 'duk_trans_dvalue.c', - 'duk_trans_dvalue.h', - 'test.c', - 'Makefile' -], os.path.join('examples', 'debug-trans-dvalue'), os.path.join(dist, 'examples', 'debug-trans-dvalue')) - -copy_files([ - 'README.rst', - 'duk_codepage_conv.c', - 'duk_codepage_conv.h', - 'test.c' -], os.path.join('examples', 'codepage-conv'), os.path.join(dist, 'examples', 'codepage-conv')) - -copy_files([ - 'README.rst', - 'dummy_date_provider.c' -], os.path.join('examples', 'dummy-date-provider'), os.path.join(dist, 'examples', 'dummy-date-provider')) - -copy_files([ - 'README.rst', - 'cpp_exceptions.cpp' -], os.path.join('examples', 'cpp-exceptions'), os.path.join(dist, 'examples', 'cpp-exceptions')) - -copy_files([ - 'README.rst' -], 'extras', os.path.join(dist, 'extras')) - - -copy_files([ - 'README.rst', - 'duk_logging.c', - 'duk_logging.h', - 'test.c', - 'Makefile' -], os.path.join('extras', 'logging'), os.path.join(dist, 'extras', 'logging')) - -copy_files([ - 'README.rst', - 'duk_v1_compat.c', - 'duk_v1_compat.h', - 'test.c', - 'Makefile', - 'test_eval1.js', - 'test_eval2.js', - 'test_compile1.js', - 'test_compile2.js' -], os.path.join('extras', 'duk-v1-compat'), os.path.join(dist, 'extras', 'duk-v1-compat')) - -copy_files([ - 'README.rst', - 'duk_print_alert.c', - 'duk_print_alert.h', - 'test.c', - 'Makefile' -], os.path.join('extras', 'print-alert'), os.path.join(dist, 'extras', 'print-alert')) - -copy_files([ - 'README.rst', - 'duk_console.c', - 'duk_console.h', - 'test.c', - 'Makefile' -], os.path.join('extras', 'console'), os.path.join(dist, 'extras', 'console')) - -copy_files([ - 'README.rst', - 'duk_minimal_printf.c', - 'duk_minimal_printf.h', - 'Makefile', - 'test.c' -], os.path.join('extras', 'minimal-printf'), os.path.join(dist, 'extras', 'minimal-printf')) - -copy_files([ - 'README.rst', - 'duk_module_duktape.c', - 'duk_module_duktape.h', - 'Makefile', - 'test.c' -], os.path.join('extras', 'module-duktape'), os.path.join(dist, 'extras', 'module-duktape')) - -copy_files([ - 'README.rst', - 'duk_module_node.c', - 'duk_module_node.h', - 'Makefile', - 'test.c' -], os.path.join('extras', 'module-node'), os.path.join(dist, 'extras', 'module-node')) - -copy_files([ - 'README.rst', - 'duk_alloc_pool.c', - 'duk_alloc_pool.h', - 'ptrcomp.yaml', - 'ptrcomp_fixup.h', - 'Makefile', - 'test.c' -], os.path.join('extras', 'alloc-pool'), os.path.join(dist, 'extras', 'alloc-pool')) - -copy_files([ - 'Makefile.cmdline', - 'Makefile.dukdebug', - 'Makefile.eventloop', - 'Makefile.hello', - 'Makefile.eval', - 'Makefile.coffee', - 'Makefile.jxpretty', - 'Makefile.sandbox', - 'Makefile.codepage', - 'mandel.js' -], 'dist-files', dist) -copy_and_replace(os.path.join('dist-files', 'Makefile.sharedlibrary'), os.path.join(dist, 'Makefile.sharedlibrary'), { - '@DUK_VERSION@': str(duk_version), - '@SONAME_VERSION@': str(int(duk_version / 100)) # 10500 -> 105 -}) - -copy_and_replace(os.path.join('dist-files', 'README.rst'), os.path.join(dist, 'README.rst'), { - '@DUK_VERSION_FORMATTED@': duk_version_formatted, - '@GIT_COMMIT@': git_commit, - '@GIT_DESCRIBE@': git_describe, - '@GIT_BRANCH@': git_branch -}) - -copy_files([ - 'LICENSE.txt', # not strict RST so keep .txt suffix - 'AUTHORS.rst' -], '.', os.path.join(dist)) - -# RELEASES.rst is only updated in master. It's not included in the dist to -# make maintenance fixes easier to make. - -copy_files([ - 'murmurhash2.txt', - 'lua.txt', - 'commonjs.txt' -], 'licenses', os.path.join(dist, 'licenses')) - -# Merge debugger metadata. -merged = exec_print_stdout([ - sys.executable, os.path.join('tools', 'merge_debug_meta.py'), - '--output', os.path.join(dist, 'debugger', 'duk_debug_meta.json'), - '--class-names', os.path.join('debugger', 'duk_classnames.yaml'), - '--debug-commands', os.path.join('debugger', 'duk_debugcommands.yaml'), - '--debug-errors', os.path.join('debugger', 'duk_debugerrors.yaml'), - '--opcodes', os.path.join('debugger', 'duk_opcodes.yaml') -]) - -print('Create duk_config.h headers') - -# Build duk_config.h without feature option support. -exec_print_stdout([ - sys.executable, os.path.join('tools', 'genconfig.py'), '--metadata', 'config', - '--output', os.path.join(dist, 'config', 'duk_config.h-modular-static'), - '--git-commit', git_commit, '--git-describe', git_describe, '--git-branch', git_branch, - '--omit-removed-config-options', '--omit-unused-config-options', - '--emit-legacy-feature-check', '--emit-config-sanity-check', - 'duk-config-header' -]) -exec_print_stdout([ - sys.executable, os.path.join('tools', 'genconfig.py'), '--metadata', 'config', - '--output', os.path.join(dist, 'config', 'duk_config.h-modular-dll'), - '--git-commit', git_commit, '--git-describe', git_describe, '--git-branch', git_branch, - '--omit-removed-config-options', '--omit-unused-config-options', - '--emit-legacy-feature-check', '--emit-config-sanity-check', - '--dll', - 'duk-config-header' -]) - -# Generate a few barebones config examples -def genconfig_barebones(platform, architecture, compiler): +def main(): + # Basic option parsing, Python module check, CWD check. + + opts, args = parse_options() + check_python_modules(opts) + check_cwd_duktape_repo_root() + + # Obsolete options check. + + if opts.rom_support or opts.rom_auto_lightfunc or len(opts.user_builtin_metadata) > 0: + raise Exception('obsolete ROM support argument(s), use tools/prepare_sources.py instead') + + # Figure out directories, git info, Duktape version, etc. + + entry_pwd = os.getcwd() + dist = os.path.join(entry_pwd, 'dist') + + duk_version, duk_major, duk_minor, duk_patch, duk_version_formatted = get_duk_version() + + if opts.git_commit is not None: + git_commit = opts.git_commit + else: + git_commit = exec_get_stdout([ 'git', 'rev-parse', 'HEAD' ], default='external').strip() + if opts.git_describe is not None: + git_describe = opts.git_describe + else: + git_describe = exec_get_stdout([ 'git', 'describe', '--always', '--dirty' ], default='external').strip() + if opts.git_branch is not None: + git_branch = opts.git_branch + else: + git_branch = exec_get_stdout([ 'git', 'rev-parse', '--abbrev-ref', 'HEAD' ], default='external').strip() + + git_commit_cstring = cstring(git_commit) + git_describe_cstring = cstring(git_describe) + git_branch_cstring = cstring(git_branch) + + print('Dist for Duktape version %s, commit %s, describe %s, branch %s' % \ + (duk_version_formatted, git_commit, git_describe, git_branch)) + + # Create dist directory structure, copy files. + + print('Create dist directories and copy static files') + + create_dist_directories(dist) + + os.chdir(entry_pwd) + + copy_files([ + 'builtins.yaml', + 'duk_alloc_default.c', + 'duk_api_buffer.c', + 'duk_api_bytecode.c', + 'duk_api_call.c', + 'duk_api_codec.c', + 'duk_api_compile.c', + 'duk_api_debug.c', + 'duk_api_heap.c', + 'duk_api_internal.h', + 'duk_api_memory.c', + 'duk_api_object.c', + 'duk_api_public.h.in', + 'duk_api_stack.c', + 'duk_api_string.c', + 'duk_api_time.c', + 'duk_bi_array.c', + 'duk_bi_boolean.c', + 'duk_bi_buffer.c', + 'duk_bi_date.c', + 'duk_bi_date_unix.c', + 'duk_bi_date_windows.c', + 'duk_bi_duktape.c', + 'duk_bi_error.c', + 'duk_bi_function.c', + 'duk_bi_global.c', + 'duk_bi_json.c', + 'duk_bi_math.c', + 'duk_bi_number.c', + 'duk_bi_object.c', + 'duk_bi_pointer.c', + 'duk_bi_protos.h', + 'duk_bi_proxy.c', + 'duk_bi_regexp.c', + 'duk_bi_string.c', + 'duk_bi_thread.c', + 'duk_bi_thrower.c', + 'duk_dblunion.h.in', + 'duk_debug_fixedbuffer.c', + 'duk_debugger.c', + 'duk_debugger.h', + 'duk_debug.h', + 'duk_debug_macros.c', + 'duk_debug_vsnprintf.c', + 'duk_error_augment.c', + 'duk_error.h', + 'duk_error_longjmp.c', + 'duk_error_macros.c', + 'duk_error_misc.c', + 'duk_error_throw.c', + 'duk_exception.h', + 'duk_forwdecl.h', + 'duk_harray.h', + 'duk_hbuffer_alloc.c', + 'duk_hbuffer.h', + 'duk_hbuffer_ops.c', + 'duk_hbufobj.h', + 'duk_hbufobj_misc.c', + 'duk_hcompfunc.h', + 'duk_heap_alloc.c', + 'duk_heap.h', + 'duk_heap_hashstring.c', + 'duk_heaphdr.h', + 'duk_heap_markandsweep.c', + 'duk_heap_memory.c', + 'duk_heap_misc.c', + 'duk_heap_refcount.c', + 'duk_heap_stringcache.c', + 'duk_heap_stringtable.c', + 'duk_hnatfunc.h', + 'duk_hobject_alloc.c', + 'duk_hobject_class.c', + 'duk_hobject_enum.c', + 'duk_hobject_finalizer.c', + 'duk_hobject.h', + 'duk_hobject_misc.c', + 'duk_hobject_pc2line.c', + 'duk_hobject_props.c', + 'duk_hstring.h', + 'duk_hstring_misc.c', + 'duk_hthread_alloc.c', + 'duk_hthread_builtins.c', + 'duk_hthread.h', + 'duk_hthread_misc.c', + 'duk_hthread_stacks.c', + 'duk_internal.h', + 'duk_jmpbuf.h', + 'duk_js_bytecode.h', + 'duk_js_call.c', + 'duk_js_compiler.c', + 'duk_js_compiler.h', + 'duk_js_executor.c', + 'duk_js.h', + 'duk_json.h', + 'duk_js_ops.c', + 'duk_js_var.c', + 'duk_lexer.c', + 'duk_lexer.h', + 'duk_numconv.c', + 'duk_numconv.h', + 'duk_regexp_compiler.c', + 'duk_regexp_executor.c', + 'duk_regexp.h', + 'duk_replacements.c', + 'duk_replacements.h', + 'duk_selftest.c', + 'duk_selftest.h', + 'duk_strings.h', + 'duktape.h.in', + 'duk_tval.c', + 'duk_tval.h', + 'duk_unicode.h', + 'duk_unicode_support.c', + 'duk_unicode_tables.c', + 'duk_util_bitdecoder.c', + 'duk_util_bitencoder.c', + 'duk_util_bufwriter.c', + 'duk_util.h', + 'duk_util_hashbytes.c', + 'duk_util_hashprime.c', + 'duk_util_misc.c', + 'duk_util_tinyrandom.c', + 'strings.yaml', + 'SpecialCasing.txt', + 'SpecialCasing-8bit.txt', + 'UnicodeData.txt', + 'UnicodeData-8bit.txt', + ], 'src', os.path.join(dist, 'src-input')) + + os.chdir(os.path.join(entry_pwd, 'config')) + create_targz(os.path.join(dist, 'config', 'genconfig_metadata.tar.gz'), [ + 'tags.yaml', + 'platforms.yaml', + 'architectures.yaml', + 'compilers.yaml', + 'platforms', + 'architectures', + 'compilers', + 'feature-options', + 'config-options', + 'helper-snippets', + 'header-snippets', + 'other-defines', + 'examples' + ]) + os.chdir(entry_pwd) + + copy_files([ + 'prepare_sources.py', + 'combine_src.py', + 'create_spdx_license.py', + 'duk_meta_to_strarray.py', + 'dukutil.py', + 'dump_bytecode.py', + 'extract_caseconv.py', + 'extract_chars.py', + 'extract_unique_options.py', + 'genbuiltins.py', + 'genconfig.py', + 'json2yaml.py', + 'merge_debug_meta.py', + 'prepare_unicode_data.py', + 'resolve_combined_lineno.py', + 'scan_strings.py', + 'scan_used_stridx_bidx.py', + 'yaml2json.py', + ], 'tools', os.path.join(dist, 'tools')) + + copy_files([ + 'README.rst' + ], 'config', os.path.join(dist, 'config')) + + copy_files([ + 'README.rst', + 'Makefile', + 'package.json', + 'duk_debug.js', + 'duk_debug_proxy.js', + 'duk_classnames.yaml', + 'duk_debugcommands.yaml', + 'duk_debugerrors.yaml', + 'duk_opcodes.yaml' + ], 'debugger', os.path.join(dist, 'debugger')) + + copy_files([ + 'index.html', + 'style.css', + 'webui.js' + ], os.path.join('debugger', 'static'), os.path.join(dist, 'debugger', 'static')) + + copy_files([ + 'console-minimal.js', + 'object-prototype-definegetter.js', + 'object-prototype-definesetter.js', + 'object-assign.js', + 'performance-now.js', + 'duktape-isfastint.js', + 'duktape-error-setter-writable.js', + 'duktape-error-setter-nonwritable.js', + 'duktape-buffer.js' + ], 'polyfills', os.path.join(dist, 'polyfills')) + + copy_files([ + 'README.rst' + ], 'examples', os.path.join(dist, 'examples')) + + copy_files([ + 'README.rst', + 'duk_cmdline.c', + 'duk_cmdline_ajduk.c' + ], os.path.join('examples', 'cmdline'), os.path.join(dist, 'examples', 'cmdline')) + + copy_files([ + 'README.rst', + 'c_eventloop.c', + 'c_eventloop.js', + 'ecma_eventloop.js', + 'main.c', + 'poll.c', + 'ncurses.c', + 'socket.c', + 'fileio.c', + 'curses-timers.js', + 'basic-test.js', + 'server-socket-test.js', + 'client-socket-test.js' + ], os.path.join('examples', 'eventloop'), os.path.join(dist, 'examples', 'eventloop')) + + copy_files([ + 'README.rst', + 'hello.c' + ], os.path.join('examples', 'hello'), os.path.join(dist, 'examples', 'hello')) + + copy_files([ + 'README.rst', + 'eval.c' + ], os.path.join('examples', 'eval'), os.path.join(dist, 'examples', 'eval')) + + copy_files([ + 'README.rst', + 'fib.js', + 'process.js', + 'processlines.c', + 'prime.js', + 'primecheck.c', + 'uppercase.c' + ], os.path.join('examples', 'guide'), os.path.join(dist, 'examples', 'guide')) + + copy_files([ + 'README.rst', + 'globals.coffee', + 'hello.coffee', + 'mandel.coffee' + ], os.path.join('examples', 'coffee'), os.path.join(dist, 'examples', 'coffee')) + + copy_files([ + 'README.rst', + 'jxpretty.c' + ], os.path.join('examples', 'jxpretty'), os.path.join(dist, 'examples', 'jxpretty')) + + copy_files([ + 'README.rst', + 'sandbox.c' + ], os.path.join('examples', 'sandbox'), os.path.join(dist, 'examples', 'sandbox')) + + copy_files([ + 'README.rst', + 'duk_alloc_logging.c', + 'duk_alloc_logging.h', + 'log2gnuplot.py' + ], os.path.join('examples', 'alloc-logging'), os.path.join(dist, 'examples', 'alloc-logging')) + + copy_files([ + 'README.rst', + 'duk_alloc_torture.c', + 'duk_alloc_torture.h' + ], os.path.join('examples', 'alloc-torture'), os.path.join(dist, 'examples', 'alloc-torture')) + + copy_files([ + 'README.rst', + 'duk_alloc_hybrid.c', + 'duk_alloc_hybrid.h' + ], os.path.join('examples', 'alloc-hybrid'), os.path.join(dist, 'examples', 'alloc-hybrid')) + + copy_files([ + 'README.rst', + 'duk_trans_socket_unix.c', + 'duk_trans_socket_windows.c', + 'duk_trans_socket.h' + ], os.path.join('examples', 'debug-trans-socket'), os.path.join(dist, 'examples', 'debug-trans-socket')) + + copy_files([ + 'README.rst', + 'duk_trans_dvalue.c', + 'duk_trans_dvalue.h', + 'test.c', + 'Makefile' + ], os.path.join('examples', 'debug-trans-dvalue'), os.path.join(dist, 'examples', 'debug-trans-dvalue')) + + copy_files([ + 'README.rst', + 'duk_codepage_conv.c', + 'duk_codepage_conv.h', + 'test.c' + ], os.path.join('examples', 'codepage-conv'), os.path.join(dist, 'examples', 'codepage-conv')) + + copy_files([ + 'README.rst', + 'dummy_date_provider.c' + ], os.path.join('examples', 'dummy-date-provider'), os.path.join(dist, 'examples', 'dummy-date-provider')) + + copy_files([ + 'README.rst', + 'cpp_exceptions.cpp' + ], os.path.join('examples', 'cpp-exceptions'), os.path.join(dist, 'examples', 'cpp-exceptions')) + + copy_files([ + 'README.rst' + ], 'extras', os.path.join(dist, 'extras')) + + copy_files([ + 'README.rst', + 'duk_logging.c', + 'duk_logging.h', + 'test.c', + 'Makefile' + ], os.path.join('extras', 'logging'), os.path.join(dist, 'extras', 'logging')) + + copy_files([ + 'README.rst', + 'duk_v1_compat.c', + 'duk_v1_compat.h', + 'test.c', + 'Makefile', + 'test_eval1.js', + 'test_eval2.js', + 'test_compile1.js', + 'test_compile2.js' + ], os.path.join('extras', 'duk-v1-compat'), os.path.join(dist, 'extras', 'duk-v1-compat')) + + copy_files([ + 'README.rst', + 'duk_print_alert.c', + 'duk_print_alert.h', + 'test.c', + 'Makefile' + ], os.path.join('extras', 'print-alert'), os.path.join(dist, 'extras', 'print-alert')) + + copy_files([ + 'README.rst', + 'duk_console.c', + 'duk_console.h', + 'test.c', + 'Makefile' + ], os.path.join('extras', 'console'), os.path.join(dist, 'extras', 'console')) + + copy_files([ + 'README.rst', + 'duk_minimal_printf.c', + 'duk_minimal_printf.h', + 'Makefile', + 'test.c' + ], os.path.join('extras', 'minimal-printf'), os.path.join(dist, 'extras', 'minimal-printf')) + + copy_files([ + 'README.rst', + 'duk_module_duktape.c', + 'duk_module_duktape.h', + 'Makefile', + 'test.c' + ], os.path.join('extras', 'module-duktape'), os.path.join(dist, 'extras', 'module-duktape')) + + copy_files([ + 'README.rst', + 'duk_module_node.c', + 'duk_module_node.h', + 'Makefile', + 'test.c' + ], os.path.join('extras', 'module-node'), os.path.join(dist, 'extras', 'module-node')) + + copy_files([ + 'README.rst', + 'duk_alloc_pool.c', + 'duk_alloc_pool.h', + 'ptrcomp.yaml', + 'ptrcomp_fixup.h', + 'Makefile', + 'test.c' + ], os.path.join('extras', 'alloc-pool'), os.path.join(dist, 'extras', 'alloc-pool')) + + copy_files([ + 'Makefile.cmdline', + 'Makefile.dukdebug', + 'Makefile.eventloop', + 'Makefile.hello', + 'Makefile.eval', + 'Makefile.coffee', + 'Makefile.jxpretty', + 'Makefile.sandbox', + 'Makefile.codepage', + 'mandel.js' + ], 'dist-files', dist) + + copy_and_replace(os.path.join('dist-files', 'Makefile.sharedlibrary'), os.path.join(dist, 'Makefile.sharedlibrary'), { + '@DUK_VERSION@': str(duk_version), + '@SONAME_VERSION@': str(int(duk_version / 100)) # 10500 -> 105 + }) + + copy_and_replace(os.path.join('dist-files', 'README.rst'), os.path.join(dist, 'README.rst'), { + '@DUK_VERSION_FORMATTED@': duk_version_formatted, + '@GIT_COMMIT@': git_commit, + '@GIT_DESCRIBE@': git_describe, + '@GIT_BRANCH@': git_branch + }) + + copy_files([ + 'LICENSE.txt', # not strict RST so keep .txt suffix + 'AUTHORS.rst' + ], '.', os.path.join(dist)) + + # RELEASES.rst is only updated in master. It's not included in the dist to + # make maintenance fixes easier to make. + + copy_files([ + 'murmurhash2.txt', + 'lua.txt', + 'commonjs.txt' + ], 'licenses', os.path.join(dist, 'licenses')) + + # Merge debugger metadata. + + merged = exec_print_stdout([ + sys.executable, os.path.join('tools', 'merge_debug_meta.py'), + '--output', os.path.join(dist, 'debugger', 'duk_debug_meta.json'), + '--class-names', os.path.join('debugger', 'duk_classnames.yaml'), + '--debug-commands', os.path.join('debugger', 'duk_debugcommands.yaml'), + '--debug-errors', os.path.join('debugger', 'duk_debugerrors.yaml'), + '--opcodes', os.path.join('debugger', 'duk_opcodes.yaml') + ]) + + # Build some example duk_config.h headers. This may go away later. + + print('Create duk_config.h headers') + exec_print_stdout([ sys.executable, os.path.join('tools', 'genconfig.py'), '--metadata', 'config', - '--output', os.path.join(dist, 'config', 'duk_config.h-%s-%s-%s' % (platform, architecture, compiler)), + '--output', os.path.join(dist, 'config', 'duk_config.h-modular-static'), '--git-commit', git_commit, '--git-describe', git_describe, '--git-branch', git_branch, - '--platform', platform, '--architecture', architecture, '--compiler', compiler, '--omit-removed-config-options', '--omit-unused-config-options', '--emit-legacy-feature-check', '--emit-config-sanity-check', 'duk-config-header' ]) -#genconfig_barebones('linux', 'x86', 'gcc') -#genconfig_barebones('linux', 'x64', 'gcc') -#genconfig_barebones('linux', 'x86', 'clang') -#genconfig_barebones('linux', 'x64', 'clang') -#genconfig_barebones('windows', 'x86', 'msvc') -#genconfig_barebones('windows', 'x64', 'msvc') -#genconfig_barebones('apple', 'x86', 'gcc') -#genconfig_barebones('apple', 'x64', 'gcc') -#genconfig_barebones('apple', 'x86', 'clang') -#genconfig_barebones('apple', 'x64', 'clang') - -# Build prepared sources (src/, src-noline/, src-separate/) with default -# config. This is done using tools and metadata in the dist directory. -print('Config-and-prepare sources for default configuration') -cmd = [ - sys.executable, os.path.join(dist, 'tools', 'prepare_sources.py'), - '--source-directory', os.path.join(dist, 'src-input'), - '--output-directory', dist, - '--config-metadata', os.path.join(dist, 'config', 'genconfig_metadata.tar.gz'), - '--git-commit', git_commit, '--git-describe', git_describe, '--git-branch', git_branch, - '--omit-removed-config-options', '--omit-unused-config-options', - '--emit-config-sanity-check', '--support-feature-options' -] -if opts.rom_support: - cmd.append('--rom-support') -if opts.rom_auto_lightfunc: - cmd.append('--rom-auto-lightfunc') -for i in opts.user_builtin_metadata: - cmd.append('--user-builtin-metadata') - cmd.append(i) -exec_print_stdout(cmd) - -# Clean up remaining temp files -delete_matching_files(dist, lambda x: x[-4:] == '.tmp') - -# Create SPDX license once all other files are in place (and cleaned) -if opts.create_spdx: - print('Create SPDX license') - try: - exec_get_stdout([ - sys.executable, - os.path.join('tools', 'create_spdx_license.py'), - os.path.join(dist, 'license.spdx') + exec_print_stdout([ + sys.executable, os.path.join('tools', 'genconfig.py'), '--metadata', 'config', + '--output', os.path.join(dist, 'config', 'duk_config.h-modular-dll'), + '--git-commit', git_commit, '--git-describe', git_describe, '--git-branch', git_branch, + '--omit-removed-config-options', '--omit-unused-config-options', + '--emit-legacy-feature-check', '--emit-config-sanity-check', + '--dll', + 'duk-config-header' + ]) + + def genconfig_barebones(platform, architecture, compiler): + exec_print_stdout([ + sys.executable, os.path.join('tools', 'genconfig.py'), '--metadata', 'config', + '--output', os.path.join(dist, 'config', 'duk_config.h-%s-%s-%s' % (platform, architecture, compiler)), + '--git-commit', git_commit, '--git-describe', git_describe, '--git-branch', git_branch, + '--platform', platform, '--architecture', architecture, '--compiler', compiler, + '--omit-removed-config-options', '--omit-unused-config-options', + '--emit-legacy-feature-check', '--emit-config-sanity-check', + 'duk-config-header' ]) - except: - print('') - print('***') - print('*** WARNING: Failed to create SPDX license, this should not happen for an official release!') - print('***') - print('') -else: - print('Skip SPDX license creation') - -print('Dist finished successfully') + + #genconfig_barebones('linux', 'x86', 'gcc') + #genconfig_barebones('linux', 'x64', 'gcc') + #genconfig_barebones('linux', 'x86', 'clang') + #genconfig_barebones('linux', 'x64', 'clang') + #genconfig_barebones('windows', 'x86', 'msvc') + #genconfig_barebones('windows', 'x64', 'msvc') + #genconfig_barebones('apple', 'x86', 'gcc') + #genconfig_barebones('apple', 'x64', 'gcc') + #genconfig_barebones('apple', 'x86', 'clang') + #genconfig_barebones('apple', 'x64', 'clang') + + # Build prepared sources (src/, src-noline/, src-separate/) with default + # config. This is done using tools and metadata in the dist directory. + + print('Config-and-prepare sources for default configuration') + + cmd = [ + sys.executable, os.path.join(dist, 'tools', 'prepare_sources.py'), + '--source-directory', os.path.join(dist, 'src-input'), + '--output-directory', dist, + '--config-metadata', os.path.join(dist, 'config', 'genconfig_metadata.tar.gz'), + '--git-commit', git_commit, '--git-describe', git_describe, '--git-branch', git_branch, + '--omit-removed-config-options', '--omit-unused-config-options', + '--emit-config-sanity-check', '--support-feature-options' + ] + if opts.rom_support: + cmd.append('--rom-support') + if opts.rom_auto_lightfunc: + cmd.append('--rom-auto-lightfunc') + for i in opts.user_builtin_metadata: + cmd.append('--user-builtin-metadata') + cmd.append(i) + exec_print_stdout(cmd) + + # Clean up remaining temp files. + + delete_matching_files(dist, lambda x: x[-4:] == '.tmp') + + # Create SPDX license once all other files are in place (and cleaned). + + if opts.create_spdx: + print('Create SPDX license') + try: + exec_get_stdout([ + sys.executable, + os.path.join('tools', 'create_spdx_license.py'), + os.path.join(dist, 'license.spdx') + ]) + except: + print('') + print('***') + print('*** WARNING: Failed to create SPDX license, this should not happen for an official release!') + print('***') + print('') + else: + print('Skip SPDX license creation') + + print('Dist finished successfully') + +if __name__ == '__main__': + main() diff --git a/util/prep_test.py b/util/prep_test.py index f360bffb..6c5af6cb 100644 --- a/util/prep_test.py +++ b/util/prep_test.py @@ -4,7 +4,7 @@ # # For Ecmascript testcases: # -# - Lift a 'use global' statement to top of file if present +# - Lift a 'use strict' statement to top of file if present # - Add a prologue which handles engine differences # - Resolve include files # diff --git a/util/underscore_test.sh b/util/underscore_test.sh index b108ced6..47cea6b9 100755 --- a/util/underscore_test.sh +++ b/util/underscore_test.sh @@ -19,6 +19,7 @@ else fi cat util/underscore-test-shim.js \ + underscore/test/vendor/qunit.js \ underscore/underscore.js \ $TEST \ > /tmp/duk-underscore-test.js From 9eed7d063ec3b8cd66228bbe989174afd8af50da Mon Sep 17 00:00:00 2001 From: Sami Vaarala Date: Sat, 27 Aug 2016 05:46:15 +0300 Subject: [PATCH 5/9] Add a few built-in removal YAML files --- util/remove_bufferobject_properties.yaml | 64 ++++++++++++++++++++++++ util/remove_math_properties.yaml | 8 +++ 2 files changed, 72 insertions(+) create mode 100644 util/remove_bufferobject_properties.yaml create mode 100644 util/remove_math_properties.yaml diff --git a/util/remove_bufferobject_properties.yaml b/util/remove_bufferobject_properties.yaml new file mode 100644 index 00000000..f199baa9 --- /dev/null +++ b/util/remove_bufferobject_properties.yaml @@ -0,0 +1,64 @@ +objects: + + # The bufferobject related built-ins have a DUK_BIDX_xxx constant so they + # must at present map to an object value (they may not be missing or map + # to e.g. null). + + - id: bi_empty_bufferobject + class: Object + internal_prototype: bi_object_prototype + + properties: [] + + - id: bi_global + modify: true + + properties: + - key: 'DataView' + value: + type: object + id: bi_empty_bufferobject + - key: 'Uint8Array' + value: + type: object + id: bi_empty_bufferobject + - key: 'Uint8ClampedArray' + value: + type: object + id: bi_empty_bufferobject + - key: 'Int8Array' + value: + type: object + id: bi_empty_bufferobject + - key: 'Uint16Array' + value: + type: object + id: bi_empty_bufferobject + - key: 'Int16Array' + value: + type: object + id: bi_empty_bufferobject + - key: 'Uint32Array' + value: + type: object + id: bi_empty_bufferobject + - key: 'Int32Array' + value: + type: object + id: bi_empty_bufferobject + - key: 'Float32Array' + value: + type: object + id: bi_empty_bufferobject + - key: 'Float64Array' + value: + type: object + id: bi_empty_bufferobject + - key: 'ArrayBuffer' + value: + type: object + id: bi_empty_bufferobject + - key: 'Buffer' + value: + type: object + id: bi_empty_bufferobject diff --git a/util/remove_math_properties.yaml b/util/remove_math_properties.yaml new file mode 100644 index 00000000..d03955c6 --- /dev/null +++ b/util/remove_math_properties.yaml @@ -0,0 +1,8 @@ +objects: + + - id: bi_math + replace: true + class: Object + internal_prototype: bi_object_prototype + + properties: [] From 690f81f1668dd5f4f9fc6ed81cb178790ee1051b Mon Sep 17 00:00:00 2001 From: Sami Vaarala Date: Sat, 27 Aug 2016 03:58:49 +0300 Subject: [PATCH 6/9] Low memory document: Unicode tables --- doc/low-memory.rst | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/low-memory.rst b/doc/low-memory.rst index 0c66fd65..e6ffae1f 100644 --- a/doc/low-memory.rst +++ b/doc/low-memory.rst @@ -158,8 +158,8 @@ Miscellaneous stub out unnecessary functions in ``duk_config.h``. Note, however, that Duktape internals at present depend on a few Math functions like ``DUK_FMOD()``. -Suggested feature options -========================= +Suggested options +================= * Use the default memory management settings: although reference counting increases heap header size, it also reduces memory usage fluctuation @@ -232,6 +232,17 @@ Suggested feature options - ``-DDUK_OPT_DEBUG_BUFSIZE=2048`` +* If strict Unicode support is not critical in your application, you can: + + - Strip the ``UnicodeData.txt`` and ``SpecialCasing.txt`` files manually. + There are example files in the distributable for Unicode data limited + to 8-bit codepoints. + + - Provide the stripped files to ``prepare_sources.py`` to reduce Unicode + table size. + + - Possible footprint savings are about 2-3kB. + More aggressive options ======================= From fe64dd0836215ef88f0495a17162e8cc3d065322 Mon Sep 17 00:00:00 2001 From: Sami Vaarala Date: Sat, 27 Aug 2016 21:34:58 +0300 Subject: [PATCH 7/9] Fix some source and doc refs to relocated tools --- doc/bytecode.rst | 2 +- doc/code-issues.rst | 2 +- doc/release-checklist.rst | 2 +- examples/cmdline/duk_cmdline_ajduk.c | 6 +++--- src/duk_bi_date.c | 2 +- src/duk_heap_hashstring.c | 2 +- src/duk_numconv.c | 2 +- src/duk_unicode_support.c | 14 +++++++------- src/duk_unicode_tables.c | 4 ++-- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/doc/bytecode.rst b/doc/bytecode.rst index 8fb4734a..2b8e0553 100644 --- a/doc/bytecode.rst +++ b/doc/bytecode.rst @@ -334,7 +334,7 @@ The exact format is ultimately defined by the source code, see: * ``src/duk_api_bytecode.c`` -* ``util/dump_bytecode.py`` +* ``tools/dump_bytecode.py`` As a simplified summary of the bytecode format: diff --git a/doc/code-issues.rst b/doc/code-issues.rst index 758460fe..b6a96a2d 100644 --- a/doc/code-issues.rst +++ b/doc/code-issues.rst @@ -811,7 +811,7 @@ The ``DUK_INTERNAL_DECL`` idiom is:: DUK_INTERNAL_DECL const char *duk_str_not_object; #endif /* !DUK_SINGLE_FILE */ -For this to work in the single file case, ``util/combine_src.py`` must +For this to work in the single file case, ``tools/combine_src.py`` must ensure that the symbol definition appears before its use. This is currently handled via manual file reordering. diff --git a/doc/release-checklist.rst b/doc/release-checklist.rst index d95fa8b3..dc027719 100644 --- a/doc/release-checklist.rst +++ b/doc/release-checklist.rst @@ -41,7 +41,7 @@ Checklist for ordinary releases - Check year range - - Also check ``util/create_spdx_license.py`` + - Also check ``tools/create_spdx_license.py`` * Ensure RELEASES.rst is up-to-date (must be done before candidate tar.xz build because dist package contains RELEASES.rst) diff --git a/examples/cmdline/duk_cmdline_ajduk.c b/examples/cmdline/duk_cmdline_ajduk.c index ae28ede3..cbe683ec 100644 --- a/examples/cmdline/duk_cmdline_ajduk.c +++ b/examples/cmdline/duk_cmdline_ajduk.c @@ -508,14 +508,14 @@ void ajsheap_extstr_free_1(const void *ptr) { * binary. * * Duktape built-in strings are available from duk_build_meta.json, see - * util/duk_meta_to_strarray.py. There may also be a lot of application + * tools/duk_meta_to_strarray.py. There may also be a lot of application * specific strings, e.g. those used by application specific APIs. These - * must be gathered through some other means, see e.g. util/scan_strings.py. + * must be gathered through some other means, see e.g. tools/scan_strings.py. */ static const char *strdata_duk_builtin_strings[] = { /* - * These strings are from util/duk_meta_to_strarray.py + * These strings are from tools/duk_meta_to_strarray.py */ "Logger", diff --git a/src/duk_bi_date.c b/src/duk_bi_date.c index 79042de7..71db846b 100644 --- a/src/duk_bi_date.c +++ b/src/duk_bi_date.c @@ -61,7 +61,7 @@ DUK_LOCAL_DECL duk_ret_t duk__set_this_timeval_from_dparts(duk_context *ctx, duk #define DUK__YEAR(x) ((duk_uint8_t) ((x) - 1970)) DUK_LOCAL duk_uint8_t duk__date_equivyear[14] = { #if 1 - /* This is based on V8 EquivalentYear() algorithm (see src/genequivyear.py): + /* This is based on V8 EquivalentYear() algorithm (see util/genequivyear.py): * http://code.google.com/p/v8/source/browse/trunk/src/date.h#146 */ diff --git a/src/duk_heap_hashstring.c b/src/duk_heap_hashstring.c index 5b2ff461..c5e3aaad 100644 --- a/src/duk_heap_hashstring.c +++ b/src/duk_heap_hashstring.c @@ -13,7 +13,7 @@ * with real world inputs). Unless the hash is cryptographic, it's always * possible to craft inputs with maximal hash collisions. * - * NOTE: The hash algorithms must match src/dukutil.py:duk_heap_hashstring() + * NOTE: The hash algorithms must match tools/dukutil.py:duk_heap_hashstring() * for ROM string support! */ diff --git a/src/duk_numconv.c b/src/duk_numconv.c index 698112ab..fe3f88d8 100644 --- a/src/duk_numconv.c +++ b/src/duk_numconv.c @@ -18,7 +18,7 @@ #define DUK__DIGITCHAR(x) duk_lc_digits[(x)] /* - * Tables generated with src/gennumdigits.py. + * Tables generated with util/gennumdigits.py. * * duk__str2num_digits_for_radix indicates, for each radix, how many input * digits should be considered significant for string-to-number conversion. diff --git a/src/duk_unicode_support.c b/src/duk_unicode_support.c index 28a6da1e..fd750d91 100644 --- a/src/duk_unicode_support.c +++ b/src/duk_unicode_support.c @@ -285,7 +285,7 @@ DUK_INTERNAL duk_ucodepoint_t duk_unicode_decode_xutf8_checked(duk_hthread *thr, * chosen from several variants, based on x64 gcc -O2 testing. See: * https://github.com/svaarala/duktape/pull/422 * - * NOTE: must match src/dukutil.py:duk_unicode_unvalidated_utf8_length(). + * NOTE: must match tools/dukutil.py:duk_unicode_unvalidated_utf8_length(). */ #if defined(DUK_USE_PREFER_SIZE) @@ -396,7 +396,7 @@ DUK_INTERNAL duk_size_t duk_unicode_unvalidated_utf8_length(const duk_uint8_t *d * Used for slow path Unicode matching. */ -/* Must match src/extract_chars.py, generate_match_table3(). */ +/* Must match tools/extract_chars.py, generate_match_table3(). */ DUK_LOCAL duk_uint32_t duk__uni_decode_value(duk_bitdecoder_ctx *bd_ctx) { duk_uint32_t t; @@ -467,7 +467,7 @@ DUK_INTERNAL duk_small_int_t duk_unicode_is_whitespace(duk_codepoint_t cp) { * FEFF;ZERO WIDTH NO-BREAK SPACE;Cf;0;BN;;;;;N;BYTE ORDER MARK;;;; * * It also specifies any Unicode category 'Zs' characters as white - * space. These can be extracted with the "src/extract_chars.py" script. + * space. These can be extracted with the "tools/extract_chars.py" script. * Current result: * * RAW OUTPUT: @@ -574,7 +574,7 @@ DUK_INTERNAL duk_small_int_t duk_unicode_is_identifier_start(duk_codepoint_t cp) * * The "UnicodeLetter" alternative of the production allows letters * from various Unicode categories. These can be extracted with the - * "src/extract_chars.py" script. + * "tools/extract_chars.py" script. * * Because the result has hundreds of Unicode codepoint ranges, matching * for any values >= 0x80 are done using a very slow range-by-range scan @@ -671,7 +671,7 @@ DUK_INTERNAL duk_small_int_t duk_unicode_is_identifier_part(duk_codepoint_t cp) * The matching code reuses the "identifier start" tables, and then * consults a separate range set for characters in "identifier part" * but not in "identifier start". These can be extracted with the - * "src/extract_chars.py" script. + * "tools/extract_chars.py" script. * * UnicodeCombiningMark -> categories Mn, Mc * UnicodeDigit -> categories Nd @@ -786,14 +786,14 @@ DUK_INTERNAL duk_small_int_t duk_unicode_is_letter(duk_codepoint_t cp) { /* * Complex case conversion helper which decodes a bit-packed conversion - * control stream generated by unicode/extract_caseconv.py. The conversion + * control stream generated by tools/extract_caseconv.py. The conversion * is very slow because it runs through the conversion data in a linear * fashion to save space (which is why ASCII characters have a special * fast path before arriving here). * * The particular bit counts etc have been determined experimentally to * be small but still sufficient, and must match the Python script - * (src/extract_caseconv.py). + * (tools/extract_caseconv.py). * * The return value is the case converted codepoint or -1 if the conversion * results in multiple characters (this is useful for regexp Canonicalization diff --git a/src/duk_unicode_tables.c b/src/duk_unicode_tables.c index 8bfe9026..ce119b45 100644 --- a/src/duk_unicode_tables.c +++ b/src/duk_unicode_tables.c @@ -13,7 +13,7 @@ * compactness is most important. * * The tables are matched using uni_range_match() and the format - * is described in src/extract_chars.py. + * is described in tools/extract_chars.py. */ #ifdef DUK_USE_SOURCE_NONBMP @@ -47,7 +47,7 @@ #endif /* - * Case conversion tables generated using src/extract_caseconv.py. + * Case conversion tables generated using tools/extract_caseconv.py. */ /* duk_unicode_caseconv_uc[] */ From 8c889011c0691e06262e36e8acd61d5f44b80466 Mon Sep 17 00:00:00 2001 From: Sami Vaarala Date: Sat, 27 Aug 2016 21:41:38 +0300 Subject: [PATCH 8/9] 2.0 migration notes for tooling changes --- doc/release-notes-v2-0.rst | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/doc/release-notes-v2-0.rst b/doc/release-notes-v2-0.rst index a5627b8d..aa0a7fbf 100644 --- a/doc/release-notes-v2-0.rst +++ b/doc/release-notes-v2-0.rst @@ -67,6 +67,43 @@ DUK_OPT_xxx feature option support removed FIXME. +Tooling changes +--------------- + +There are some tooling changes in this release: + +* The distributable now includes raw sources (``src/`` in Duktape main repo) + in ``src-input/`` and some tooling in ``tools/``. + +* The tooling includes a new ``tools/prepare_sources.py`` tool which creates + a ``duk_config.h`` and matching prepared sources simultaneously. This + allows use of ROM built-ins from the distributable (previously required a + manual ``make_dist.py --rom-support ...`` command. + +* The ``make_dist.py`` utility in Duktape main repo no longer supports + ``--rom-support``, ``--rom-auto-lightfunc``, and ``--user-builtin-metadata`` + options. Use the ``tools/prepare_sources.py`` tool instead, which supports + these options. + +* The distributable still includes sources prepared using default configuration + (``src/``, ``src-noline/``, and ``src-separate``) and some configuration + examples. + +* The ``config/genconfig.py`` has been relocated to ``tools/genconfig.py`` in + the distributable. It can still be used as a standalone tool, but over time + the intent is that configuration and sources are prepared in one atomic step. + +To upgrade: + +* If you're just using the default sources and ``duk_config.h`` in the + distributable, no changes are needed. + +* If you're using ``genconfig.py``, check the path; correct path is now + ``tools/genconfig.py``. + +* If you're using ROM built-ins via ``make_dist.py``, change your build to + use ``tools/prepare_sources.py`` instead. + Buffer behavior changes ----------------------- From 9d63f8d42b3e5133515a405f295793b4c351ca9f Mon Sep 17 00:00:00 2001 From: Sami Vaarala Date: Sat, 27 Aug 2016 21:42:42 +0300 Subject: [PATCH 9/9] Releases: tooling changes --- RELEASES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/RELEASES.rst b/RELEASES.rst index 224351e5..ec8f0b38 100644 --- a/RELEASES.rst +++ b/RELEASES.rst @@ -1715,6 +1715,12 @@ Planned and constructor call argument count from 511 to 255, and maximum Ecmascript function constant count from 262144 to 65536 (GH-903) +* Incompatible change: genconfig.py has been relocated to tools/genconfig.py + in the end user distributable (GH-929) + +* Incompatible change: make_dist.py no longer supports ROM built-ins, use + tools/prepare_sources.py instead (GH-929) + * Include raw input sources and a prepare-and-config tool in the distributable, which allow user code to regenerate a config file and source code files for specified options; this is more comprehensive than just running genconfig.py