Merge pull request #619 from svaarala/rom-user-cleanups

Minor ROM built-in support cleanups

doc/low-memory.rst

@@ -281,16 +281,16 @@ The following may be appropriate when even less memory is available
 
 * Consider also moving your own built-in objects and strings into ROM:
 
-  - There is no direct support for this at present, but you can manually
-    modify ``src/strings.yaml`` and ``src/builtins.yaml`` to include your
-    own bindings.  Then rerun ``make_dist.py`` with ``--rom-support``
-    option.
-
-  - As a future work ``make_dist.py`` (``genbuiltins.py`` internally)
-    will accept external YAML config files to add custom strings and
-    objects (and modify existing objects) without editing Duktape's
-    ``strings.yaml`` and ``builtins.yaml``.  This is logically equivalent
-    to making direct edits but easier to manage in the build process.
+  - User strings and objects can also be moved into ROM.  You can also
+    modify default Duktape built-ins, adding and removing properties, etc.
+    For more details, see:
+
+    + ``util/example_user_builtins1.yaml``: examples of user builtins
+
+    + ``src/builtins.yaml``: documents some more format details
+
+    + ``util/example_rombuild.sh``: illustrates how to run ``make_dist.py``
+      with user builtins
 
 Notes on pointer compression
 ============================

src/duk_api_stack.c

@@ -4070,11 +4070,7 @@ DUK_EXTERNAL duk_idx_t duk_push_error_object_va_raw(duk_context *ctx, duk_errcod
 		duk_xdef_prop_stridx(ctx, -2, DUK_STRIDX_MESSAGE, DUK_PROPDESC_FLAGS_WC);
 	}
 
-#if 0
-	/* Disabled for now, not sure this is a useful property */
-	duk_push_int(ctx, err_code);
-	duk_xdef_prop_stridx(ctx, -2, DUK_STRIDX_CODE, DUK_PROPDESC_FLAGS_WC);
-#endif
+	/* XXX: .code = err_code disabled, not sure if useful */
 
 	/* Creation time error augmentation */
 #ifdef DUK_USE_AUGMENT_ERROR_CREATE

src/genbuiltins.py

@@ -187,32 +187,59 @@ def metadata_delete_dangling_references_to_object(meta, obj_id):
 
 # Merge a user YAML file into current metadata.
 def metadata_merge_user_objects(meta, user_meta):
+	# XXX: could be reused from other call sites
 	def _findObject(objid):
 		for i,t in enumerate(meta['objects']):
 			if t['id'] == objid:
 				return t, i
 		return None, None
+	def _findProp(obj, key):
+		for i,t in enumerate(obj['properties']):
+			if t['key'] == key:
+				return t, i
+		return None, None
 
-	for o in user_meta.get('add_objects', []):
-		#print('Add user object %s' % o['id'])
-		targ, targ_idx = _findObject(o['id'])
-		if targ is not None:
-			raise Exception('Cannot add object %s which already exists' % o['id'])
-		meta['objects'].append(o)
+	if user_meta.has_key('add_objects'):
+		raise Exception('"add_objects" removed, use "objects" with "add: True"')
+	if user_meta.has_key('replace_objects'):
+		raise Exception('"replace_objects" removed, use "objects" with "replace: True"')
+	if user_meta.has_key('modify_objects'):
+		raise Exception('"modify_objects" removed, use "objects" with "modify: True"')
 
-	for o in user_meta.get('modify_objects', []):
+	for o in user_meta.get('objects', []):
 		if o.get('disable', False):
 			print('Skip disabled object: %s' % o['id'])
 			continue
 		targ, targ_idx = _findObject(o['id'])
-		assert(targ is not None)
 
 		if o.get('delete', False):
 			print('Delete object: %s' % targ['id'])
+			if targ is None:
+				raise Exception('Cannot delete object %s which doesn\'t exist' % o['id'])
 			meta['objects'].pop(targ_idx)
 			metadata_delete_dangling_references_to_object(meta, targ['id'])
 			continue
 
+		if o.get('replace', False):
+			print('Replace object %s' % o['id'])
+			if targ is None:
+				print('WARNING: object to be replaced doesn\'t exist, append new object')
+				meta['objects'].append(o)
+			else:
+				meta['objects'][targ_idx] = o
+			continue
+
+		if o.get('add', False) or not o.get('modify', False):  # 'add' is the default
+			print('Add object %s' % o['id'])
+			if targ is not None:
+				raise Exception('Cannot add object %s which already exists' % o['id'])
+			meta['objects'].append(o)
+			continue
+
+		assert(o.get('modify', False))  # modify handling
+		if targ is None:
+			raise Exception('Cannot modify object %s which doesn\'t exist' % o['id'])
+
 		for k in sorted(o.keys()):
 			# Merge top level keys by copying over, except 'properties'
 			if k == 'properties':
@@ -224,11 +251,7 @@ def metadata_merge_user_objects(meta, user_meta):
 				continue
 			prop = None
 			prop_idx = None
-			for idx,targ_prop in enumerate(targ['properties']):
-				if targ_prop['key'] == p['key']:
-					prop = targ_prop
-					prop_idx = idx
-					break
+			prop, prop_idx = _findProp(targ, p['key'])
 			if prop is not None:
 				if p.get('delete', False):
 					print('Delete property %s of %s' % (p['key'], o['id']))
@@ -668,19 +691,31 @@ def metadata_add_string_define_names(strlist, special_defs):
 
 # Add a 'stridx_used' flag for strings which need a stridx.
 def metadata_add_string_used_stridx(strlist, used_stridx_meta):
-	defs = {}
+	defs_needed = {}
+	defs_found = {}
 	for s in used_stridx_meta['used_stridx_defines']:
-		defs[s] = True
+		defs_needed[s] = True
 
+	# strings whose define is referenced
 	for s in strlist:
-		if s.has_key('define') and defs.has_key(s['define']):
+		if s.has_key('define') and defs_needed.has_key(s['define']):
 			s['stridx_used'] = True
+			defs_found[s['define']] = True
 
 	# duk_lexer.h needs all reserved words
 	for s in strlist:
 		if s.get('reserved_word', False):
 			s['stridx_used'] = True
 
+	# ensure all needed defines are provided
+	defs_found['DUK_STRIDX_START_RESERVED'] = True  # special defines provided automatically
+	defs_found['DUK_STRIDX_START_STRICT_RESERVED'] = True
+	defs_found['DUK_STRIDX_END_RESERVED'] = True
+	defs_found['DUK_STRIDX_TO_TOK'] = True
+	for k in sorted(defs_needed.keys()):
+		if not defs_found.has_key(k):
+			raise Exception('source code needs define %s not provided by strings' % repr(k))
+
 # Merge duplicate strings in string metadata.
 def metadata_merge_string_entries(strlist):
 	# The raw string list may contain duplicates so merge entries.

util/example_user_builtins1.yaml

@@ -4,8 +4,7 @@
 #  Top level keys:
 #
 #    - add_forced_strings: provide a list of keys to be forced into ROM
-#    - add_objects: provide additional objects
-#    - modify_objects: modify previously added objects
+#    - objects: objects to add, replace, delete, or modify
 #
 #  See examples below for details on how to use these.
 #
@@ -87,23 +86,39 @@ add_forced_strings:
   - str: "quux"
   - str: "baz"
 
-# Objects to add.  Properties may reference other objects using the 'id'
-# strings even if the object hasn't been introduced yet (and may be introduced
-# by a different user builtins YAML file).  Error if an object with the same
-# 'id' already exists.
-
-add_objects:
-  # Basic example: add a built-in object with plain property values.  This
-  # just creates the built-in object; for the object to be actually useful
-  # you must use the 'modify_objects' key to add a reference to the object
-  # to some existing object reachable from the global object (see below for
-  # an example pointing to this object).
+# Objects to add, replace, delete, or modify:
+#
+#   - If the object entry contains 'delete': true', the object will be
+#     deleted.  Error if the object doesn't exist.
+#
+#   - If the object entry contains 'replace: true', the object will be
+#     replaced; if the object doesn't exist it'll be added with a warning.
+#
+#   - If the object entry contains 'modify: true', the object will be
+#     modified incrementally.  Error if the object doesn't exist.
+#
+#   - If the object entry contains 'add: true', the object will be added.
+#     Error if the object already exists.  This is the default if no keys
+#     listed above are given.
+
+objects:
+
+  # Example of adding an object.  Properties may reference other objects using
+  # the 'id' strings even if the object hasn't been introduced yet (and may be
+  # introduced by a different user builtins YAML file).
+  #
+  # Add a built-in object with plain property values.  This just creates the
+  # built-in object; for the object to be actually useful you must use e.g.
+  # 'modify: true' to add a reference to the object to some existing object
+  # reachable from the global object (see below for an example pointing to
+  # this object).
   #
   # (In more detail, it's sufficient for the custom object to be reachable
   # from any object with a 'bidx', but in practice objects are reachable
   # directly on indirectly through the global object.)
 
   - id: bi_star_trek
+    add: true
     class: Object  # recommended for objects
     internal_prototype: bi_object_prototype  # recommended so that e.g. .toString() works
 
@@ -141,6 +156,7 @@ add_objects:
   #       print(struct.pack('>d', 12345.6789).encode('hex'))
 
   - id: bi_type_examples
+    add: true
     class: Object
     internal_prototype: bi_object_prototype
 
@@ -176,6 +192,7 @@ add_objects:
   # a separate YAML file not yet loaded).
 
   - id: bi_circular1
+    add: true
     class: Object
     internal_prototype: bi_object_prototype
 
@@ -188,6 +205,7 @@ add_objects:
           id: bi_circular2
 
   - id: bi_circular2
+    add: true
     class: Object
     internal_prototype: bi_object_prototype
 
@@ -203,6 +221,7 @@ add_objects:
   # with a note in the build log.
 
   - id: my_unreachable_object1
+    add: true
     class: Object
     internal_prototype: bi_object_prototype
 
@@ -211,6 +230,7 @@ add_objects:
         value: "bar"
 
   - id: my_unreachable_object2
+    add: true
     class: Object
     internal_prototype: bi_object_prototype
 
@@ -218,43 +238,98 @@ add_objects:
       - key: "foo"
         value: "bar"
 
-# Objects to modify, i.e. delete, augment top level metadata, add property,
-# delete property, or modify a property.  You can operate on the same object
-# multiple times in the same 'modify_objects' list.
-#
-# Top level keys other than 'properties' are copied over the existing object
-# so that you can e.g. change the class of an object.
-#
-# If the object has "delete: true" the entire object is deleted.  Dangling
-# references to the object are deleted automatically, with a note in the
-# build log.
-#
-# Property list is then walked in order.
-#
-# If a property has "delete: true" the property is deleted:
-#
-#   - If the key doesn't exist in the existing object, ignore silently.
-#
-#   - If the key exists, delete the property.
-#
-# Otherwise the property is added/modified:
-#
-#   - If the key already exists the existing property is replaced
-#     (and keeps its enumeration position).
-#
-#   - If the key doesn't exist the property is added to the end of
-#     the property list.
-#
-# XXX: At present there's no support for reordering properties which would
-# be nice because it affects enumeration order (and may affect performance
-# marginally).
+  - id: my_disabled
+    disable: true  # will be skipped in metadata
+    add: true
+    class: Object
+    internal_prototype: bi_object_prototype
+
+    properties:
+      - key: "bar"
+        value: "quux"
+
+  # Example of an object to be replaced entirely.
+  #
+  # Replace Error.prototype entirely with a stripped one.
+
+  - id: bi_error_prototype
+    replace: true
+    class: Error
+    internal_prototype: bi_object_prototype
+
+    properties:
+      - key: "constructor"
+        value:
+          type: object
+          id: bi_error_constructor
+        attributes: "wc"
+      - key: "name"
+        value: "Error"
+      - key: "message"
+        value: ""
+
+  - id: bi_json
+    disable: true  # disabled in metadata
+    replace: true
+
+    class: Object
+    internal_prototype: bi_object_prototype
+
+    properties:
+      - key: "bar"
+        value: "quux"
+
+  # Example of how to delete an existing built-in object entirely.  Dangling
+  # references are automatically deleted with a note in the build log.
+  #
+  # This doesn't currently work very well for Duktape built-ins because most
+  # built-ins are expected to be present in Duktape internals (and they have
+  # a DUK_BIDX_xxx index).
+  #
+  # Deleting user objects may be useful e.g. if a base YAML file provides your
+  # custom built-ins and a target specific YAML file removes bindings not
+  # needed for a certain target.
+  #
+  # In this example we'd delete the StarTrek object.  The global reference
+  # global.StarTrek would be deleted automatically.
+
+  - id: bi_star_trek
+    disable: true  # disabled in metadata
+    delete: true
+
+  # Examples of modifying an object, i.e. augment top level metadata, add
+  # property, delete property, or modify a property.  You can operate on the
+  # same object multiple times in the same 'objects' list.
+  #
+  # Top level keys other than 'properties' are copied over the existing object
+  # so that you can e.g. change the class of an object.
+  #
+  # Property list is then walked in order.
+  #
+  # If a property has "delete: true" the property is deleted:
+  #
+  #   - If the key doesn't exist in the existing object, ignore silently.
+  #
+  #   - If the key exists, delete the property.
+  #
+  # Otherwise the property is added/modified:
+  #
+  #   - If the key already exists the existing property is replaced
+  #     (and keeps its enumeration position).
+  #
+  #   - If the key doesn't exist the property is added to the end of
+  #     the property list.
+  #
+  # XXX: At present there's no support for reordering properties which would
+  # be nice because it affects enumeration order (and may affect performance
+  # marginally).
 
-modify_objects:
   # Add references to the example objects added above into the global object.
   # The references can be in any object reachable from the global object of
   # course.  Unreachable objects will be dropped with a build note.
 
   - id: bi_global
+    modify: true
 
     properties:
       - key: 'StarTrek'
@@ -329,24 +404,8 @@ modify_objects:
   # example Math.cos() is deleted.
 
   - id: bi_math
+    modify: true
 
     properties:
       - key: 'cos'
         delete: true
-
-  # Example of how to delete an existing built-in object entirely.  Dangling
-  # references are automatically deleted with a note in the build log.
-  #
-  # This doesn't currently work very well for Duktape built-ins because most
-  # built-ins are expected to be present in Duktape internals (and they have
-  # a DUK_BIDX_xxx index).
-  #
-  # Deleting user objects may be useful e.g. if a base YAML file provides your
-  # custom built-ins and a target specific YAML file removes bindings not
-  # needed for a certain target.
-  #
-  # In this example we'd delete the StarTrek object.  The global reference
-  # global.StarTrek would be deleted automatically.
-
-  #- id: bi_star_trek
-  #  delete: true

util/example_user_builtins2.yaml

@@ -3,9 +3,11 @@
 #  util/example_user_builtins1.yaml.
 #
 
-modify_objects:
+objects:
 
   - id: bi_global
+    modify: true
+
     properties:
       - key: meaningOfLife
         value: 42