Merge branch 'add-example-allocators'

Add two example allocators to the distributable:

- A logging allocator writes alloc/realloc/free sizes and results to
  /tmp/duk-alloc-log.txt.  The log includes previous allocation size
  for realloc() and free() so that the memory behavior can be played
  back and used for e.g. optimizing allocator pool sizes.

- A torture allocator uses red zones to detect out-of-bound writes,
  inits data to a non-zero value, and wipes freed memory areas so
  accidental reads after freeing are detected more easily.  This
  allocator can be used on almost any target to detect basic issues
  and is useful especially when Valgrind doesn't work on the target.

Also fix some NULL checks in sandbox allocator example.

Makefile

@@ -158,7 +158,9 @@ DUKTAPE_SOURCES = $(DUKTAPE_SOURCES_COMBINED)
 # Duktape command line tool - example of a main() program, used
 # for unit testing
 DUKTAPE_CMDLINE_SOURCES = \
-	$(DISTCMD)/duk_cmdline.c
+	$(DISTCMD)/duk_cmdline.c \
+	dist/examples/alloc-logging/duk_alloc_logging.c \
+	dist/examples/alloc-torture/duk_alloc_torture.c
 
 # Compiler setup for Linux
 CC	= gcc
@@ -169,6 +171,8 @@ CCOPTS_SHARED += -Wall
 CCOPTS_SHARED += -Wextra  # very picky but catches e.g. signed/unsigned comparisons
 CCOPTS_SHARED += -Wunused-result
 CCOPTS_SHARED += -I./dist/src
+CCOPTS_SHARED += -I./dist/examples/alloc-logging
+CCOPTS_SHARED += -I./dist/examples/alloc-torture
 #CCOPTS_SHARED += -I./dist/src-separate
 #CCOPTS_SHARED += -m32                             # force 32-bit compilation on a 64-bit host
 #CCOPTS_SHARED += -mx32                            # force X32 compilation on a 64-bit host
@@ -215,6 +219,8 @@ CCOPTS_SHARED += -DDUK_OPT_DEBUG_BUFSIZE=512
 #CCOPTS_SHARED += -DDUK_OPT_NO_ZERO_BUFFER_DATA
 #CCOPTS_SHARED += -DDUK_OPT_USER_INITJS='"this.foo = 123"'
 CCOPTS_SHARED += -DDUK_CMDLINE_FANCY
+CCOPTS_SHARED += -DDUK_CMDLINE_ALLOC_LOGGING
+CCOPTS_SHARED += -DDUK_CMDLINE_ALLOC_TORTURE
 CCOPTS_NONDEBUG = $(CCOPTS_SHARED) -Os -fomit-frame-pointer
 CCOPTS_NONDEBUG += -g -ggdb
 #CCOPTS_NONDEBUG += -DDUK_OPT_ASSERTIONS

RELEASES.rst

@@ -632,6 +632,14 @@ Planned
 * Use deep C stack for dukweb.js to remove some compiler recursion limit
   limitations (see GH-67)
 
+* Add an example allocator with alloc/realloc/free logging, which is
+  useful when optimizing e.g. pool sizes for low memory targets
+
+* Add an example allocator with memory wiping, red zones for invalid
+  writes, and forced address change on realloc, which can be used for
+  detecting memory safety issues on platforms where valgrind is not
+  available
+
 1.2.0 (2015-XX-XX)
 ------------------
 

examples/alloc-logging/README.rst

@@ -0,0 +1,7 @@
+======================
+Allocator with logging
+======================
+
+Example allocator that writes all memory alloc/realloc/free calls into a
+log file so that memory usage can replayed later.  This is useful to e.g.
+optimize pool sizes.

examples/alloc-logging/duk_alloc_logging.c

@@ -0,0 +1,138 @@
+/*
+ *  Example memory allocator with machine parseable logging.
+ *
+ *  Also sizes for reallocs and frees are logged so that the memory
+ *  behavior can be essentially replayed to accurately determine e.g.
+ *  optimal pool sizes for a pooled allocator.
+ *
+ *  Allocation structure:
+ *
+ *    [ alloc_hdr | user area ]
+ *
+ *     ^           ^
+ *     |           `--- pointer returned to Duktape
+ *     `--- underlying malloc ptr
+ */
+
+#include "duktape.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+
+#define  ALLOC_LOG_FILE  "/tmp/duk-alloc-log.txt"
+
+typedef struct {
+	/* The double value in the union is there to ensure alignment is
+	 * good for IEEE doubles too.  In many 32-bit environments 4 bytes
+	 * would be sufficiently aligned and the double value is unnecessary.
+	 */
+	union {
+		size_t sz;
+		double d;
+	} u;
+} alloc_hdr;
+
+static FILE *log_file = NULL;
+
+static void write_log(const char *fmt, ...) {
+	va_list ap;
+
+	if (!log_file) {
+		log_file = fopen(ALLOC_LOG_FILE, "wb");
+		if (!log_file) {
+			return;
+		}
+	}
+
+	va_start(ap, fmt);
+	vfprintf(log_file, fmt, ap);
+	va_end(ap);
+}
+
+void *duk_alloc_logging(void *udata, duk_size_t size) {
+	alloc_hdr *hdr;
+	void *ret;
+
+	(void) udata;  /* Suppress warning. */
+
+	if (size == 0) {
+		write_log("A NULL %ld\n", (long) size);
+		return NULL;
+	}
+
+	hdr = (alloc_hdr *) malloc(size + sizeof(alloc_hdr));
+	if (!hdr) {
+		write_log("A FAIL %ld\n", (long) size);
+		return NULL;
+	}
+	hdr->u.sz = size;
+	ret = (void *) (hdr + 1);
+	write_log("A %p %ld\n", ret, (long) size);
+	return ret;
+}
+
+void *duk_realloc_logging(void *udata, void *ptr, duk_size_t size) {
+	alloc_hdr *hdr;
+	size_t old_size;
+	void *t;
+	void *ret;
+
+	(void) udata;  /* Suppress warning. */
+
+	/* Handle the ptr-NULL vs. size-zero cases explicitly to minimize
+	 * platform assumptions.  You can get away with much less in specific
+	 * well-behaving environments.
+	 */
+
+	if (ptr) {
+		hdr = (alloc_hdr *) ((unsigned char *) ptr - sizeof(alloc_hdr));
+		old_size = hdr->u.sz;
+
+		if (size == 0) {
+			free((void *) hdr);
+			write_log("R %p %ld NULL 0\n", ptr, (long) old_size);
+			return NULL;
+		} else {
+			t = realloc((void *) hdr, size + sizeof(alloc_hdr));
+			if (!t) {
+				write_log("R %p %ld FAIL %ld\n", ptr, (long) old_size, (long) size);
+				return NULL;
+			}
+			hdr = (alloc_hdr *) t;
+			hdr->u.sz = size;
+			ret = (void *) (hdr + 1);
+			write_log("R %p %ld %p %ld\n", ptr, (long) old_size, ret, (long) size);
+			return ret;
+		}
+	} else {
+		if (size == 0) {
+			write_log("R NULL 0 NULL 0\n");
+			return NULL;
+		} else {
+			hdr = (alloc_hdr *) malloc(size + sizeof(alloc_hdr));
+			if (!hdr) {
+				write_log("R NULL 0 FAIL %ld\n", (long) size);
+				return NULL;
+			}
+			hdr->u.sz = size;
+			ret = (void *) (hdr + 1);
+			write_log("R NULL 0 %p %ld\n", ret, (long) size);
+			return ret;
+		}
+	}
+}
+
+void duk_free_logging(void *udata, void *ptr) {
+	alloc_hdr *hdr;
+
+	(void) udata;  /* Suppress warning. */
+
+	if (!ptr) {
+		write_log("F NULL 0\n");
+		return;
+	}
+	hdr = (alloc_hdr *) ((unsigned char *) ptr - sizeof(alloc_hdr));
+	write_log("F %p %ld\n", ptr, (long) hdr->u.sz);
+	free((void *) hdr);
+}

examples/alloc-logging/duk_alloc_logging.h

@@ -0,0 +1,10 @@
+#ifndef DUK_ALLOC_LOGGING_H_INCLUDED
+#define DUK_ALLOC_LOGGING_H_INCLUDED
+
+#include "duktape.h"
+
+void *duk_alloc_logging(void *udata, duk_size_t size);
+void *duk_realloc_logging(void *udata, void *ptr, duk_size_t size);
+void duk_free_logging(void *udata, void *ptr);
+
+#endif  /* DUK_ALLOC_LOGGING_H_INCLUDED */

examples/alloc-logging/log2gnuplot.py

@@ -0,0 +1,39 @@
+#!/usr/bin/python
+#
+#  Analyze allocator logs and write total-bytes-in-use after every
+#  operation to stdout.  The output can be gnuplotted as:
+#
+#  $ python log2gnuplot.py </tmp/duk-alloc-log.txt >/tmp/output.txt
+#  $ gnuplot
+#  > plot "output.txt" with lines
+#
+
+import os
+import sys
+
+def main():
+	allocated = 0
+
+	for line in sys.stdin:
+		line = line.strip()
+		parts = line.split(' ')
+
+		# A ptr/NULL/FAIL size
+		# F ptr/NULL size
+		# R ptr/NULL oldsize ptr/NULL/FAIL newsize
+
+		if parts[0] == 'A':
+			if parts[1] != 'NULL' and parts[1] != 'FAIL':
+				allocated += long(parts[2])
+		elif parts[0] == 'F':
+			allocated -= long(parts[2])
+		elif parts[0] == 'R':
+			allocated -= long(parts[2])
+			if parts[3] != 'NULL' and parts[3] != 'FAIL':
+				allocated += long(parts[4])
+		print(allocated)
+
+	print(allocated)
+
+if __name__ == '__main__':
+	main()

examples/alloc-torture/README.rst

@@ -0,0 +1,10 @@
+==========================================
+Allocator with memory wiping and red zones
+==========================================
+
+Example allocator that wipes memory on free and checks that no out-of-bounds
+writes have been made to bytes just before and after the allocated area.
+
+Valgrind is a better tool for detecting these memory issues, but it's not
+available for all targets so you can use something like this to detect
+memory lifecycle or out-of-bounds issues.

examples/alloc-torture/duk_alloc_torture.c

@@ -0,0 +1,182 @@
+/*
+ *  Example torture memory allocator with memory wiping and check for
+ *  out-of-bounds writes.
+ *
+ *  Allocation structure:
+ *
+ *    [ alloc_hdr | red zone before | user area | red zone after ]
+ *
+ *     ^                             ^
+ *     |                             `--- pointer returned to Duktape
+ *     `--- underlying malloc ptr
+ */
+
+#include "duktape.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+
+#define  RED_ZONE_SIZE  16
+#define  RED_ZONE_BYTE  0x5a
+#define  INIT_BYTE      0xa5
+#define  WIPE_BYTE      0x27
+
+typedef struct {
+	/* The double value in the union is there to ensure alignment is
+	 * good for IEEE doubles too.  In many 32-bit environments 4 bytes
+	 * would be sufficiently aligned and the double value is unnecessary.
+	 */
+	union {
+		size_t sz;
+		double d;
+	} u;
+} alloc_hdr;
+
+static void check_red_zone(alloc_hdr *hdr) {
+	size_t size;
+	int i;
+	int err;
+	unsigned char *p;
+	unsigned char *userptr;
+
+	size = hdr->u.sz;
+	userptr = (unsigned char *) hdr + sizeof(alloc_hdr) + RED_ZONE_SIZE;
+
+	err = 0;
+	p = (unsigned char *) hdr + sizeof(alloc_hdr);
+	for (i = 0; i < RED_ZONE_SIZE; i++) {
+		if (p[i] != RED_ZONE_BYTE) {
+			err = 1;
+		}
+	}
+	if (err) {
+		fprintf(stderr, "RED ZONE CORRUPTED BEFORE ALLOC: hdr=%p ptr=%p size=%ld\n",
+		        (void *) hdr, (void *) userptr, (long) size);
+		fflush(stderr);
+	}
+
+	err = 0;
+	p = (unsigned char *) hdr + sizeof(alloc_hdr) + RED_ZONE_SIZE + size;
+	for (i = 0; i < RED_ZONE_SIZE; i++) {
+		if (p[i] != RED_ZONE_BYTE) {
+			err = 1;
+		}
+	}
+	if (err) {
+		fprintf(stderr, "RED ZONE CORRUPTED AFTER ALLOC: hdr=%p ptr=%p size=%ld\n",
+		        (void *) hdr, (void *) userptr, (long) size);
+		fflush(stderr);
+	}
+}
+
+void *duk_alloc_torture(void *udata, duk_size_t size) {
+	unsigned char *p;
+
+	(void) udata;  /* Suppress warning. */
+
+	if (size == 0) {
+		return NULL;
+	}
+
+	p = (unsigned char *) malloc(size + sizeof(alloc_hdr) + 2 * RED_ZONE_SIZE);
+	if (!p) {
+		return NULL;
+	}
+
+	((alloc_hdr *) p)->u.sz = size;
+	p += sizeof(alloc_hdr);
+	memset((void *) p, RED_ZONE_BYTE, RED_ZONE_SIZE);
+	p += RED_ZONE_SIZE;
+	memset((void *) p, INIT_BYTE, size);
+	p += size;
+	memset((void *) p, RED_ZONE_BYTE, RED_ZONE_SIZE);
+	p -= size;
+	return (void *) p;
+}
+
+void *duk_realloc_torture(void *udata, void *ptr, duk_size_t size) {
+	unsigned char *p, *old_p;
+	size_t old_size;
+
+	(void) udata;  /* Suppress warning. */
+
+	/* Handle the ptr-NULL vs. size-zero cases explicitly to minimize
+	 * platform assumptions.  You can get away with much less in specific
+	 * well-behaving environments.
+	 */
+
+	if (ptr) {
+		old_p = (unsigned char *) ptr - sizeof(alloc_hdr) - RED_ZONE_SIZE;
+		old_size = ((alloc_hdr *) old_p)->u.sz;
+		check_red_zone((alloc_hdr *) old_p);
+
+		if (size == 0) {
+			memset((void *) old_p, WIPE_BYTE, old_size + sizeof(alloc_hdr) + 2 * RED_ZONE_SIZE);
+			free((void *) old_p);
+			return NULL;
+		} else {
+			/* Force address change on every realloc. */
+			p = (unsigned char *) malloc(size + sizeof(alloc_hdr) + 2 * RED_ZONE_SIZE);
+			if (!p) {
+				return NULL;
+			}
+
+			((alloc_hdr *) p)->u.sz = size;
+			p += sizeof(alloc_hdr);
+			memset((void *) p, RED_ZONE_BYTE, RED_ZONE_SIZE);
+			p += RED_ZONE_SIZE;
+			if (size > old_size) {
+				memcpy((void *) p, (void *) (old_p + sizeof(alloc_hdr) + RED_ZONE_SIZE), old_size);
+				memset((void *) (p + old_size), INIT_BYTE, size - old_size);
+			} else {
+				memcpy((void *) p, (void *) (old_p + sizeof(alloc_hdr) + RED_ZONE_SIZE), size);
+			}
+			p += size;
+			memset((void *) p, RED_ZONE_BYTE, RED_ZONE_SIZE);
+			p -= size;
+
+			memset((void *) old_p, WIPE_BYTE, old_size + sizeof(alloc_hdr) + 2 * RED_ZONE_SIZE);
+			free((void *) old_p);
+
+			return (void *) p;
+		}
+	} else {
+		if (size == 0) {
+			return NULL;
+		} else {
+			p = (unsigned char *) malloc(size + sizeof(alloc_hdr) + 2 * RED_ZONE_SIZE);
+			if (!p) {
+				return NULL;
+			}
+
+			((alloc_hdr *) p)->u.sz = size;
+			p += sizeof(alloc_hdr);
+			memset((void *) p, RED_ZONE_BYTE, RED_ZONE_SIZE);
+			p += RED_ZONE_SIZE;
+			memset((void *) p, INIT_BYTE, size);
+			p += size;
+			memset((void *) p, RED_ZONE_BYTE, RED_ZONE_SIZE);
+			p -= size;
+			return (void *) p;
+		}
+	}
+}
+
+void duk_free_torture(void *udata, void *ptr) {
+	unsigned char *p;
+	size_t old_size;
+
+	(void) udata;  /* Suppress warning. */
+
+	if (!ptr) {
+		return;
+	}
+
+	p = (unsigned char *) ptr - sizeof(alloc_hdr) - RED_ZONE_SIZE;
+	old_size = ((alloc_hdr *) p)->u.sz;
+
+	check_red_zone((alloc_hdr *) p);
+	memset((void *) p, WIPE_BYTE, old_size + sizeof(alloc_hdr) + 2 * RED_ZONE_SIZE);
+	free((void *) p);
+}

examples/alloc-torture/duk_alloc_torture.h

@@ -0,0 +1,10 @@
+#ifndef DUK_ALLOC_TORTURE_H_INCLUDED
+#define DUK_ALLOC_TORTURE_H_INCLUDED
+
+#include "duktape.h"
+
+void *duk_alloc_torture(void *udata, duk_size_t size);
+void *duk_realloc_torture(void *udata, void *ptr, duk_size_t size);
+void duk_free_torture(void *udata, void *ptr);
+
+#endif  /* DUK_ALLOC_TORTURE_H_INCLUDED */

examples/cmdline/duk_cmdline.c

@@ -33,7 +33,12 @@
 #include <readline/readline.h>
 #include <readline/history.h>
 #endif
-
+#ifdef DUK_CMDLINE_ALLOC_LOGGING
+#include "duk_alloc_logging.h"
+#endif
+#ifdef DUK_CMDLINE_ALLOC_TORTURE
+#include "duk_alloc_torture.h"
+#endif
 #include "duktape.h"
 
 #define  MEM_LIMIT_NORMAL   (128*1024*1024)   /* 128 MB */
@@ -386,6 +391,8 @@ int main(int argc, char *argv[]) {
 	int have_eval = 0;
 	int interactive = 0;
 	int memlimit_high = 1;
+	int alloc_logging = 0;
+	int alloc_torture = 0;
 	int i;
 
 	/*
@@ -408,7 +415,7 @@ int main(int argc, char *argv[]) {
 		if (!arg) {
 			goto usage;
 		}
-		if (strcmp(arg, "-r") == 0) {
+		if (strcmp(arg, "--restrict-memory") == 0) {
 			memlimit_high = 0;
 		} else if (strcmp(arg, "-i") == 0) {
 			interactive = 1;
@@ -418,6 +425,10 @@ int main(int argc, char *argv[]) {
 				goto usage;
 			}
 			i++;  /* skip code */
+		} else if (strcmp(arg, "--alloc-logging") == 0) {
+			alloc_logging = 1;
+		} else if (strcmp(arg, "--alloc-torture") == 0) {
+			alloc_torture = 1;
 		} else if (strlen(arg) >= 1 && arg[0] == '-') {
 			goto usage;
 		} else {
@@ -435,14 +446,44 @@ int main(int argc, char *argv[]) {
 #ifndef NO_RLIMIT
 	set_resource_limits(memlimit_high ? MEM_LIMIT_HIGH : MEM_LIMIT_NORMAL);
 #else
-	(void) memlimit_high;  /* suppress warning */
+	if (memlimit_high == 0) {
+		fprintf(stderr, "Warning: option --restrict-memory ignored, no rlimit support\n");
+		fflush(stderr);
+	}
 #endif
 
 	/*
 	 *  Create context and execute any argument file(s)
 	 */
 
-	ctx = duk_create_heap_default();
+	ctx = NULL;
+	if (!ctx && alloc_logging) {
+#ifdef DUK_CMDLINE_ALLOC_LOGGING
+		ctx = duk_create_heap(duk_alloc_logging,
+		                      duk_realloc_logging,
+		                      duk_free_logging,
+		                      NULL,
+		                      NULL);
+#else
+		fprintf(stderr, "Warning: option --alloc-logging ignored, no logging allocator support\n");
+		fflush(stderr);
+#endif
+	}
+	if (!ctx && alloc_torture) {
+#ifdef DUK_CMDLINE_ALLOC_TORTURE
+		ctx = duk_create_heap(duk_alloc_torture,
+		                      duk_realloc_torture,
+		                      duk_free_torture,
+		                      NULL,
+		                      NULL);
+#else
+		fprintf(stderr, "Warning: option --alloc-torture ignored, no torture allocator support\n");
+		fflush(stderr);
+#endif
+	}
+	if (!ctx) {
+		ctx = duk_create_heap_default();
+	}
 
 	for (i = 1; i < argc; i++) {
 		char *arg = argv[i];
@@ -502,16 +543,13 @@ int main(int argc, char *argv[]) {
 	 */
 
  usage:
-	fprintf(stderr, "Usage: duk [-i] [-r] [<filenames>]\n"
-	                "\n"
-	                "   -i       enter interactive mode after executing argument file(s) / eval code\n"
-	                "   -e CODE  evaluate code\n"
-	                "   -r       use lower memory limit (used by test runner)"
-#ifdef NO_RLIMIT
-	                " (disabled)\n"
-#else
+	fprintf(stderr, "Usage: duk [options] [<filenames>]\n"
 	                "\n"
-#endif
+	                "   -i                 enter interactive mode after executing argument file(s) / eval code\n"
+	                "   -e CODE            evaluate code\n"
+	                "   --restrict-memory  use lower memory limit (used by test runner)\n"
+	                "   --alloc-logging    use logging allocator (writes to /tmp)\n"
+	                "   --alloc-torture    use torture allocator\n"
 	                "\n"
 	                "If <filename> is omitted, interactive mode is started automatically.\n");
 	fflush(stderr);

examples/sandbox/sandbox.c

@@ -54,6 +54,9 @@ static void *sandbox_alloc(void *udata, duk_size_t size) {
 	}
 
 	hdr = (alloc_hdr *) malloc(size + sizeof(alloc_hdr));
+	if (!hdr) {
+		return NULL;
+	}
 	hdr->u.sz = size;
 	total_allocated += size;
 	sandbox_dump_memstate();
@@ -112,6 +115,9 @@ static void *sandbox_realloc(void *udata, void *ptr, duk_size_t size) {
 			}
 
 			hdr = (alloc_hdr *) malloc(size + sizeof(alloc_hdr));
+			if (!hdr) {
+				return NULL;
+			}
 			hdr->u.sz = size;
 			total_allocated += size;
 			sandbox_dump_memstate();

runtests/runtests.js

@@ -207,8 +207,9 @@ function executeTest(options, callback) {
 		} else {
 		    cmd.push(options.engine.fullPath);
 		    if (!options.valgrind && options.engine.name === 'duk') {
-			cmd.push('-r');  // restricted memory
+			cmd.push('--restrict-memory');  // restricted memory
 		    }
+		    // cmd.push('--alloc-torture');
 		    cmd.push(tempInput || options.testPath);
 		}
 		cmdline = cmd.join(' ');

util/make_dist.sh

@@ -67,6 +67,8 @@ mkdir $DIST/examples/guide
 mkdir $DIST/examples/coffee
 mkdir $DIST/examples/jxpretty
 mkdir $DIST/examples/sandbox
+mkdir $DIST/examples/alloc-logging
+mkdir $DIST/examples/alloc-torture
 
 # Copy most files directly
 
@@ -272,6 +274,23 @@ for i in \
 	cp examples/sandbox/$i $DIST/examples/sandbox/ || exit 1
 done
 
+for i in \
+	README.rst \
+	duk_alloc_logging.c \
+	duk_alloc_logging.h \
+	log2gnuplot.py \
+	; do
+	cp examples/alloc-logging/$i $DIST/examples/alloc-logging/ || exit 1
+done
+
+for i in \
+	README.rst \
+	duk_alloc_torture.c \
+	duk_alloc_torture.h \
+	; do
+	cp examples/alloc-torture/$i $DIST/examples/alloc-torture/ || exit 1
+done
+
 cp extras/README.rst $DIST/extras/ || exit
 # XXX: copy extras