Rework lexer to use bufwriter

src/duk_internal.h

@@ -48,6 +48,7 @@ DUK_USE_USER_DECLARE()
 #include "duk_forwdecl.h"
 #include "duk_builtins.h"  /* autogenerated: strings and built-in object init data */
 
+#include "duk_util.h"
 #include "duk_strings.h"
 #include "duk_js_bytecode.h"
 #include "duk_lexer.h"
@@ -67,7 +68,6 @@ DUK_USE_USER_DECLARE()
 #include "duk_debugger.h"
 #include "duk_debug.h"
 #include "duk_error.h"
-#include "duk_util.h"
 #include "duk_unicode.h"
 #include "duk_json.h"
 #include "duk_js.h"

src/duk_lexer.c

@@ -510,12 +510,14 @@ DUK_LOCAL void duk__init_lexer_window(duk_lexer_ctx *lex_ctx) {
  */
 
 DUK_LOCAL void duk__initbuffer(duk_lexer_ctx *lex_ctx) {
-	if (DUK_HBUFFER_DYNAMIC_GET_ALLOC_SIZE(lex_ctx->buf) < DUK_LEXER_TEMP_BUF_LIMIT) {
-		/* Resize (zero) without realloc. */
-		DUK_HBUFFER_DYNAMIC_SET_SIZE(lex_ctx->buf, 0);
+	/* Reuse buffer as is unless buffer has grown large. */
+	if (DUK_HBUFFER_DYNAMIC_GET_SIZE(lex_ctx->buf) < DUK_LEXER_TEMP_BUF_LIMIT) {
+		/* Keep current size */
 	} else {
-		duk_hbuffer_resize(lex_ctx->thr, lex_ctx->buf, 0, DUK_LEXER_TEMP_BUF_LIMIT);
+		duk_hbuffer_resize(lex_ctx->thr, lex_ctx->buf, DUK_LEXER_TEMP_BUF_LIMIT, DUK_LEXER_TEMP_BUF_LIMIT);
 	}
+
+	DUK_BW_INIT_WITHBUF(lex_ctx->thr, &lex_ctx->bw, lex_ctx->buf);
 }
 
 /*
@@ -537,7 +539,7 @@ DUK_LOCAL void duk__appendbuffer(duk_lexer_ctx *lex_ctx, duk_codepoint_t x) {
 
 	DUK_ASSERT(x >= 0 && x <= 0x10ffff);
 
-	duk_hbuffer_append_cesu8(lex_ctx->thr, lex_ctx->buf, (duk_ucodepoint_t) x);
+	DUK_BW_WRITE_ENSURE_CESU8(lex_ctx->thr, &lex_ctx->bw, (duk_ucodepoint_t) x);
 }
 
 /*
@@ -550,8 +552,7 @@ DUK_LOCAL void duk__internbuffer(duk_lexer_ctx *lex_ctx, duk_idx_t valstack_idx)
 
 	DUK_ASSERT(valstack_idx == lex_ctx->slot1_idx || valstack_idx == lex_ctx->slot2_idx);
 
-	duk_dup(ctx, lex_ctx->buf_idx);
-	duk_to_string(ctx, -1);
+	DUK_BW_PUSH_AS_STRING(lex_ctx->thr, &lex_ctx->bw);
 	duk_replace(ctx, valstack_idx);
 }
 

src/duk_lexer.h

@@ -341,8 +341,7 @@ typedef void (*duk_re_range_callback)(void *user, duk_codepoint_t r1, duk_codepo
 #define DUK_RETOK_ATOM_START_CHARCLASS_INVERTED    21
 #define DUK_RETOK_ATOM_END_GROUP                   22
 
-/* constants for duk_lexer_ctx.buf */
-#define DUK_LEXER_TEMP_BUF_INITIAL                 64
+/* Constants for duk_lexer_ctx.buf. */
 #define DUK_LEXER_TEMP_BUF_LIMIT                   256
 
 /* A token value.  Can be memcpy()'d, but note that slot1/slot2 values are on the valstack.
@@ -405,7 +404,8 @@ struct duk_lexer_ctx {
 	duk_idx_t slot1_idx;                           /* valstack slot for 1st token value */
 	duk_idx_t slot2_idx;                           /* valstack slot for 2nd token value */
 	duk_idx_t buf_idx;                             /* valstack slot for temp buffer */
-	duk_hbuffer_dynamic *buf;                      /* temp accumulation buffer (on valstack) */
+	duk_hbuffer_dynamic *buf;                      /* temp accumulation buffer */
+	duk_bufwriter_ctx bw;                          /* bufwriter for temp accumulation */
 
 	duk_int_t token_count;                         /* number of tokens parsed */
 	duk_int_t token_limit;                         /* maximum token count before error (sanity backstop) */