You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

746 lines
23 KiB

11 years ago
/* lexer.c -- simple tokeniser for Python implementation
*/
#include <stdint.h>
#include <stdio.h>
#include <string.h>
11 years ago
#include <assert.h>
#include "misc.h"
#include "mpconfig.h"
#include "qstr.h"
11 years ago
#include "lexer.h"
#define TAB_SIZE (8)
// TODO seems that CPython allows NULL byte in the input stream
// don't know if that's intentional or not, but we don't allow it
struct _mp_lexer_t {
qstr source_name; // name of source
void *stream_data; // data for stream
mp_lexer_stream_next_char_t stream_next_char; // stream callback to get next char
mp_lexer_stream_close_t stream_close; // stream callback to free
11 years ago
unichar chr0, chr1, chr2; // current cached characters from source
11 years ago
uint line; // source line
uint column; // source column
int emit_dent; // non-zero when there are INDENT/DEDENT tokens to emit
int nested_bracket_level; // >0 when there are nested brackets over multiple lines
11 years ago
uint alloc_indent_level;
uint num_indent_level;
uint16_t *indent_level;
vstr_t vstr;
mp_token_t tok_cur;
11 years ago
};
// TODO replace with a call to a standard function
bool str_strn_equal(const char *str, const char *strn, int len) {
11 years ago
uint i = 0;
while (i < len && *str == *strn) {
11 years ago
++i;
++str;
++strn;
11 years ago
}
return i == len && *str == 0;
11 years ago
}
void mp_token_show(const mp_token_t *tok) {
printf("(%d:%d) kind:%d str:%p len:%d", tok->src_line, tok->src_column, tok->kind, tok->str, tok->len);
11 years ago
if (tok->str != NULL && tok->len > 0) {
const char *i = tok->str;
const char *j = i + tok->len;
printf(" ");
while (i < j) {
unichar c = utf8_get_char(i);
i = utf8_next_char(i);
if (unichar_isprint(c)) {
11 years ago
printf("%c", c);
} else {
printf("?");
}
}
}
printf("\n");
}
#define CUR_CHAR(lex) ((lex)->chr0)
STATIC bool is_end(mp_lexer_t *lex) {
return lex->chr0 == MP_LEXER_CHAR_EOF;
11 years ago
}
STATIC bool is_physical_newline(mp_lexer_t *lex) {
11 years ago
return lex->chr0 == '\n' || lex->chr0 == '\r';
}
STATIC bool is_char(mp_lexer_t *lex, char c) {
11 years ago
return lex->chr0 == c;
}
STATIC bool is_char_or(mp_lexer_t *lex, char c1, char c2) {
11 years ago
return lex->chr0 == c1 || lex->chr0 == c2;
}
STATIC bool is_char_or3(mp_lexer_t *lex, char c1, char c2, char c3) {
11 years ago
return lex->chr0 == c1 || lex->chr0 == c2 || lex->chr0 == c3;
}
/*
STATIC bool is_char_following(mp_lexer_t *lex, char c) {
11 years ago
return lex->chr1 == c;
}
*/
STATIC bool is_char_following_or(mp_lexer_t *lex, char c1, char c2) {
11 years ago
return lex->chr1 == c1 || lex->chr1 == c2;
}
STATIC bool is_char_following_following_or(mp_lexer_t *lex, char c1, char c2) {
11 years ago
return lex->chr2 == c1 || lex->chr2 == c2;
}
STATIC bool is_char_and(mp_lexer_t *lex, char c1, char c2) {
11 years ago
return lex->chr0 == c1 && lex->chr1 == c2;
}
STATIC bool is_whitespace(mp_lexer_t *lex) {
return unichar_isspace(lex->chr0);
11 years ago
}
STATIC bool is_letter(mp_lexer_t *lex) {
return unichar_isalpha(lex->chr0);
11 years ago
}
STATIC bool is_digit(mp_lexer_t *lex) {
return unichar_isdigit(lex->chr0);
11 years ago
}
STATIC bool is_following_digit(mp_lexer_t *lex) {
return unichar_isdigit(lex->chr1);
11 years ago
}
STATIC bool is_following_odigit(mp_lexer_t *lex) {
return lex->chr1 >= '0' && lex->chr1 <= '7';
}
11 years ago
// TODO UNICODE include unicode characters in definition of identifiers
STATIC bool is_head_of_identifier(mp_lexer_t *lex) {
11 years ago
return is_letter(lex) || lex->chr0 == '_';
}
// TODO UNICODE include unicode characters in definition of identifiers
STATIC bool is_tail_of_identifier(mp_lexer_t *lex) {
11 years ago
return is_head_of_identifier(lex) || is_digit(lex);
}
STATIC void next_char(mp_lexer_t *lex) {
if (lex->chr0 == MP_LEXER_CHAR_EOF) {
11 years ago
return;
}
int advance = 1;
if (lex->chr0 == '\n') {
// LF is a new line
++lex->line;
lex->column = 1;
} else if (lex->chr0 == '\r') {
// CR is a new line
++lex->line;
lex->column = 1;
if (lex->chr1 == '\n') {
// CR LF is a single new line
advance = 2;
}
} else if (lex->chr0 == '\t') {
// a tab
lex->column = (((lex->column - 1 + TAB_SIZE) / TAB_SIZE) * TAB_SIZE) + 1;
} else {
// a character worth one column
++lex->column;
}
for (; advance > 0; advance--) {
lex->chr0 = lex->chr1;
lex->chr1 = lex->chr2;
lex->chr2 = lex->stream_next_char(lex->stream_data);
if (lex->chr2 == MP_LEXER_CHAR_EOF) {
11 years ago
// EOF
if (lex->chr1 != MP_LEXER_CHAR_EOF && lex->chr1 != '\n' && lex->chr1 != '\r') {
11 years ago
lex->chr2 = '\n'; // insert newline at end of file
}
}
}
}
void indent_push(mp_lexer_t *lex, uint indent) {
11 years ago
if (lex->num_indent_level >= lex->alloc_indent_level) {
lex->indent_level = m_renew(uint16_t, lex->indent_level, lex->alloc_indent_level, lex->alloc_indent_level * 2);
11 years ago
lex->alloc_indent_level *= 2;
}
lex->indent_level[lex->num_indent_level++] = indent;
}
uint indent_top(mp_lexer_t *lex) {
11 years ago
return lex->indent_level[lex->num_indent_level - 1];
}
void indent_pop(mp_lexer_t *lex) {
11 years ago
lex->num_indent_level -= 1;
}
// some tricky operator encoding:
// <op> = begin with <op>, if this opchar matches then begin here
// e<op> = end with <op>, if this opchar matches then end
// E<op> = mandatory end with <op>, this opchar must match, then end
// c<op> = continue with <op>, if this opchar matches then continue matching
// this means if the start of two ops are the same then they are equal til the last char
STATIC const char *tok_enc =
11 years ago
"()[]{},:;@~" // singles
"<e=c<e=" // < <= << <<=
">e=c>e=" // > >= >> >>=
"*e=c*e=" // * *= ** **=
"+e=" // + +=
"-e=e>" // - -= ->
"&e=" // & &=
"|e=" // | |=
"/e=c/e=" // / /= // //=
"%e=" // % %=
"^e=" // ^ ^=
"=e=" // = ==
"!E=" // !=
".c.E."; // . ...
// TODO static assert that number of tokens is less than 256 so we can safely make this table with byte sized entries
STATIC const uint8_t tok_enc_kind[] = {
MP_TOKEN_DEL_PAREN_OPEN, MP_TOKEN_DEL_PAREN_CLOSE,
MP_TOKEN_DEL_BRACKET_OPEN, MP_TOKEN_DEL_BRACKET_CLOSE,
MP_TOKEN_DEL_BRACE_OPEN, MP_TOKEN_DEL_BRACE_CLOSE,
MP_TOKEN_DEL_COMMA, MP_TOKEN_DEL_COLON, MP_TOKEN_DEL_SEMICOLON, MP_TOKEN_DEL_AT, MP_TOKEN_OP_TILDE,
MP_TOKEN_OP_LESS, MP_TOKEN_OP_LESS_EQUAL, MP_TOKEN_OP_DBL_LESS, MP_TOKEN_DEL_DBL_LESS_EQUAL,
MP_TOKEN_OP_MORE, MP_TOKEN_OP_MORE_EQUAL, MP_TOKEN_OP_DBL_MORE, MP_TOKEN_DEL_DBL_MORE_EQUAL,
MP_TOKEN_OP_STAR, MP_TOKEN_DEL_STAR_EQUAL, MP_TOKEN_OP_DBL_STAR, MP_TOKEN_DEL_DBL_STAR_EQUAL,
MP_TOKEN_OP_PLUS, MP_TOKEN_DEL_PLUS_EQUAL,
MP_TOKEN_OP_MINUS, MP_TOKEN_DEL_MINUS_EQUAL, MP_TOKEN_DEL_MINUS_MORE,
MP_TOKEN_OP_AMPERSAND, MP_TOKEN_DEL_AMPERSAND_EQUAL,
MP_TOKEN_OP_PIPE, MP_TOKEN_DEL_PIPE_EQUAL,
MP_TOKEN_OP_SLASH, MP_TOKEN_DEL_SLASH_EQUAL, MP_TOKEN_OP_DBL_SLASH, MP_TOKEN_DEL_DBL_SLASH_EQUAL,
MP_TOKEN_OP_PERCENT, MP_TOKEN_DEL_PERCENT_EQUAL,
MP_TOKEN_OP_CARET, MP_TOKEN_DEL_CARET_EQUAL,
MP_TOKEN_DEL_EQUAL, MP_TOKEN_OP_DBL_EQUAL,
MP_TOKEN_OP_NOT_EQUAL,
MP_TOKEN_DEL_PERIOD, MP_TOKEN_ELLIPSIS,
11 years ago
};
// must have the same order as enum in lexer.h
STATIC const char *tok_kw[] = {
11 years ago
"False",
"None",
"True",
"and",
"as",
"assert",
"break",
"class",
"continue",
"def",
"del",
"elif",
"else",
"except",
"finally",
"for",
"from",
"global",
"if",
"import",
"in",
"is",
"lambda",
"nonlocal",
"not",
"or",
"pass",
"raise",
"return",
"try",
"while",
"with",
"yield",
NULL,
};
STATIC int hex_digit(unichar c) {
// c is assumed to be hex digit
int n = c - '0';
if (n > 9) {
n &= ~('a' - 'A');
n -= ('A' - ('9' + 1));
}
return n;
}
// This is called with CUR_CHAR() before first hex digit, and should return with
// it pointing to last hex digit
STATIC bool get_hex(mp_lexer_t *lex, int num_digits, uint *result) {
uint num = 0;
while (num_digits-- != 0) {
next_char(lex);
unichar c = CUR_CHAR(lex);
if (!unichar_isxdigit(c)) {
return false;
}
num = (num << 4) + hex_digit(c);
}
*result = num;
return true;
}
STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool first_token) {
// skip white space and comments
11 years ago
bool had_physical_newline = false;
while (!is_end(lex)) {
if (is_physical_newline(lex)) {
had_physical_newline = true;
next_char(lex);
} else if (is_whitespace(lex)) {
next_char(lex);
} else if (is_char(lex, '#')) {
next_char(lex);
while (!is_end(lex) && !is_physical_newline(lex)) {
next_char(lex);
}
// had_physical_newline will be set on next loop
} else if (is_char(lex, '\\')) {
// backslash (outside string literals) must appear just before a physical newline
next_char(lex);
if (!is_physical_newline(lex)) {
// SyntaxError: unexpected character after line continuation character
tok->src_line = lex->line;
tok->src_column = lex->column;
tok->kind = MP_TOKEN_BAD_LINE_CONTINUATION;
vstr_reset(&lex->vstr);
tok->str = vstr_str(&lex->vstr);
tok->len = 0;
return;
11 years ago
} else {
next_char(lex);
}
} else {
break;
}
}
// set token source information
11 years ago
tok->src_line = lex->line;
tok->src_column = lex->column;
// start new token text
vstr_reset(&lex->vstr);
if (first_token && lex->line == 1 && lex->column != 1) {
// check that the first token is in the first column
// if first token is not on first line, we get a physical newline and
// this check is done as part of normal indent/dedent checking below
// (done to get equivalence with CPython)
tok->kind = MP_TOKEN_INDENT;
} else if (lex->emit_dent < 0) {
tok->kind = MP_TOKEN_DEDENT;
11 years ago
lex->emit_dent += 1;
} else if (lex->emit_dent > 0) {
tok->kind = MP_TOKEN_INDENT;
11 years ago
lex->emit_dent -= 1;
} else if (had_physical_newline && lex->nested_bracket_level == 0) {
tok->kind = MP_TOKEN_NEWLINE;
11 years ago
uint num_spaces = lex->column - 1;
lex->emit_dent = 0;
if (num_spaces == indent_top(lex)) {
} else if (num_spaces > indent_top(lex)) {
indent_push(lex, num_spaces);
lex->emit_dent += 1;
} else {
while (num_spaces < indent_top(lex)) {
indent_pop(lex);
lex->emit_dent -= 1;
}
if (num_spaces != indent_top(lex)) {
tok->kind = MP_TOKEN_DEDENT_MISMATCH;
11 years ago
}
}
} else if (is_end(lex)) {
if (indent_top(lex) > 0) {
tok->kind = MP_TOKEN_NEWLINE;
11 years ago
lex->emit_dent = 0;
while (indent_top(lex) > 0) {
indent_pop(lex);
lex->emit_dent -= 1;
}
} else {
tok->kind = MP_TOKEN_END;
11 years ago
}
} else if (is_char_or(lex, '\'', '\"')
|| (is_char_or3(lex, 'r', 'u', 'b') && is_char_following_or(lex, '\'', '\"'))
|| ((is_char_and(lex, 'r', 'b') || is_char_and(lex, 'b', 'r')) && is_char_following_following_or(lex, '\'', '\"'))) {
// a string or bytes literal
// parse type codes
bool is_raw = false;
bool is_bytes = false;
if (is_char(lex, 'u')) {
next_char(lex);
} else if (is_char(lex, 'b')) {
is_bytes = true;
next_char(lex);
if (is_char(lex, 'r')) {
is_raw = true;
next_char(lex);
}
} else if (is_char(lex, 'r')) {
is_raw = true;
next_char(lex);
if (is_char(lex, 'b')) {
is_bytes = true;
next_char(lex);
}
}
// set token kind
if (is_bytes) {
tok->kind = MP_TOKEN_BYTES;
11 years ago
} else {
tok->kind = MP_TOKEN_STRING;
11 years ago
}
// get first quoting character
char quote_char = '\'';
if (is_char(lex, '\"')) {
quote_char = '\"';
}
next_char(lex);
// work out if it's a single or triple quoted literal
int num_quotes;
if (is_char_and(lex, quote_char, quote_char)) {
// triple quotes
next_char(lex);
next_char(lex);
num_quotes = 3;
} else {
// single quotes
num_quotes = 1;
}
// parse the literal
int n_closing = 0;
while (!is_end(lex) && (num_quotes > 1 || !is_char(lex, '\n')) && n_closing < num_quotes) {
if (is_char(lex, quote_char)) {
n_closing += 1;
vstr_add_char(&lex->vstr, CUR_CHAR(lex));
11 years ago
} else {
n_closing = 0;
if (!is_raw && is_char(lex, '\\')) {
next_char(lex);
unichar c = CUR_CHAR(lex);
switch (c) {
case MP_LEXER_CHAR_EOF: break; // TODO a proper error message?
case '\n': c = MP_LEXER_CHAR_EOF; break; // TODO check this works correctly (we are supposed to ignore it
case '\\': break;
case '\'': break;
case '"': break;
case 'a': c = 0x07; break;
case 'b': c = 0x08; break;
case 't': c = 0x09; break;
case 'n': c = 0x0a; break;
case 'v': c = 0x0b; break;
case 'f': c = 0x0c; break;
case 'r': c = 0x0d; break;
case 'x':
{
uint num = 0;
if (!get_hex(lex, 2, &num)) {
// TODO error message
assert(0);
}
c = num;
break;
}
case 'N': break; // TODO \N{name} only in strings
case 'u': break; // TODO \uxxxx only in strings
case 'U': break; // TODO \Uxxxxxxxx only in strings
default:
if (c >= '0' && c <= '7') {
// Octal sequence, 1-3 chars
int digits = 3;
int num = c - '0';
while (is_following_odigit(lex) && --digits != 0) {
next_char(lex);
num = num * 8 + (CUR_CHAR(lex) - '0');
}
c = num;
} else {
// unrecognised escape character; CPython lets this through verbatim as '\' and then the character
vstr_add_char(&lex->vstr, '\\');
}
break;
}
if (c != MP_LEXER_CHAR_EOF) {
vstr_add_char(&lex->vstr, c);
}
} else {
vstr_add_char(&lex->vstr, CUR_CHAR(lex));
11 years ago
}
}
next_char(lex);
}
// check we got the required end quotes
if (n_closing < num_quotes) {
tok->kind = MP_TOKEN_LONELY_STRING_OPEN;
11 years ago
}
// cut off the end quotes from the token text
vstr_cut_tail(&lex->vstr, n_closing);
11 years ago
} else if (is_head_of_identifier(lex)) {
tok->kind = MP_TOKEN_NAME;
11 years ago
// get first char
vstr_add_char(&lex->vstr, CUR_CHAR(lex));
11 years ago
next_char(lex);
// get tail chars
11 years ago
while (!is_end(lex) && is_tail_of_identifier(lex)) {
vstr_add_char(&lex->vstr, CUR_CHAR(lex));
11 years ago
next_char(lex);
}
} else if (is_digit(lex) || (is_char(lex, '.') && is_following_digit(lex))) {
tok->kind = MP_TOKEN_NUMBER;
11 years ago
// get first char
vstr_add_char(&lex->vstr, CUR_CHAR(lex));
11 years ago
next_char(lex);
// get tail chars
11 years ago
while (!is_end(lex)) {
if (is_char_or(lex, 'e', 'E')) {
vstr_add_char(&lex->vstr, 'e');
11 years ago
next_char(lex);
if (is_char(lex, '+') || is_char(lex, '-')) {
vstr_add_char(&lex->vstr, CUR_CHAR(lex));
11 years ago
next_char(lex);
}
} else if (is_letter(lex) || is_digit(lex) || is_char_or(lex, '_', '.')) {
vstr_add_char(&lex->vstr, CUR_CHAR(lex));
11 years ago
next_char(lex);
} else {
break;
}
}
} else {
// search for encoded delimiter or operator
const char *t = tok_enc;
uint tok_enc_index = 0;
for (; *t != 0 && !is_char(lex, *t); t += 1) {
if (*t == 'e' || *t == 'c') {
t += 1;
} else if (*t == 'E') {
tok_enc_index -= 1;
t += 1;
}
tok_enc_index += 1;
}
next_char(lex);
if (*t == 0) {
// didn't match any delimiter or operator characters
tok->kind = MP_TOKEN_INVALID;
11 years ago
} else {
// matched a delimiter or operator character
// get the maximum characters for a valid token
t += 1;
uint t_index = tok_enc_index;
for (;;) {
for (; *t == 'e'; t += 1) {
t += 1;
t_index += 1;
if (is_char(lex, *t)) {
next_char(lex);
tok_enc_index = t_index;
break;
}
}
if (*t == 'E') {
t += 1;
if (is_char(lex, *t)) {
next_char(lex);
tok_enc_index = t_index;
} else {
tok->kind = MP_TOKEN_INVALID;
11 years ago
}
break;
}
if (*t == 'c') {
t += 1;
t_index += 1;
if (is_char(lex, *t)) {
next_char(lex);
tok_enc_index = t_index;
t += 1;
} else {
break;
}
} else {
break;
}
}
// set token kind
tok->kind = tok_enc_kind[tok_enc_index];
// compute bracket level for implicit line joining
if (tok->kind == MP_TOKEN_DEL_PAREN_OPEN || tok->kind == MP_TOKEN_DEL_BRACKET_OPEN || tok->kind == MP_TOKEN_DEL_BRACE_OPEN) {
11 years ago
lex->nested_bracket_level += 1;
} else if (tok->kind == MP_TOKEN_DEL_PAREN_CLOSE || tok->kind == MP_TOKEN_DEL_BRACKET_CLOSE || tok->kind == MP_TOKEN_DEL_BRACE_CLOSE) {
11 years ago
lex->nested_bracket_level -= 1;
}
}
}
// point token text to vstr buffer
tok->str = vstr_str(&lex->vstr);
tok->len = vstr_len(&lex->vstr);
11 years ago
// check for keywords
if (tok->kind == MP_TOKEN_NAME) {
11 years ago
for (int i = 0; tok_kw[i] != NULL; i++) {
if (str_strn_equal(tok_kw[i], tok->str, tok->len)) {
tok->kind = MP_TOKEN_KW_FALSE + i;
11 years ago
break;
}
}
}
}
mp_lexer_t *mp_lexer_new(qstr src_name, void *stream_data, mp_lexer_stream_next_char_t stream_next_char, mp_lexer_stream_close_t stream_close) {
mp_lexer_t *lex = m_new(mp_lexer_t, 1);
11 years ago
lex->source_name = src_name;
lex->stream_data = stream_data;
lex->stream_next_char = stream_next_char;
lex->stream_close = stream_close;
11 years ago
lex->line = 1;
lex->column = 1;
lex->emit_dent = 0;
lex->nested_bracket_level = 0;
lex->alloc_indent_level = 16;
lex->num_indent_level = 1;
lex->indent_level = m_new(uint16_t, lex->alloc_indent_level);
lex->indent_level[0] = 0;
vstr_init(&lex->vstr, 32);
11 years ago
// preload characters
lex->chr0 = stream_next_char(stream_data);
lex->chr1 = stream_next_char(stream_data);
lex->chr2 = stream_next_char(stream_data);
// if input stream is 0, 1 or 2 characters long and doesn't end in a newline, then insert a newline at the end
if (lex->chr0 == MP_LEXER_CHAR_EOF) {
lex->chr0 = '\n';
} else if (lex->chr1 == MP_LEXER_CHAR_EOF) {
11 years ago
if (lex->chr0 != '\n' && lex->chr0 != '\r') {
lex->chr1 = '\n';
11 years ago
}
} else if (lex->chr2 == MP_LEXER_CHAR_EOF) {
11 years ago
if (lex->chr1 != '\n' && lex->chr1 != '\r') {
lex->chr2 = '\n';
11 years ago
}
}
// preload first token
mp_lexer_next_token_into(lex, &lex->tok_cur, true);
11 years ago
return lex;
}
void mp_lexer_free(mp_lexer_t *lex) {
if (lex) {
if (lex->stream_close) {
lex->stream_close(lex->stream_data);
}
vstr_clear(&lex->vstr);
m_del(uint16_t, lex->indent_level, lex->alloc_indent_level);
m_del_obj(mp_lexer_t, lex);
11 years ago
}
}
qstr mp_lexer_source_name(mp_lexer_t *lex) {
return lex->source_name;
}
void mp_lexer_to_next(mp_lexer_t *lex) {
mp_lexer_next_token_into(lex, &lex->tok_cur, false);
11 years ago
}
const mp_token_t *mp_lexer_cur(const mp_lexer_t *lex) {
11 years ago
return &lex->tok_cur;
}
bool mp_lexer_is_kind(mp_lexer_t *lex, mp_token_kind_t kind) {
11 years ago
return lex->tok_cur.kind == kind;
}
/*
bool mp_lexer_is_str(mp_lexer_t *lex, const char *str) {
return mp_token_is_str(&lex->tok_cur, str);
11 years ago
}
bool mp_lexer_opt_kind(mp_lexer_t *lex, mp_token_kind_t kind) {
if (mp_lexer_is_kind(lex, kind)) {
mp_lexer_to_next(lex);
11 years ago
return true;
}
return false;
}
bool mp_lexer_opt_str(mp_lexer_t *lex, const char *str) {
if (mp_lexer_is_str(lex, str)) {
mp_lexer_to_next(lex);
11 years ago
return true;
}
return false;
}
*/
bool mp_lexer_show_error_pythonic_prefix(mp_lexer_t *lex) {
printf(" File \"%s\", line %d column %d\n", qstr_str(lex->source_name), lex->tok_cur.src_line, lex->tok_cur.src_column);
return false;
11 years ago
}
bool mp_lexer_show_error_pythonic(mp_lexer_t *lex, const char *msg) {
printf(" File \"%s\", line %d column %d\n%s\n", qstr_str(lex->source_name), lex->tok_cur.src_line, lex->tok_cur.src_column, msg);
return false;
}