py/lexer: Make lexer use an mp_reader as its source.

1 files changed, 13 insertions, 18 deletions

diff --git a/py/lexer.c b/py/lexer.c
index 9342ce8cc..c6ecdf1f8 100644
--- a/py/lexer.c
+++ b/py/lexer.c
@@ -52,6 +52,7 @@ STATIC bool str_strn_equal(const char *str, const char *strn, mp_uint_t len) {
     return i == len && *str == 0;
 }
 
+#define MP_LEXER_EOF ((unichar)MP_READER_EOF)
 #define CUR_CHAR(lex) ((lex)->chr0)
 
 STATIC bool is_end(mp_lexer_t *lex) {
@@ -145,7 +146,7 @@ STATIC void next_char(mp_lexer_t *lex) {
 
     lex->chr0 = lex->chr1;
     lex->chr1 = lex->chr2;
-    lex->chr2 = lex->stream_next_byte(lex->stream_data);
+    lex->chr2 = lex->reader.readbyte(lex->reader.data);
 
     if (lex->chr0 == '\r') {
         // CR is a new line, converted to LF
@@ -153,7 +154,7 @@ STATIC void next_char(mp_lexer_t *lex) {
         if (lex->chr1 == '\n') {
             // CR LF is a single new line
             lex->chr1 = lex->chr2;
-            lex->chr2 = lex->stream_next_byte(lex->stream_data);
+            lex->chr2 = lex->reader.readbyte(lex->reader.data);
         }
     }
 
@@ -689,21 +690,17 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, bool first_token) {
     }
 }
 
-mp_lexer_t *mp_lexer_new(qstr src_name, void *stream_data, mp_lexer_stream_next_byte_t stream_next_byte, mp_lexer_stream_close_t stream_close) {
+mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader) {
     mp_lexer_t *lex = m_new_obj_maybe(mp_lexer_t);
 
     // check for memory allocation error
     if (lex == NULL) {
-        if (stream_close) {
-            stream_close(stream_data);
-        }
+        reader.close(reader.data);
         return NULL;
     }
 
     lex->source_name = src_name;
-    lex->stream_data = stream_data;
-    lex->stream_next_byte = stream_next_byte;
-    lex->stream_close = stream_close;
+    lex->reader = reader;
     lex->line = 1;
     lex->column = 1;
     lex->emit_dent = 0;
@@ -724,9 +721,9 @@ mp_lexer_t *mp_lexer_new(qstr src_name, void *stream_data, mp_lexer_stream_next_
     lex->indent_level[0] = 0;
 
     // preload characters
-    lex->chr0 = stream_next_byte(stream_data);
-    lex->chr1 = stream_next_byte(stream_data);
-    lex->chr2 = stream_next_byte(stream_data);
+    lex->chr0 = reader.readbyte(reader.data);
+    lex->chr1 = reader.readbyte(reader.data);
+    lex->chr2 = reader.readbyte(reader.data);
 
     // if input stream is 0, 1 or 2 characters long and doesn't end in a newline, then insert a newline at the end
     if (lex->chr0 == MP_LEXER_EOF) {
@@ -756,7 +753,7 @@ mp_lexer_t *mp_lexer_new_from_str_len(qstr src_name, const char *str, mp_uint_t
     if (!mp_reader_new_mem(&reader, (const byte*)str, len, free_len)) {
         return NULL;
     }
-    return mp_lexer_new(src_name, reader.data, (mp_lexer_stream_next_byte_t)reader.readbyte, (mp_lexer_stream_close_t)reader.close);
+    return mp_lexer_new(src_name, reader);
 }
 
 #if MICROPY_READER_POSIX || MICROPY_READER_FATFS
@@ -767,7 +764,7 @@ mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
     if (ret != 0) {
         return NULL;
     }
-    return mp_lexer_new(qstr_from_str(filename), reader.data, (mp_lexer_stream_next_byte_t)reader.readbyte, (mp_lexer_stream_close_t)reader.close);
+    return mp_lexer_new(qstr_from_str(filename), reader);
 }
 
 #if MICROPY_HELPER_LEXER_UNIX
@@ -778,7 +775,7 @@ mp_lexer_t *mp_lexer_new_from_fd(qstr filename, int fd, bool close_fd) {
     if (ret != 0) {
         return NULL;
     }
-    return mp_lexer_new(filename, reader.data, (mp_lexer_stream_next_byte_t)reader.readbyte, (mp_lexer_stream_close_t)reader.close);
+    return mp_lexer_new(filename, reader);
 }
 
 #endif
@@ -787,9 +784,7 @@ mp_lexer_t *mp_lexer_new_from_fd(qstr filename, int fd, bool close_fd) {
 
 void mp_lexer_free(mp_lexer_t *lex) {
     if (lex) {
-        if (lex->stream_close) {
-            lex->stream_close(lex->stream_data);
-        }
+        lex->reader.close(lex->reader.data);
         vstr_clear(&lex->vstr);
         m_del(uint16_t, lex->indent_level, lex->alloc_indent_level);
         m_del_obj(mp_lexer_t, lex);