py: Parse big-int/float/imag constants directly in parser.

Previous to this patch, a big-int, float or imag constant was interned
(made into a qstr) and then parsed at runtime to create an object each
time it was needed.  This is wasteful in RAM and not efficient.  Now,
these constants are parsed straight away in the parser and turned into
objects.  This allows constants with large numbers of digits (so
addresses issue #1103) and takes us a step closer to #722.

1 files changed, 50 insertions, 21 deletions

diff --git a/py/parsenum.c b/py/parsenum.c
index 4706fef81..ffd065df4 100644
--- a/py/parsenum.c
+++ b/py/parsenum.c
@@ -35,7 +35,16 @@
 #include <math.h>
 #endif
 
-mp_obj_t mp_parse_num_integer(const char *restrict str_, mp_uint_t len, mp_uint_t base) {
+STATIC NORETURN void raise(mp_obj_t exc, mp_lexer_t *lex) {
+    // if lex!=NULL then the parser called us and we need to make a SyntaxError with traceback
+    if (lex != NULL) {
+        ((mp_obj_base_t*)exc)->type = &mp_type_SyntaxError;
+        mp_obj_exception_add_traceback(exc, lex->source_name, lex->tok_line, MP_QSTR_NULL);
+    }
+    nlr_raise(exc);
+}
+
+mp_obj_t mp_parse_num_integer(const char *restrict str_, mp_uint_t len, mp_uint_t base, mp_lexer_t *lex) {
     const byte *restrict str = (const byte *)str_;
     const byte *restrict top = str + len;
     bool neg = false;
@@ -43,6 +52,7 @@ mp_obj_t mp_parse_num_integer(const char *restrict str_, mp_uint_t len, mp_uint_
 
     // check radix base
     if ((base != 0 && base < 2) || base > 36) {
+        // this won't be reached if lex!=NULL
         nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "int() arg 2 must be >= 2 and <= 36"));
     }
 
@@ -132,12 +142,15 @@ overflow:
     }
 
 value_error:
+    // if lex!=NULL then the parser called us and we need to make a SyntaxError with traceback
     if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) {
-        nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError,
-            "invalid syntax for integer"));
+        mp_obj_t exc = mp_obj_new_exception_msg(&mp_type_SyntaxError,
+            "invalid syntax for integer");
+        raise(exc, lex);
     } else {
-        nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ValueError,
-            "invalid syntax for integer with base %d: '%s'", base, str));
+        mp_obj_t exc = mp_obj_new_exception_msg_varg(&mp_type_ValueError,
+            "invalid syntax for integer with base %d: '%s'", base, str_val_start);
+        raise(exc, lex);
     }
 }
 
@@ -147,7 +160,7 @@ typedef enum {
     PARSE_DEC_IN_EXP,
 } parse_dec_in_t;
 
-mp_obj_t mp_parse_num_decimal(const char *str, mp_uint_t len, bool allow_imag, bool force_complex) {
+mp_obj_t mp_parse_num_decimal(const char *str, mp_uint_t len, bool allow_imag, bool force_complex, mp_lexer_t *lex) {
 #if MICROPY_PY_BUILTINS_FLOAT
     const char *top = str + len;
     mp_float_t dec_val = 0;
@@ -168,6 +181,8 @@ mp_obj_t mp_parse_num_decimal(const char *str, mp_uint_t len, bool allow_imag, b
         }
     }
 
+    const char *str_val_start = str;
+
     // determine what the string is
     if (str < top && (str[0] | 0x20) == 'i') {
         // string starts with 'i', should be 'inf' or 'infinity' (case insensitive)
@@ -191,36 +206,43 @@ mp_obj_t mp_parse_num_decimal(const char *str, mp_uint_t len, bool allow_imag, b
         // string should be a decimal number
         parse_dec_in_t in = PARSE_DEC_IN_INTG;
         bool exp_neg = false;
+        mp_float_t frac_mult = 0.1;
         mp_int_t exp_val = 0;
-        mp_int_t exp_extra = 0;
-        for (; str < top; str++) {
-            mp_uint_t dig = *str;
+        while (str < top) {
+            mp_uint_t dig = *str++;
             if ('0' <= dig && dig <= '9') {
                 dig -= '0';
                 if (in == PARSE_DEC_IN_EXP) {
                     exp_val = 10 * exp_val + dig;
                 } else {
-                    dec_val = 10 * dec_val + dig;
                     if (in == PARSE_DEC_IN_FRAC) {
-                        exp_extra -= 1;
+                        dec_val += dig * frac_mult;
+                        frac_mult *= 0.1;
+                    } else {
+                        dec_val = 10 * dec_val + dig;
                     }
                 }
             } else if (in == PARSE_DEC_IN_INTG && dig == '.') {
                 in = PARSE_DEC_IN_FRAC;
             } else if (in != PARSE_DEC_IN_EXP && ((dig | 0x20) == 'e')) {
                 in = PARSE_DEC_IN_EXP;
-                if (str[1] == '+') {
-                    str++;
-                } else if (str[1] == '-') {
-                    str++;
-                    exp_neg = true;
+                if (str < top) {
+                    if (str[0] == '+') {
+                        str++;
+                    } else if (str[0] == '-') {
+                        str++;
+                        exp_neg = true;
+                    }
+                }
+                if (str == top) {
+                    goto value_error;
                 }
             } else if (allow_imag && (dig | 0x20) == 'j') {
-                str++;
                 imag = true;
                 break;
             } else {
                 // unknown character
+                str--;
                 break;
             }
         }
@@ -229,7 +251,6 @@ mp_obj_t mp_parse_num_decimal(const char *str, mp_uint_t len, bool allow_imag, b
         if (exp_neg) {
             exp_val = -exp_val;
         }
-        exp_val += exp_extra;
 
         // apply the exponent
         for (; exp_val > 0; exp_val--) {
@@ -245,13 +266,18 @@ mp_obj_t mp_parse_num_decimal(const char *str, mp_uint_t len, bool allow_imag, b
         dec_val = -dec_val;
     }
 
+    // check we parsed something
+    if (str == str_val_start) {
+        goto value_error;
+    }
+
     // skip trailing space
     for (; str < top && unichar_isspace(*str); str++) {
     }
 
     // check we reached the end of the string
     if (str != top) {
-        nlr_raise(mp_obj_new_exception_msg(&mp_type_SyntaxError, "invalid syntax for number"));
+        goto value_error;
     }
 
     // return the object
@@ -262,13 +288,16 @@ mp_obj_t mp_parse_num_decimal(const char *str, mp_uint_t len, bool allow_imag, b
         return mp_obj_new_complex(dec_val, 0);
 #else
     if (imag || force_complex) {
-        mp_not_implemented("complex values not supported");
+        raise(mp_obj_new_exception_msg(&mp_type_ValueError, "complex values not supported"), lex);
 #endif
     } else {
         return mp_obj_new_float(dec_val);
     }
 
+value_error:
+    raise(mp_obj_new_exception_msg(&mp_type_ValueError, "invalid syntax for number"), lex);
+
 #else
-    nlr_raise(mp_obj_new_exception_msg(&mp_type_SyntaxError, "decimal numbers not supported"));
+    raise(mp_obj_new_exception_msg(&mp_type_ValueError, "decimal numbers not supported"), lex);
 #endif
 }