aboutsummaryrefslogtreecommitdiff
path: root/py/formatfloat.c
diff options
context:
space:
mode:
authorDamien George <damien.p.george@gmail.com>2016-03-29 22:03:13 +0100
committerDamien George <damien.p.george@gmail.com>2016-03-29 22:03:13 +0100
commit03b8bb7ec9f2da5829794ebef0394c0a69b4f643 (patch)
treef3f7ec5dab888c49ae0ed02268fc38a9706df91d /py/formatfloat.c
parentd88250c06eb9f2f56b790c1411718134a148d9f4 (diff)
py/formatfloat: Fix case of float format where leading digit was "10".
When taking the logarithm of the float to determine the exponent, there are some edge cases that finish the log loop too large. Eg for an input value of 1e32-epsilon, this is actually less than 1e32 from the log-loop table and finishes as 10.0e31 when it should be 1.0e32. It is thus rendered as :e32 (: comes after 9 in ascii). There was the same problem with numbers less than 1.
Diffstat (limited to 'py/formatfloat.c')
-rw-r--r--py/formatfloat.c8
1 files changed, 7 insertions, 1 deletions
diff --git a/py/formatfloat.c b/py/formatfloat.c
index f7762b07d..21ed2d550 100644
--- a/py/formatfloat.c
+++ b/py/formatfloat.c
@@ -198,7 +198,7 @@ int mp_format_float(FPTYPE f, char *buf, size_t buf_size, char fmt, int prec, ch
if (e == 0) {
e_sign_char = '+';
}
- } else {
+ } else if (fp_isless1(f)) {
e++;
f *= FPCONST(10.0);
}
@@ -250,6 +250,12 @@ int mp_format_float(FPTYPE f, char *buf, size_t buf_size, char fmt, int prec, ch
}
}
+ // It can be that f was right on the edge of an entry in pos_pow needs to be reduced
+ if (f >= FPCONST(10.0)) {
+ e += 1;
+ f *= FPCONST(0.1);
+ }
+
// If the user specified fixed format (fmt == 'f') and e makes the
// number too big to fit into the available buffer, then we'll
// switch to the 'e' format.