ubsan: d-demangle.c:214 signed integer overflow

Running the libiberty testsuite
./test-demangle < libiberty/testsuite/d-demangle-expected
libiberty/d-demangle.c:214:14: runtime error: signed integer overflow: 922337203 * 10 cannot be represented in type 'long int'

On looking at silencing ubsan, I found a real bug in dlang_number.
For a 32-bit long, some overflows won't be detected.  For example,
21474836480.  Why?  Well 214748364 * 10 is 0x7FFFFFF8 (no overflow so
far).  Adding 8 gives 0x80000000 (which does overflow but there is no
test for that overflow in the code).  Then multiplying 0x80000000 * 10
= 0x500000000 = 0 won't be caught by the multiplication overflow test.
The same holds for a 64-bit long using similarly crafted digit
sequences.

	* d-demangle.c: Include limits.h.
	(ULONG_MAX, UINT_MAX): Provide fall-back definition.
	(dlang_number): Simplify and correct overflow test.  Only
	write *ret on returning non-NULL.  Make "ret" an unsigned long*.
	Only succeed for result of [0,UINT_MAX].
	(dlang_decode_backref): Simplify and correct overflow test.
	Only write *ret on returning non-NULL.  Only succeed for
	result [1,MAX_LONG].
	(dlang_backref): Remove now unnecessary range check.
	(dlang_symbol_name_p): Likewise.
	(string_need): Take a size_t n arg, and use size_t tem.
	(string_append): Use size_t n.
	(string_appendn, string_prependn): Take a size_t n arg.
	(TEMPLATE_LENGTH_UNKNOWN): Define as -1UL.
	(dlang_lname, dlang_parse_template): Take an unsigned long len
	arg.
	(dlang_symbol_backref, dlang_identifier, dlang_parse_integer),
	(dlang_parse_integer, dlang_parse_string),
	(dlang_parse_arrayliteral, dlang_parse_assocarray),
	(dlang_parse_structlit, dlang_parse_tuple),
	(dlang_template_symbol_param, dlang_template_args): Use
	unsigned long variables.
	* testsuite/d-demangle-expected: Add new tests.

2 files changed, 69 insertions, 45 deletions

diff --git a/libiberty/d-demangle.c b/libiberty/d-demangle.c
index 2e52eff7617..cd3bc96303e 100644
--- a/libiberty/d-demangle.c
+++ b/libiberty/d-demangle.c
@@ -31,6 +31,9 @@ If not, see <http://www.gnu.org/licenses/>.  */
 #ifdef HAVE_CONFIG_H
 #include "config.h"
 #endif
+#ifdef HAVE_LIMITS_H
+#include <limits.h>
+#endif
 
 #include "safe-ctype.h"
 
@@ -45,6 +48,13 @@ If not, see <http://www.gnu.org/licenses/>.  */
 #include <demangle.h>
 #include "libiberty.h"
 
+#ifndef ULONG_MAX
+#define	ULONG_MAX	(~0UL)
+#endif
+#ifndef UINT_MAX
+#define	UINT_MAX	(~0U)
+#endif
+
 /* A mini string-handling package */
 
 typedef struct string		/* Beware: these aren't required to be */
@@ -55,9 +65,9 @@ typedef struct string		/* Beware: these aren't required to be */
 } string;
 
 static void
-string_need (string *s, int n)
+string_need (string *s, size_t n)
 {
-  int tem;
+  size_t tem;
 
   if (s->b == NULL)
     {
@@ -68,7 +78,7 @@ string_need (string *s, int n)
       s->p = s->b = XNEWVEC (char, n);
       s->e = s->b + n;
     }
-  else if (s->e - s->p < n)
+  else if ((size_t) (s->e - s->p) < n)
     {
       tem = s->p - s->b;
       n += tem;
@@ -117,14 +127,14 @@ string_setlength (string *s, int n)
 static void
 string_append (string *p, const char *s)
 {
-  int n = strlen (s);
+  size_t n = strlen (s);
   string_need (p, n);
   memcpy (p->p, s, n);
   p->p += n;
 }
 
 static void
-string_appendn (string *p, const char *s, int n)
+string_appendn (string *p, const char *s, size_t n)
 {
   if (n != 0)
     {
@@ -135,7 +145,7 @@ string_appendn (string *p, const char *s, int n)
 }
 
 static void
-string_prependn (string *p, const char *s, int n)
+string_prependn (string *p, const char *s, size_t n)
 {
   char *q;
 
@@ -170,7 +180,7 @@ struct dlang_info
 };
 
 /* Pass as the LEN to dlang_parse_template if symbol length is not known.  */
-enum { TEMPLATE_LENGTH_UNKNOWN = -1 };
+#define TEMPLATE_LENGTH_UNKNOWN (-1UL)
 
 /* Prototypes for forward referenced functions */
 static const char *dlang_function_type (string *, const char *,
@@ -193,38 +203,39 @@ static const char *dlang_parse_tuple (string *, const char *,
 				      struct dlang_info *);
 
 static const char *dlang_parse_template (string *, const char *,
-					 struct dlang_info *, long);
+					 struct dlang_info *, unsigned long);
 
-static const char *dlang_lname (string *, const char *, long);
+static const char *dlang_lname (string *, const char *, unsigned long);
 
 
 /* Extract the number from MANGLED, and assign the result to RET.
-   Return the remaining string on success or NULL on failure.  */
+   Return the remaining string on success or NULL on failure.
+   A result larger than UINT_MAX is considered a failure.  */
 static const char *
-dlang_number (const char *mangled, long *ret)
+dlang_number (const char *mangled, unsigned long *ret)
 {
   /* Return NULL if trying to extract something that isn't a digit.  */
   if (mangled == NULL || !ISDIGIT (*mangled))
     return NULL;
 
-  (*ret) = 0;
+  unsigned long val = 0;
 
   while (ISDIGIT (*mangled))
     {
-      (*ret) *= 10;
+      unsigned long digit = mangled[0] - '0';
 
-      /* If an overflow occured when multiplying by ten, the result
-	 will not be a multiple of ten.  */
-      if ((*ret % 10) != 0)
+      /* Check for overflow.  */
+      if (val > (UINT_MAX - digit) / 10)
 	return NULL;
 
-      (*ret) += mangled[0] - '0';
+      val = val * 10 + digit;
       mangled++;
     }
 
-  if (*mangled == '\0' || *ret < 0)
+  if (*mangled == '\0')
     return NULL;
 
+  *ret = val;
   return mangled;
 }
 
@@ -273,7 +284,8 @@ dlang_call_convention_p (const char *mangled)
 }
 
 /* Extract the back reference position from MANGLED, and assign the result
-   to RET.  Return the remaining string on success or NULL on failure.  */
+   to RET.  Return the remaining string on success or NULL on failure.
+   A result <= 0 is a failure.  */
 static const char *
 dlang_decode_backref (const char *mangled, long *ret)
 {
@@ -294,24 +306,26 @@ dlang_decode_backref (const char *mangled, long *ret)
 	    [A-Z] NumberBackRef
 	    ^
    */
-  (*ret) = 0;
+  unsigned long val = 0;
 
   while (ISALPHA (*mangled))
     {
-      (*ret) *= 26;
+      /* Check for overflow.  */
+      if (val > (ULONG_MAX - 25) / 26)
+	break;
 
-      /* If an overflow occured when multiplying by 26, the result
-	 will not be a multiple of 26.  */
-      if ((*ret % 26) != 0)
-	return NULL;
+      val *= 26;
 
       if (mangled[0] >= 'a' && mangled[0] <= 'z')
 	{
-	  (*ret) += mangled[0] - 'a';
+	  val += mangled[0] - 'a';
+	  if ((long) val <= 0)
+	    break;
+	  *ret = val;
 	  return mangled + 1;
 	}
 
-      (*ret) += mangled[0] - 'A';
+      val += mangled[0] - 'A';
       mangled++;
     }
 
@@ -337,7 +351,7 @@ dlang_backref (const char *mangled, const char **ret, struct dlang_info *info)
   if (mangled == NULL)
     return NULL;
 
-  if (refpos <= 0 || refpos > qpos - info->s)
+  if (refpos > qpos - info->s)
     return NULL;
 
   /* Set the position of the back reference.  */
@@ -359,7 +373,7 @@ dlang_symbol_backref (string *decl, const char *mangled,
 	    ^
    */
   const char *backref;
-  long len;
+  unsigned long len;
 
   /* Get position of the back reference.  */
   mangled = dlang_backref (mangled, &backref, info);
@@ -435,7 +449,7 @@ dlang_symbol_name_p (const char *mangled, struct dlang_info *info)
     return 0;
 
   mangled = dlang_decode_backref (mangled + 1, &ret);
-  if (mangled == NULL || ret <= 0 || ret > qref - info->s)
+  if (mangled == NULL || ret > qref - info->s)
     return 0;
 
   return ISDIGIT (qref[-ret]);
@@ -993,7 +1007,7 @@ dlang_type (string *decl, const char *mangled, struct dlang_info *info)
 static const char *
 dlang_identifier (string *decl, const char *mangled, struct dlang_info *info)
 {
-  long len;
+  unsigned long len;
 
   if (mangled == NULL || *mangled == '\0')
     return NULL;
@@ -1011,7 +1025,7 @@ dlang_identifier (string *decl, const char *mangled, struct dlang_info *info)
   if (endptr == NULL || len == 0)
     return NULL;
 
-  if (strlen (endptr) < (size_t) len)
+  if (strlen (endptr) < len)
     return NULL;
 
   mangled = endptr;
@@ -1028,7 +1042,7 @@ dlang_identifier (string *decl, const char *mangled, struct dlang_info *info)
    with special treatment for some magic compiler generted symbols.
    Return the remaining string on success or NULL on failure.  */
 static const char *
-dlang_lname (string *decl, const char *mangled, long len)
+dlang_lname (string *decl, const char *mangled, unsigned long len)
 {
   switch (len)
     {
@@ -1127,7 +1141,7 @@ dlang_parse_integer (string *decl, const char *mangled, char type)
       char value[20];
       int pos = sizeof(value);
       int width = 0;
-      long val;
+      unsigned long val;
 
       mangled = dlang_number (mangled, &val);
       if (mangled == NULL)
@@ -1183,7 +1197,7 @@ dlang_parse_integer (string *decl, const char *mangled, char type)
   else if (type == 'b')
     {
       /* Parse boolean value.  */
-      long val;
+      unsigned long val;
 
       mangled = dlang_number (mangled, &val);
       if (mangled == NULL)
@@ -1302,7 +1316,7 @@ static const char *
 dlang_parse_string (string *decl, const char *mangled)
 {
   char type = *mangled;
-  long len;
+  unsigned long len;
 
   mangled++;
   mangled = dlang_number (mangled, &len);
@@ -1366,7 +1380,7 @@ dlang_parse_string (string *decl, const char *mangled)
 static const char *
 dlang_parse_arrayliteral (string *decl, const char *mangled)
 {
-  long elements;
+  unsigned long elements;
 
   mangled = dlang_number (mangled, &elements);
   if (mangled == NULL)
@@ -1392,7 +1406,7 @@ dlang_parse_arrayliteral (string *decl, const char *mangled)
 static const char *
 dlang_parse_assocarray (string *decl, const char *mangled)
 {
-  long elements;
+  unsigned long elements;
 
   mangled = dlang_number (mangled, &elements);
   if (mangled == NULL)
@@ -1423,7 +1437,7 @@ dlang_parse_assocarray (string *decl, const char *mangled)
 static const char *
 dlang_parse_structlit (string *decl, const char *mangled, const char *name)
 {
-  long args;
+  unsigned long args;
 
   mangled = dlang_number (mangled, &args);
   if (mangled == NULL)
@@ -1650,7 +1664,7 @@ dlang_parse_qualified (string *decl, const char *mangled,
 static const char *
 dlang_parse_tuple (string *decl, const char *mangled, struct dlang_info *info)
 {
-  long elements;
+  unsigned long elements;
 
   mangled = dlang_number (mangled, &elements);
   if (mangled == NULL)
@@ -1685,7 +1699,7 @@ dlang_template_symbol_param (string *decl, const char *mangled,
   if (*mangled == 'Q')
     return dlang_parse_qualified (decl, mangled, info, 0);
 
-  long len;
+  unsigned long len;
   const char *endptr = dlang_number (mangled, &len);
 
   if (endptr == NULL || len == 0)
@@ -1798,12 +1812,12 @@ dlang_template_args (string *decl, const char *mangled, struct dlang_info *info)
 	}
 	case 'X': /* Externally mangled parameter.  */
 	{
-	  long len;
+	  unsigned long len;
 	  const char *endptr;
 
 	  mangled++;
 	  endptr = dlang_number (mangled, &len);
-	  if (endptr == NULL || strlen (endptr) < (size_t) len)
+	  if (endptr == NULL || strlen (endptr) < len)
 	    return NULL;
 
 	  string_appendn (decl, endptr, len);
@@ -1823,7 +1837,7 @@ dlang_template_args (string *decl, const char *mangled, struct dlang_info *info)
    Returns the remaining signature on success or NULL on failure.  */
 static const char *
 dlang_parse_template (string *decl, const char *mangled,
-		      struct dlang_info *info, long len)
+		      struct dlang_info *info, unsigned long len)
 {
   const char *start = mangled;
   string args;
@@ -1859,7 +1873,9 @@ dlang_parse_template (string *decl, const char *mangled,
   string_delete (&args);
 
   /* Check for template name length mismatch.  */
-  if (len != TEMPLATE_LENGTH_UNKNOWN && mangled && (mangled - start) != len)
+  if (len != TEMPLATE_LENGTH_UNKNOWN
+      && mangled
+      && (unsigned long) (mangled - start) != len)
     return NULL;
 
   return mangled;
diff --git a/libiberty/testsuite/d-demangle-expected b/libiberty/testsuite/d-demangle-expected
index ffb317f23d4..ba0ffed5c8d 100644
--- a/libiberty/testsuite/d-demangle-expected
+++ b/libiberty/testsuite/d-demangle-expected
@@ -1140,6 +1140,14 @@ _D4test34__T3barVG3uw3_616263VG3wd3_646566Z1xi
 test.bar!("abc"w, "def"d).x
 #
 --format=dlang
+_D4test21__T3funVwi4294967295Z3funFNaNbNiNfZv
+test.fun!('\Uffffffff').fun()
+#
+--format=dlang
+_D4test21__T3funVwi4294967296Z3funFNaNbNiNfZv
+_D4test21__T3funVwi4294967296Z3funFNaNbNiNfZv
+#
+--format=dlang
 _D6plugin8generateFiiZAya
 plugin.generate(int, int)
 #