Use ValueError, ignore underscore, scale limit

python · gpshead · Sep 2, 2022 · May 5, 2020 · Jan 19, 2022 · Jan 19, 2022
commit fbd14b753d2a27d51a89322c4f2aaf5ad24a0102
diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
@@ -911,8 +911,10 @@ are always available.  They are listed here in alphabetical order.
       The delegation to :meth:`__trunc__` is deprecated.
 
    .. versionchanged:: 3.12
-      :class:`int` string inputs are now limited, see :ref:`int maximum
-      digits limitation <intmaxdigits>`.
+      :class:`int` string inputs and string representation can be limited.
+      A :exc:`ValueError` is raised when the input or string representation
+      exceeds the limit. See :ref:`int maximum
+      digits limitation <intmaxdigits>` for more information.
 
 .. function:: isinstance(object, classinfo)
 

diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
@@ -5466,16 +5466,31 @@ Integer maximum digits limitation
 =================================
 
 CPython has a global limit for converting between :class:`int` and class:`str`
-to mitigate denial of service attacks. The limit is necessary because there
-exists no efficient algorithm, that can convert a string to an integer or
+to mitigate denial of service attacks. The limit is necessary because Python's
+integer type is an abitrary length number (also known as bignum). There
+exists no efficient algorithm that can convert a string to an integer or
 an integer to a string in linear time, unless the base is a power of *2*. Even
 the best known algorithms for base *10* have sub-quadratic complexity. A large
-input like::
-
-   int('1' * 500_000)
+input like ``int('1' * 500_000)`` takes about a second at 100% CPU load on
+an X86_64 CPU from 2020 with 4.2 GHz max frequency.
+
+The limit value uses base 10 as a reference point and scales with base.
+That means :class:`int` accepts longer input strings for smaller bases and
+fails earlier for larger bases. Underscores in input strings don't count
+towards the limit. 
+
+When an operation exceeds the limit, an :exc:`ValueError` is raised::
+
+   >>> sys.setintmaxdigits(2048)
+   >>> i = 10 ** 2047
+   >>> len(str(i))
+   2048
+   >>> i = 10 ** 2048
+   >>> len(str(i))
+   Traceback (most recent call last):
+   ...
+   ValueError: input exceeds maximum integer digit limit
 
-takes about a second at 100% CPU load on an X86_64 CPU from 2020 with 4.2 GHz
-max frequency.
 
 Configure limitations
 ---------------------
@@ -5494,13 +5509,8 @@ Configure limitations
   precedence. The flag defaults to *-1*.
 
 * :func:`sys.getintmaxdigits` and :func:`sys.setintmaxdigits` are getter
-  and setter for interpreter-wide limit.
-
-Recommended configuration::
-
-   import sys
-   if hasattr(sys.flags, "intmaxdigits") and sys.flags.intmaxdigits == -1:
-       sys.setintmaxdigits(4096)
+  and setter for interpreter-wide limit. Subinterpreters have their own
+  limit.
 
 Affected APIs
 -------------
@@ -5521,10 +5531,17 @@ The limitations do not apply to functions with a linear algorithm:
 * :func:`int.from_bytes` and :func:`int.to_bytes`
 * :func:`hex`, :func:`oct`, :func:`bin` (the resulting string may consume
   a substantial amount of memory)
-* :ref:`formatspec` for hex, octet, and binary types
+* :ref:`formatspec` for hex, octet, and binary numbers
 * :class:`str` to :class:`float`
 * :class:`str` to :class:`decimal.Decimal`
 
+Recommended configuration
+-------------------------
+
+   import sys
+   if hasattr(sys.flags, "intmaxdigits") and sys.flags.intmaxdigits == -1:
+       sys.setintmaxdigits(4096)
+
 
 .. rubric:: Footnotes
 

diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py
@@ -591,24 +591,40 @@ def _test_maxdigits(self, c):
             i = c('1' * 100_000)
             str(i)
 
-        # OverflowError
         def check(i, base=None):
-            with self.assertRaises(OverflowError):
+            with self.assertRaises(ValueError):
                 if base is None:
                     c(i)
                 else:
                     c(i, base)
 
-        maxdigits = 1024
+        maxdigits = 2048
         with support.setintmaxdigits(maxdigits):
+            assert maxdigits == sys.getintmaxdigits()
             check('1' * (maxdigits + 1))
             check('+' + '1' * (maxdigits + 1))
             check('1' * (maxdigits + 1))
 
             i = 10 ** maxdigits
-            with self.assertRaises(OverflowError):
+            with self.assertRaises(ValueError):
                 str(i)
 
+            # ignore power of two
+            for base in (2, 4, 8, 16, 32):
+                c('1' * (maxdigits + 1), base)
+                c('1' * 100_000, base)
+
+            # limit ignores underscores
+            s = '1111_' * ((maxdigits) // 4)
+            s = s[:-1]
+            int(s)
+            check(s + '1')
+
+            # limit is in equivalent of base 10 digits
+            s = '1' * 2147
+            assert len(str(int(s, 9))) == maxdigits
+            int(s + '1', 9)
+
     def test_maxdigits(self):
         self._test_maxdigits(int)
         self._test_maxdigits(IntSubclass)

diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py
@@ -101,7 +101,7 @@ def test_limit_int(self):
         maxdigits = 5000
         with support.setintmaxdigits(maxdigits):
             self.loads('1' * maxdigits)
-            with self.assertRaises(OverflowError):
+            with self.assertRaises(ValueError):
                 self.loads('1' * (maxdigits + 1))
 
 

diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py
@@ -293,7 +293,7 @@ def test_limit_int(self):
         check = self.check_loads
         with self.assertRaises(OverflowError):
             check('<int>123456780123456789</int>', None)
-        with self.assertRaises(OverflowError):
+        with self.assertRaises(ValueError):
             maxdigits = 5000
             with support.setintmaxdigits(maxdigits):
                 s = '1' * (maxdigits + 1)

diff --git a/Objects/longobject.c b/Objects/longobject.c
@@ -1819,8 +1819,8 @@ long_to_decimal_string_internal(PyObject *aa,
         PyInterpreterState *interp = _PyInterpreterState_GET();
         if ((interp->intmaxdigits > 0) && (strlen > interp->intmaxdigits)) {
             Py_DECREF(scratch);
-            PyErr_SetString(PyExc_OverflowError,
-                           "too many digits in integer");
+            PyErr_SetString(PyExc_ValueError,
+                           "input exceeds maximum integer digit limit");
             return -1;
         }
     }
@@ -2434,6 +2434,7 @@ digit beyond the first.
         twodigits c;           /* current input character */
         Py_ssize_t size_z;
         Py_ssize_t digits = 0;
+        Py_ssize_t underscores = 0;
         int i;
         int convwidth;
         twodigits convmultmax, convmult;
@@ -2470,6 +2471,7 @@ digit beyond the first.
 
         while (_PyLong_DigitValue[Py_CHARMASK(*scan)] < base || *scan == '_') {
             if (*scan == '_') {
+                ++underscores;
                 if (prev == '_') {
                     /* Only one underscore allowed. */
                     str = lastdigit + 1;
@@ -2490,12 +2492,24 @@ digit beyond the first.
             goto onError;
         }
 
-        slen = scan - str;
+        /* intmaxdigits limit ignores underscores and uses base 10
+         * as reference point.
+         * For other bases slen is transformed into base 10 equivalents.
+         * Our string to integer conversion algorithm scales less than
+         * linear with base value, for example int('1' * 300_000", 30)
+         * is slightly more than five times slower than int(..., 5).
+         * The naive scaling "slen / 10 * base" is close enough to
+         * compensate.
+         */
+        slen = scan - str - underscores;
+        if (base != 10) {
+            slen = (Py_ssize_t)(slen / 10 * base);
+        }
         if (slen > _PY_LONG_MAX_DIGITS_THRESHOLD) {
             PyInterpreterState *interp = _PyInterpreterState_GET();
             if ((interp->intmaxdigits > 0 ) && (slen > interp->intmaxdigits)) {
-                PyErr_SetString(PyExc_OverflowError,
-                               "too many digits in integer");
+                PyErr_SetString(PyExc_ValueError,
+                               "input exceeds maximum integer digit limit");
                 return NULL;
             }
         }