From 8b4fb4fe140e9cf57fcfa258d0d2d6fe19090fc5 Mon Sep 17 00:00:00 2001
From: Damien George <damien.p.george@gmail.com>
Date: Tue, 15 Sep 2015 16:15:57 +0100
Subject: [PATCH] py/mpz: Fix calculation of max digit storage for mpz; fix
 sys.maxsize.

When creating constant mpz's, the length of the mpz must be exactly how
many digits are used (not allocated) otherwise these numbers are not
compatible with dynamically allocated numbers.

Addresses issue #1448.
---
 py/mpz.h                |  5 +++--
 py/objint_mpz.c         | 29 ++++++++++++++++++++---------
 tests/basics/int_mpz.py |  4 ++++
 3 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/py/mpz.h b/py/mpz.h
index b00d2b6555..e287cdd105 100644
--- a/py/mpz.h
+++ b/py/mpz.h
@@ -78,8 +78,9 @@ typedef int8_t mpz_dbl_dig_signed_t;
   #define MPZ_LONG_1 1L
 #endif
 
-#define MPZ_NUM_DIG_FOR_INT (sizeof(mp_int_t) * 8 / MPZ_DIG_SIZE + 1)
-#define MPZ_NUM_DIG_FOR_LL (sizeof(long long) * 8 / MPZ_DIG_SIZE + 1)
+// these define the maximum storage needed to hold an int or long long
+#define MPZ_NUM_DIG_FOR_INT ((sizeof(mp_int_t) * 8 + MPZ_DIG_SIZE - 1) / MPZ_DIG_SIZE)
+#define MPZ_NUM_DIG_FOR_LL ((sizeof(long long) * 8 + MPZ_DIG_SIZE - 1) / MPZ_DIG_SIZE)
 
 typedef struct _mpz_t {
     mp_uint_t neg : 1;
diff --git a/py/objint_mpz.c b/py/objint_mpz.c
index 27f1ddbfc8..69a81d2c39 100644
--- a/py/objint_mpz.c
+++ b/py/objint_mpz.c
@@ -44,23 +44,34 @@
 #if MICROPY_PY_SYS_MAXSIZE
 // Export value for sys.maxsize
 #define DIG_MASK ((MPZ_LONG_1 << MPZ_DIG_SIZE) - 1)
-STATIC const mpz_dig_t maxsize_dig[MPZ_NUM_DIG_FOR_INT] = {
+STATIC const mpz_dig_t maxsize_dig[] = {
+    #define NUM_DIG 1
     (MP_SSIZE_MAX >> MPZ_DIG_SIZE * 0) & DIG_MASK,
     #if (MP_SSIZE_MAX >> MPZ_DIG_SIZE * 0) > DIG_MASK
-    (MP_SSIZE_MAX >> MPZ_DIG_SIZE * 1) & DIG_MASK,
-    #if (MP_SSIZE_MAX >> MPZ_DIG_SIZE * 1) > DIG_MASK
-    (MP_SSIZE_MAX >> MPZ_DIG_SIZE * 2) & DIG_MASK,
-    (MP_SSIZE_MAX >> MPZ_DIG_SIZE * 3) & DIG_MASK,
-    (MP_SSIZE_MAX >> MPZ_DIG_SIZE * 4) & DIG_MASK,
-//    (MP_SSIZE_MAX >> MPZ_DIG_SIZE * 5) & DIG_MASK,
-    #endif
+     #undef NUM_DIG
+     #define NUM_DIG 2
+     (MP_SSIZE_MAX >> MPZ_DIG_SIZE * 1) & DIG_MASK,
+     #if (MP_SSIZE_MAX >> MPZ_DIG_SIZE * 1) > DIG_MASK
+      #undef NUM_DIG
+      #define NUM_DIG 3
+      (MP_SSIZE_MAX >> MPZ_DIG_SIZE * 2) & DIG_MASK,
+      #if (MP_SSIZE_MAX >> MPZ_DIG_SIZE * 2) > DIG_MASK
+       #undef NUM_DIG
+       #define NUM_DIG 4
+       (MP_SSIZE_MAX >> MPZ_DIG_SIZE * 3) & DIG_MASK,
+       #if (MP_SSIZE_MAX >> MPZ_DIG_SIZE * 3) > DIG_MASK
+        #error cannot encode MP_SSIZE_MAX as mpz
+       #endif
+      #endif
+     #endif
     #endif
 };
 const mp_obj_int_t mp_maxsize_obj = {
     {&mp_type_int},
-    {.fixed_dig = 1, .len = MPZ_NUM_DIG_FOR_INT, .alloc = MPZ_NUM_DIG_FOR_INT, .dig = (mpz_dig_t*)maxsize_dig}
+    {.fixed_dig = 1, .len = NUM_DIG, .alloc = NUM_DIG, .dig = (mpz_dig_t*)maxsize_dig}
 };
 #undef DIG_MASK
+#undef NUM_DIG
 #endif
 
 STATIC mp_obj_int_t *mp_obj_int_new_mpz(void) {
diff --git a/tests/basics/int_mpz.py b/tests/basics/int_mpz.py
index 6d99accf27..8c347302a5 100644
--- a/tests/basics/int_mpz.py
+++ b/tests/basics/int_mpz.py
@@ -77,3 +77,7 @@ x = 4611686018427387903 # small
 x = -4611686018427387903 # small
 x = 4611686018427387904 # big
 x = -4611686018427387904 # big
+
+# sys.maxsize is a constant mpz, so test it's compatible with dynamic ones
+import sys
+print(sys.maxsize + 1 - 1 == sys.maxsize)