[issue4258] Use 30-bit digits instead of 15-bit digits for Python integers. - Code Review (original) (raw)

OLD

NEW

2 /* Write Python objects to files and read them back.

3 This is intended for writing and reading compiled Python code only;

4 a true persistent storage facility would be much harder, since

5 it would have to take circular links and sharing into account. */

7 #define PY_SSIZE_T_CLEAN

9 #include "Python.h"

10 #include "longintrepr.h"

11 #include "code.h"

12 #include "marshal.h"

14 #define ABS(x) ((x) < 0 ? -(x) : (x))

14 /* High water mark to determine when the marshalled object is dangerously deep

16 /* High water mark to determine when the marshalled object is dangerously deep

15 * and risks coring the interpreter. When the object stack gets this deep,

17 * and risks coring the interpreter. When the object stack gets this deep,

16 * raise an exception instead of continuing.

18 * raise an exception instead of continuing.

17 * On Windows debug builds, reduce this value.

19 * On Windows debug builds, reduce this value.

18 */

20 */

19 #if defined(MS_WINDOWS) && defined(_DEBUG)

21 #if defined(MS_WINDOWS) && defined(_DEBUG)

20 #define MAX_MARSHAL_STACK_DEPTH 1500

22 #define MAX_MARSHAL_STACK_DEPTH 1500

21 #else

23 #else

22 #define MAX_MARSHAL_STACK_DEPTH 2000

24 #define MAX_MARSHAL_STACK_DEPTH 2000

23 #endif

25 #endif

(...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...

115

117

116 #if SIZEOF_LONG > 4

118 #if SIZEOF_LONG > 4

117 static void

119 static void

118 w_long64(long x, WFILE *p)

120 w_long64(long x, WFILE *p)

119 {

121 {

120 w_long(x, p);

122 w_long(x, p);

121 w_long(x>>32, p);

123 w_long(x>>32, p);

122 }

124 }

123 #endif

125 #endif

124

126

127 /* We assume that Python longs are stored internally in base some power of

128 2**15; for the sake of portability we'll always read and write them in base

129 exactly 2**15. */

130

131 #define PyLong_MARSHAL_SHIFT 15

132 #define PyLong_MARSHAL_BASE ((short)1 << PyLong_MARSHAL_SHIFT)

133 #define PyLong_MARSHAL_MASK (PyLong_MARSHAL_BASE - 1)

134 #if PyLong_SHIFT % PyLong_MARSHAL_SHIFT != 0

135 #error "PyLong_SHIFT must be a multiple of PyLong_MARSHAL_SHIFT"

136 #endif

137 #define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT)

138

139 static void

140 w_PyLong(const PyLongObject *ob, WFILE *p)

141 {

142 Py_ssize_t i, j, n, l;

143 digit d;

144

145 w_byte(TYPE_LONG, p);

146 if (Py_SIZE(ob) == 0) {

147 w_long((long)0, p);

148 return;

149 }

150

151 /* set l to number of base PyLong_MARSHAL_BASE digits */

152 n = ABS(Py_SIZE(ob));

153 l = (n-1) * PyLong_MARSHAL_RATIO;

154 d = ob->ob_digit[n-1];

155 assert(d != 0); /* a PyLong is always normalized */

156 do {

157 d >>= PyLong_MARSHAL_SHIFT;

158 l++;

159 } while (d != 0);

160 w_long((long)(Py_SIZE(ob) > 0 ? l : -l), p);

161

162 for (i=0; i < n-1; i++) {

163 d = ob->ob_digit[i];

164 for (j=0; j < PyLong_MARSHAL_RATIO; j++) {

165 w_short(d & PyLong_MARSHAL_MASK, p);

166 d >>= PyLong_MARSHAL_SHIFT;

167 }

168 assert (d == 0);

169 }

170 d = ob->ob_digit[n-1];

171 do {

172 w_short(d & PyLong_MARSHAL_MASK, p);

173 d >>= PyLong_MARSHAL_SHIFT;

174 } while (d != 0);

175 }

176

125 static void

177 static void

126 w_object(PyObject *v, WFILE *p)

178 w_object(PyObject *v, WFILE *p)

127 {

179 {

128 Py_ssize_t i, n;

180 Py_ssize_t i, n;

129

181

130 p->depth++;

182 p->depth++;

131

183

132 if (p->depth > MAX_MARSHAL_STACK_DEPTH) {

184 if (p->depth > MAX_MARSHAL_STACK_DEPTH) {

133 p->error = 2;

185 p->error = 2;

134 }

186 }

(...skipping 13 matching lines...) Expand all Loading...

148 w_byte(TYPE_FALSE, p);

200 w_byte(TYPE_FALSE, p);

149 }

201 }

150 else if (v == Py_True) {

202 else if (v == Py_True) {

151 w_byte(TYPE_TRUE, p);

203 w_byte(TYPE_TRUE, p);

152 }

204 }

153 else if (PyLong_CheckExact(v)) {

205 else if (PyLong_CheckExact(v)) {

154 long x = PyLong_AsLong(v);

206 long x = PyLong_AsLong(v);

155 if ((x == -1) && PyErr_Occurred()) {

207 if ((x == -1) && PyErr_Occurred()) {

156 PyLongObject *ob = (PyLongObject *)v;

208 PyLongObject *ob = (PyLongObject *)v;

157 PyErr_Clear();

209 PyErr_Clear();

158 » » » w_byte(TYPE_LONG, p);

210 » » » w_PyLong(ob, p);

159 » » » n = Py_SIZE(ob);

211 » » }

160 » » » w_long((long)n, p);

161 » » » if (n < 0)

162 » » » » n = -n;

163 » » » for (i = 0; i < n; i++)

164 » » » » w_short(ob->ob_digit[i], p);

165 » » }·

166 else {

212 else {

167 #if SIZEOF_LONG > 4

213 #if SIZEOF_LONG > 4

168 long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);

214 long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);

169 if (y && y != -1) {

215 if (y && y != -1) {

170 w_byte(TYPE_INT64, p);

216 w_byte(TYPE_INT64, p);

171 w_long64(x, p);

217 w_long64(x, p);

172 }

218 }

173 else

219 else

174 #endif

220 #endif

175 {

221 {

(...skipping 298 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...

474 }

520 }

475 else {

521 else {

476 memcpy(buf, &hi4, 4);

522 memcpy(buf, &hi4, 4);

477 memcpy(buf+4, &lo4, 4);

523 memcpy(buf+4, &lo4, 4);

478 }

524 }

479 return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);

525 return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);

480 #endif

526 #endif

481 }

527 }

482

528

483 static PyObject *

529 static PyObject *

530 r_PyLong(RFILE *p)

531 {

532 PyLongObject *ob;

533 int size, i, j, md;

534 long n;

535 digit d;

536

537 n = r_long(p);

538 if (n == 0)

539 return (PyObject *)_PyLong_New(0);

540 if (n < -INT_MAX || n > INT_MAX)

541 goto bad_data;

542

543 size = 1 + (ABS(n)-1) / PyLong_MARSHAL_RATIO;

544 ob = _PyLong_New(size);

545 if (ob == NULL)

546 return NULL;

547 Py_SIZE(ob) = n > 0 ? size : -size;

548

549 for (i = 0; i < size-1; i++) {

550 d = 0;

551 for (j=0; j < PyLong_MARSHAL_RATIO; j++) {

552 md = r_short(p);

553 if (md < 0 || md > PyLong_MARSHAL_BASE) {

554 Py_DECREF(ob);

555 goto bad_data;

556 }

557 d += (digit)md << j*PyLong_MARSHAL_SHIFT;

558 }

559 ob->ob_digit[i] = d;

560 }

561 d = 0;

562 for (j=0; j < (ABS(n)-1)%PyLong_MARSHAL_RATIO + 1; j++) {

563 md = r_short(p);

564 if (md < 0 || md > PyLong_MARSHAL_BASE) {

565 Py_DECREF(ob);

566 goto bad_data;

567 }

568 d += (digit)md << (j*PyLong_MARSHAL_SHIFT);

569 }

570 ob->ob_digit[size-1] = d;

571 return (PyObject *)ob;

572 bad_data:

573 PyErr_SetString(PyExc_ValueError, "bad marshal data");

574 return NULL;

575 }

576

577

578 static PyObject *

484 r_object(RFILE *p)

579 r_object(RFILE *p)

485 {

580 {

486 /* NULL is a valid return value, it does not necessarily means that

581 /* NULL is a valid return value, it does not necessarily means that

487 an exception is set. */

582 an exception is set. */

488 PyObject *v, *v2;

583 PyObject *v, *v2;

489 long i, n;

584 long i, n;

490 int type = r_byte(p);

585 int type = r_byte(p);

491 PyObject *retval;

586 PyObject *retval;

492

587

493 p->depth++;

588 p->depth++;

(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...

537

632

538 case TYPE_INT:

633 case TYPE_INT:

539 retval = PyLong_FromLong(r_long(p));

634 retval = PyLong_FromLong(r_long(p));

540 break;

635 break;

541

636

542 case TYPE_INT64:

637 case TYPE_INT64:

543 retval = r_long64(p);

638 retval = r_long64(p);

544 break;

639 break;

545

640

546 case TYPE_LONG:

641 case TYPE_LONG:

547 » » {

642 » » retval = r_PyLong(p);

548 » » » int size;

643 » » break;

549 » » » PyLongObject *ob;

550 » » » n = r_long(p);

551 » » » if (n < -INT_MAX || n > INT_MAX) {

552 » » » » PyErr_SetString(PyExc_ValueError,

553 » » » » » » "bad marshal data");

554 » » » » retval = NULL;

555 » » » » break;

556 » » » }

557 » » » size = n<0 ? -n : n;

558 » » » ob = _PyLong_New(size);

559 » » » if (ob == NULL) {

560 » » » » retval = NULL;

561 » » » » break;

562 » » » }

563 » » » Py_SIZE(ob) = n;

564 » » » for (i = 0; i < size; i++) {

565 » » » » int digit = r_short(p);

566 » » » » if (digit < 0) {

567 » » » » » Py_DECREF(ob);

568 » » » » » PyErr_SetString(PyExc_ValueError,

569 » » » » » » » "bad marshal data");

570 » » » » » ob = NULL;

571 » » » » » break;

572 » » » » }

573 » » » » if (ob != NULL)

574 » » » » » ob->ob_digit[i] = digit;

575 » » » }

576 » » » retval = (PyObject *)ob;

577 » » » break;

578 » » }

579

644

580 case TYPE_FLOAT:

645 case TYPE_FLOAT:

581 {

646 {

582 char buf[256];

647 char buf[256];

583 double dx;

648 double dx;

584 n = r_byte(p);

649 n = r_byte(p);

585 if (n == EOF || r_string(buf, (int)n, p) != n) {

650 if (n == EOF || r_string(buf, (int)n, p) != n) {

586 PyErr_SetString(PyExc_EOFError,

651 PyErr_SetString(PyExc_EOFError,

587 "EOF read where object expected");

652 "EOF read where object expected");

588 retval = NULL;

653 retval = NULL;

(...skipping 622 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...

1211

1276

1212 PyMODINIT_FUNC

1277 PyMODINIT_FUNC

1213 PyMarshal_Init(void)

1278 PyMarshal_Init(void)

1214 {

1279 {

1215 PyObject *mod = PyModule_Create(&marshalmodule);

1280 PyObject *mod = PyModule_Create(&marshalmodule);

1216 if (mod == NULL)

1281 if (mod == NULL)

1217 return NULL;

1282 return NULL;

1218 PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);

1283 PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);

1219 return mod;

1284 return mod;

1220 }

1285 }

OLD

NEW