cpython: db3e15017172 (original) (raw)

Mercurial > cpython

changeset 72749:db3e15017172

Issue #3163: The struct module gets new format characters 'n' and 'N' supporting C integer types `ssize_t` and `size_t`, respectively. [#3163]

Antoine Pitrou solipsis@pitrou.net
date	Thu, 06 Oct 2011 15:27:40 +0200
parents	542c8da10de9
children	1a0715386d27
files	Doc/library/struct.rst Lib/test/test_struct.py Misc/NEWS Modules/_struct.c
diffstat	4 files changed, 150 insertions(+), 30 deletions(-)[+] [-] Doc/library/struct.rst 21 Lib/test/test_struct.py 66 Misc/NEWS 3 Modules/_struct.c 90

line wrap: on

line diff

--- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -187,17 +187,24 @@ platform-dependent. | Q | :c:type:unsigned long | integer | 8 | \(2), \(3) |[](#l1.4) | | long | | | | +--------+--------------------------+--------------------+----------------+------------+ -| f | :c:type:float | float | 4 | (4) | +| n | :c:type:ssize_t | integer | | (4) | ++--------+--------------------------+--------------------+----------------+------------+ +| N | :c:type:size_t | integer | | (4) | +--------+--------------------------+--------------------+----------------+------------+ -| d | :c:type:double | float | 8 | (4) | +| f | :c:type:float | float | 4 | (5) | ++--------+--------------------------+--------------------+----------------+------------+ +| d | :c:type:double | float | 8 | (5) | +--------+--------------------------+--------------------+----------------+------------+ | s | :c:type:char[] | bytes | | | +--------+--------------------------+--------------------+----------------+------------+ | p | :c:type:char[] | bytes | | | +--------+--------------------------+--------------------+----------------+------------+ -| P | :c:type:void \* | integer | | (5) | +| P | :c:type:void \* | integer | | (6) | +--------+--------------------------+--------------------+----------------+------------+ +.. versionchanged:: 3.3

Added support for the 'n' and 'N' formats. +

Notes: (1) @@ -219,11 +226,17 @@ Notes: Use of the :meth:__index__ method for non-integers is new in 3.2. (4)

The 'n' and 'N' conversion codes are only available for the native
size (selected as the default or with the '@' byte order character).
For the standard size, you can use whichever of the other integer formats
fits your application. +

+(5) For the 'f' and 'd' conversion codes, the packed representation uses the IEEE 754 binary32 (for 'f') or binary64 (for 'd') format, regardless of the floating-point format used by the platform. -(5) +(6) The 'P' format character is only available for the native byte ordering (selected as the default or with the '@' byte order character). The byte order character '=' chooses to use little- or big-endian ordering based

--- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -8,9 +8,19 @@ from test.support import run_unittest ISBIGENDIAN = sys.byteorder == "big" IS32BIT = sys.maxsize == 0x7fffffff -integer_codes = 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q' +integer_codes = 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'n', 'N' byteorders = '', '@', '=', '<', '>', '!' +def iter_integer_formats(byteorders=byteorders):

for code in integer_codes:

   for byteorder in byteorders:[](#l2.13)

       if (byteorder in ('', '@') and code in ('q', 'Q') and[](#l2.14)

           not HAVE_LONG_LONG):[](#l2.15)

```
           continue[](#l2.16)
```

       if (byteorder not in ('', '@') and code in ('n', 'N')):[](#l2.17)

```
           continue[](#l2.18)
```
```
       yield code, byteorder[](#l2.19)
```

Native 'q' packing isn't available on systems that don't have the C

long long type.

try: @@ -141,14 +151,13 @@ class StructTest(unittest.TestCase): } # standard integer sizes

```
   for code in integer_codes:[](#l2.28)
```

       for byteorder in '=', '<', '>', '!':[](#l2.29)

           format = byteorder+code[](#l2.30)

           size = struct.calcsize(format)[](#l2.31)

           self.assertEqual(size, expected_size[code])[](#l2.32)

   for code, byteorder in iter_integer_formats(('=', '<', '>', '!')):[](#l2.33)

       format = byteorder+code[](#l2.34)

       size = struct.calcsize(format)[](#l2.35)

       self.assertEqual(size, expected_size[code])[](#l2.36)

# native integer sizes

   native_pairs = 'bB', 'hH', 'iI', 'lL'[](#l2.39)

   native_pairs = 'bB', 'hH', 'iI', 'lL', 'nN'[](#l2.40)
   if HAVE_LONG_LONG:[](#l2.41)
       native_pairs += 'qQ',[](#l2.42)
   for format_pair in native_pairs:[](#l2.43)

@@ -166,9 +175,11 @@ class StructTest(unittest.TestCase): if HAVE_LONG_LONG: self.assertLessEqual(8, struct.calcsize('q')) self.assertLessEqual(struct.calcsize('l'), struct.calcsize('q'))

   self.assertGreaterEqual(struct.calcsize('n'), struct.calcsize('i'))[](#l2.48)

   self.assertGreaterEqual(struct.calcsize('n'), struct.calcsize('P'))[](#l2.49)

def test_integers(self):

   # Integer tests (bBhHiIlLqQ).[](#l2.52)

   # Integer tests (bBhHiIlLqQnN).[](#l2.53)
   import binascii[](#l2.54)

class IntTester(unittest.TestCase): @@ -182,11 +193,11 @@ class StructTest(unittest.TestCase): self.byteorder) self.bytesize = struct.calcsize(format) self.bitsize = self.bytesize * 8

           if self.code in tuple('bhilq'):[](#l2.61)

           if self.code in tuple('bhilqn'):[](#l2.62)
               self.signed = True[](#l2.63)
               self.min_value = -(2**(self.bitsize-1))[](#l2.64)
               self.max_value = 2**(self.bitsize-1) - 1[](#l2.65)

           elif self.code in tuple('BHILQ'):[](#l2.66)

           elif self.code in tuple('BHILQN'):[](#l2.67)
               self.signed = False[](#l2.68)
               self.min_value = 0[](#l2.69)
               self.max_value = 2**self.bitsize - 1[](#l2.70)

@@ -316,14 +327,23 @@ class StructTest(unittest.TestCase): struct.pack, self.format, obj)

```
   for code in integer_codes:[](#l2.75)
```

       for byteorder in byteorders:[](#l2.76)

           if (byteorder in ('', '@') and code in ('q', 'Q') and[](#l2.77)

               not HAVE_LONG_LONG):[](#l2.78)

```
               continue[](#l2.79)
```

   for code, byteorder in iter_integer_formats():[](#l2.80)

       format = byteorder+code[](#l2.81)

```
       t = IntTester(format)[](#l2.82)
```
```
       t.run()[](#l2.83)
```

def test_nN_code(self):

   # n and N don't exist in standard sizes[](#l2.86)

   def assertStructError(func, *args, **kwargs):[](#l2.87)

       with self.assertRaises(struct.error) as cm:[](#l2.88)

           func(*args, **kwargs)[](#l2.89)

       self.assertIn("bad char in struct format", str(cm.exception))[](#l2.90)

```
   for code in 'nN':[](#l2.91)
```

       for byteorder in ('=', '<', '>', '!'):[](#l2.92)
           format = byteorder+code[](#l2.93)

           t = IntTester(format)[](#l2.94)

```
           t.run()[](#l2.95)
```

           assertStructError(struct.calcsize, format)[](#l2.96)

           assertStructError(struct.pack, format, 0)[](#l2.97)

           assertStructError(struct.unpack, format, b"")[](#l2.98)

def test_p_code(self): # Test p ("Pascal string") code. @@ -377,14 +397,10 @@ class StructTest(unittest.TestCase): self.assertRaises(OverflowError, struct.pack, ">f", big) def test_1530559(self):

   for byteorder in '', '@', '=', '<', '>', '!':[](#l2.106)

       for code in integer_codes:[](#l2.107)

           if (byteorder in ('', '@') and code in ('q', 'Q') and[](#l2.108)

               not HAVE_LONG_LONG):[](#l2.109)

```
               continue[](#l2.110)
```

           format = byteorder + code[](#l2.111)

           self.assertRaises(struct.error, struct.pack, format, 1.0)[](#l2.112)

           self.assertRaises(struct.error, struct.pack, format, 1.5)[](#l2.113)

   for code, byteorder in iter_integer_formats():[](#l2.114)

       format = byteorder + code[](#l2.115)

       self.assertRaises(struct.error, struct.pack, format, 1.0)[](#l2.116)

       self.assertRaises(struct.error, struct.pack, format, 1.5)[](#l2.117)
   self.assertRaises(struct.error, struct.pack, 'P', 1.0)[](#l2.118)
   self.assertRaises(struct.error, struct.pack, 'P', 1.5)[](#l2.119)

--- a/Misc/NEWS +++ b/Misc/NEWS @@ -294,6 +294,9 @@ Core and Builtins Library ------- +- Issue #3163: The struct module gets new format characters 'n' and 'N'

supporting C integer types ssize_t and size_t, respectively. +

Issue #13099: Fix sqlite3.Cursor.lastrowid under a Turkish locale. Reported and diagnosed by Thomas Kluyver.

--- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -58,6 +58,7 @@ typedef struct { char c; long x; } st_lo typedef struct { char c; float x; } st_float; typedef struct { char c; double x; } st_double; typedef struct { char c; void *x; } st_void_p; +typedef struct { char c; size_t x; } st_size_t; #define SHORT_ALIGN (sizeof(st_short) - sizeof(short)) #define INT_ALIGN (sizeof(st_int) - sizeof(int)) @@ -65,6 +66,7 @@ typedef struct { char c; void *x; } st_v #define FLOAT_ALIGN (sizeof(st_float) - sizeof(float)) #define DOUBLE_ALIGN (sizeof(st_double) - sizeof(double)) #define VOID_P_ALIGN (sizeof(st_void_p) - sizeof(void )) +#define SIZE_T_ALIGN (sizeof(st_size_t) - sizeof(size_t)) / We can't support q and Q in native mode unless the compiler does; in std mode, they're 8 bytes on all platforms. */ @@ -213,6 +215,52 @@ get_ulonglong(PyObject v, unsigned PY_L #endif +/ Same, but handling Py_ssize_t */ + +static int +get_ssize_t(PyObject *v, Py_ssize_t *p) +{

Py_ssize_t x;

v = get_pylong(v);
if (v == NULL)
```
   return -1;[](#l4.32)
```
assert(PyLong_Check(v));
x = PyLong_AsSsize_t(v);
Py_DECREF(v);
if (x == (Py_ssize_t)-1 && PyErr_Occurred()) {

   if (PyErr_ExceptionMatches(PyExc_OverflowError))[](#l4.37)

       PyErr_SetString(StructError,[](#l4.38)

                       "argument out of range");[](#l4.39)

```
   return -1;[](#l4.40)
```
}
*p = x;
return 0;

+} + +/* Same, but handling size_t */ + +static int +get_size_t(PyObject *v, size_t *p) +{

size_t x;

v = get_pylong(v);
if (v == NULL)
```
   return -1;[](#l4.55)
```
assert(PyLong_Check(v));
x = PyLong_AsSize_t(v);
Py_DECREF(v);
if (x == (size_t)-1 && PyErr_Occurred()) {

   if (PyErr_ExceptionMatches(PyExc_OverflowError))[](#l4.60)

       PyErr_SetString(StructError,[](#l4.61)

                       "argument out of range");[](#l4.62)

```
   return -1;[](#l4.63)
```
}
*p = x;
return 0;

+} + #define RANGE_ERROR(x, f, flag, mask) return _range_error(f, flag) @@ -369,6 +417,23 @@ nu_ulong(const char *p, const formatdef return PyLong_FromUnsignedLong(x); } +static PyObject * +nu_ssize_t(const char *p, const formatdef *f) +{

Py_ssize_t x;
memcpy((char *)&x, p, sizeof x);
return PyLong_FromSsize_t(x);

+} + +static PyObject * +nu_size_t(const char *p, const formatdef *f) +{

size_t x;
memcpy((char *)&x, p, sizeof x);
return PyLong_FromSize_t(x);

+} + + /* Native mode doesn't support q or Q unless the platform C supports long long (or, on Windows, __int64). */ @@ -558,6 +623,26 @@ np_ulong(char *p, PyObject *v, const for return 0; } +static int +np_ssize_t(char *p, PyObject *v, const formatdef *f) +{

Py_ssize_t x;
if (get_ssize_t(v, &x) < 0)
```
   return -1;[](#l4.105)
```
memcpy(p, (char *)&x, sizeof x);
return 0;

+} + +static int +np_size_t(char *p, PyObject *v, const formatdef *f) +{

size_t x;
if (get_size_t(v, &x) < 0)
```
   return -1;[](#l4.115)
```
memcpy(p, (char *)&x, sizeof x);
return 0;

+} + #ifdef HAVE_LONG_LONG static int @@ -651,6 +736,8 @@ static formatdef native_table[] = { {'I', sizeof(int), INT_ALIGN, nu_uint, np_uint}, {'l', sizeof(long), LONG_ALIGN, nu_long, np_long}, {'L', sizeof(long), LONG_ALIGN, nu_ulong, np_ulong},

{'n', sizeof(size_t), SIZE_T_ALIGN, nu_ssize_t, np_ssize_t},
{'N', sizeof(size_t), SIZE_T_ALIGN, nu_size_t, np_size_t},

#ifdef HAVE_LONG_LONG {'q', sizeof(PY_LONG_LONG), LONG_LONG_ALIGN, nu_longlong, np_longlong}, {'Q', sizeof(PY_LONG_LONG), LONG_LONG_ALIGN, nu_ulonglong,np_ulonglong}, @@ -1951,7 +2038,8 @@ these can be preceded by a decimal repea l:long; L:unsigned long; f:float; d:double.\n[](#l4.133) Special cases (preceding decimal count indicates length):\n[](#l4.134) s:string (array of char); p: pascal string (with count byte).\n[](#l4.135) -Special case (only available in native format):\n[](#l4.136) +Special cases (only available in native format):\n[](#l4.137)

n:ssize_t; N:size_t;\n[](#l4.138) P:an integer type that is wide enough to hold a pointer.\n[](#l4.139) Special case (not in native mode unless 'long long' in platform C):\n[](#l4.140) q:long long; Q:unsigned long long\n[](#l4.141)