cpython: d2127cdec10e (original) (raw)

Mercurial > cpython

changeset 79803:d2127cdec10e

Issue #16166: Add PY_LITTLE_ENDIAN and PY_BIG_ENDIAN macros and unified endianess detection and handling. [#16166]

Christian Heimes christian@cheimes.de
date	Wed, 17 Oct 2012 23:52:17 +0200
parents	943d3e289ab4
children	f6fcff683866
files	Include/pyport.h Misc/NEWS Modules/_io/textio.c Modules/_sha3/cleanup.py Modules/_sha3/keccak/KeccakF-1600-opt32.c Modules/_sha3/keccak/KeccakF-1600-opt64.c Modules/_sha3/sha3module.c Modules/_struct.c Modules/arraymodule.c Modules/sha256module.c Modules/sha512module.c Objects/longobject.c Objects/stringlib/codecs.h Objects/unicodeobject.c Python/sysmodule.c
diffstat	15 files changed, 78 insertions(+), 107 deletions(-)[+] [-] Include/pyport.h 14 Misc/NEWS 3 Modules/_io/textio.c 18 Modules/_sha3/cleanup.py 8 Modules/_sha3/keccak/KeccakF-1600-opt32.c 2 Modules/_sha3/keccak/KeccakF-1600-opt64.c 2 Modules/_sha3/sha3module.c 13 Modules/_struct.c 21 Modules/arraymodule.c 7 Modules/sha256module.c 22 Modules/sha512module.c 22 Objects/longobject.c 8 Objects/stringlib/codecs.h 6 Objects/unicodeobject.c 18 Python/sysmodule.c 21

line wrap: on

line diff

--- a/Include/pyport.h +++ b/Include/pyport.h @@ -865,4 +865,18 @@ extern pid_t forkpty(int *, char , stru #endif #endif +/

- Convenient macros to deal with endianess of the platform. WORDS_BIGENDIAN is
- detected by configure and defined in pyconfig.h. The code in pyconfig.h
- also also takes care of Apple's universal builds.
*/ + +#ifdef WORDS_BIGENDIAN +#define PY_BIG_ENDIAN 1 +#define PY_LITTLE_ENDIAN 0 +#else +#define PY_BIG_ENDIAN 0 +#define PY_LITTLE_ENDIAN 1 +#endif +

#endif /* Py_PYPORT_H */

--- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.4.0 Alpha 1? Core and Builtins ----------------- +- Issue #16166: Add PY_LITTLE_ENDIAN and PY_BIG_ENDIAN macros and unified

endianess detection and handling. +

Issue #15958: bytes.join and bytearray.join now accept arbitrary buffer objects.

--- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -747,7 +747,7 @@ utf16_encode(textio *self, PyObject tex { if (!self->encoding_start_of_stream) { / Skip the BOM and use native byte ordering */ -#if defined(WORDS_BIGENDIAN) +#if PY_BIG_ENDIAN return utf16be_encode(self, text); #else return utf16le_encode(self, text); @@ -776,7 +776,7 @@ utf32_encode(textio *self, PyObject tex { if (!self->encoding_start_of_stream) { / Skip the BOM and use native byte ordering / -#if defined(WORDS_BIGENDIAN) +#if PY_BIG_ENDIAN return utf32be_encode(self, text); #else return utf32le_encode(self, text); @@ -1913,10 +1913,7 @@ typedef struct { #define COOKIE_BUF_LEN (sizeof(Py_off_t) + 3 * sizeof(int) + sizeof(char)) -#if defined(WORDS_BIGENDIAN) - -# define IS_LITTLE_ENDIAN 0 - +#if PY_BIG_ENDIAN / We want the least significant byte of start_pos to also be the least significant byte of the cookie, which means that in big-endian mode we must copy the fields in reverse order. */ @@ -1928,9 +1925,6 @@ typedef struct {

define OFF_NEED_EOF 0

#else - -# define IS_LITTLE_ENDIAN 1 - /* Little-endian mode: the least significant byte of start_pos will naturally end up the least significant byte of the cookie. */ @@ -1951,7 +1945,7 @@ textiowrapper_parse_cookie(cookie_type * return -1; if (_PyLong_AsByteArray(cookieLong, buffer, sizeof(buffer),

                       IS_LITTLE_ENDIAN, 0) < 0) {[](#l3.47)

```
                       PY_LITTLE_ENDIAN, 0) < 0) {[](#l3.48)
   Py_DECREF(cookieLong);[](#l3.49)
   return -1;[](#l3.50)
```
} @@ -1977,9 +1971,9 @@ textiowrapper_build_cookie(cookie_type * memcpy(buffer + OFF_CHARS_TO_SKIP, &cookie->chars_to_skip, sizeof(cookie->chars_to_skip)); memcpy(buffer + OFF_NEED_EOF, &cookie->need_eof, sizeof(cookie->need_eof));

return _PyLong_FromByteArray(buffer, sizeof(buffer), IS_LITTLE_ENDIAN, 0);

return _PyLong_FromByteArray(buffer, sizeof(buffer),

                            PY_LITTLE_ENDIAN, 0);[](#l3.58)

} -#undef IS_LITTLE_ENDIAN static int _textiowrapper_decoder_setstate(textio *self, cookie_type *cookie)

--- a/Modules/_sha3/cleanup.py +++ b/Modules/_sha3/cleanup.py @@ -32,10 +32,10 @@ def cleanup(f): if line.startswith("typedef unsigned long long int"): buf.append("/* %s */\n" % line.strip()) continue

   ## remove #include "brg_endian.h"[](#l4.7)

```
   #if "brg_endian.h" in line:[](#l4.8)
```

   #    buf.append("/* %s */\n" % line.strip())[](#l4.9)

```
   #    continue[](#l4.10)
```

   # remove #include "brg_endian.h"[](#l4.11)

```
   if "brg_endian.h" in line:[](#l4.12)
```

       buf.append("/* %s */\n" % line.strip())[](#l4.13)

       continue[](#l4.14)
   # transform C++ comments into ANSI C comments[](#l4.15)
   line = CPP1.sub(r"/* \1 */", line)[](#l4.16)
   line = CPP2.sub(r" /* \1 */", line)[](#l4.17)

--- a/Modules/_sha3/keccak/KeccakF-1600-opt32.c +++ b/Modules/_sha3/keccak/KeccakF-1600-opt32.c @@ -12,7 +12,7 @@ http://creativecommons.org/publicdomain/[](#l5.3) / #include <string.h> -#include "brg_endian.h" +/ #include "brg_endian.h" */ #include "KeccakF-1600-opt32-settings.h" #include "KeccakF-1600-interface.h"

--- a/Modules/_sha3/keccak/KeccakF-1600-opt64.c +++ b/Modules/_sha3/keccak/KeccakF-1600-opt64.c @@ -12,7 +12,7 @@ http://creativecommons.org/publicdomain/[](#l6.3) / #include <string.h> -#include "brg_endian.h" +/ #include "brg_endian.h" */ #include "KeccakF-1600-opt64-settings.h" #include "KeccakF-1600-interface.h"

--- a/Modules/_sha3/sha3module.c +++ b/Modules/_sha3/sha3module.c @@ -124,11 +124,14 @@ #define UseInterleaveTables #endif -/* replacement for brg_endian.h -#define IS_BIG_ENDIAN BIG_ENDIAN -#define IS_LITTLE_ENDIAN LITTLE_ENDIAN -#define PLATFORM_BYTE_ORDER BYTE_ORDER -/ +/ replacement for brg_endian.h / +#define IS_BIG_ENDIAN 4321 +#define IS_LITTLE_ENDIAN 1234 +#if PY_BIG_ENDIAN +# define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN +#else +# define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN +#endif / inline all Keccak dependencies */ #include "keccak/KeccakNISTInterface.h"

--- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -1199,12 +1199,11 @@ whichtable(char *pfmt) case '!': / Network byte order is big-endian / return bigendian_table; case '=': { / Host byte order -- different from native in alignment! */

```
   int n = 1;[](#l8.7)
```
```
   char *p = (char *) &n;[](#l8.8)
```
```
   if (*p == 1)[](#l8.9)
```

       return lilendian_table;[](#l8.10)

```
   else[](#l8.11)
```

       return bigendian_table;[](#l8.12)

+#if PY_LITTLE_ENDIAN

```
   return lilendian_table;[](#l8.14)
```

+#else

```
   return bigendian_table;[](#l8.16)
```

+#endif } default: --pfmt; / Back out of pointer increment / @@ -2088,13 +2087,13 @@ PyInit__struct(void) / Check endian and swap in faster functions */ {

   int one = 1;[](#l8.25)
   formatdef *native = native_table;[](#l8.26)
   formatdef *other, *ptr;[](#l8.27)

   if ((int)*(unsigned char*)&one)[](#l8.28)

       other = lilendian_table;[](#l8.29)

```
   else[](#l8.30)
```

       other = bigendian_table;[](#l8.31)

+#if PY_LITTLE_ENDIAN

```
   other = lilendian_table;[](#l8.33)
```

+#else

```
   other = bigendian_table;[](#l8.35)
```

+#endif /* Scan through the native table, find a matching entry in the endian table and swap in the native implementations whenever possible

--- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -1664,11 +1664,8 @@ static const struct mformatdescr { static enum machine_format_code typecode_to_mformat_code(char typecode) { -#ifdef WORDS_BIGENDIAN

const int is_big_endian = 1;

-#else

const int is_big_endian = 0;

-#endif

const int is_big_endian = PY_BIG_ENDIAN;

+ size_t intsize; int is_signed;

--- a/Modules/sha256module.c +++ b/Modules/sha256module.c @@ -21,13 +21,6 @@ #include "hashlib.h" -/* Endianness testing and definitions */ -#define TestEndianness(variable) {int i=1; variable=PCT_BIG_ENDIAN;[](#l10.8)

   if (*((char*)&i)==1) variable=PCT_LITTLE_ENDIAN;}[](#l10.9)

- -#define PCT_LITTLE_ENDIAN 1 -#define PCT_BIG_ENDIAN 0 - /* Some useful types / typedef unsigned char SHA_BYTE; @@ -50,7 +43,6 @@ typedef struct { SHA_INT32 digest[8]; / Message digest / SHA_INT32 count_lo, count_hi; / 64-bit bit count / SHA_BYTE data[SHA_BLOCKSIZE]; / SHA data buffer */

int Endianness; int local; /* unprocessed amount in data / int digestsize; } SHAobject; @@ -58,13 +50,11 @@ typedef struct { / When run on a little-endian CPU we need to perform byte reversal on an array of longwords. */ -static void longReverse(SHA_INT32 *buffer, int byteCount, int Endianness) +#if PY_LITTLE_ENDIAN +static void longReverse(SHA_INT32 *buffer, int byteCount) { SHA_INT32 value;
if ( Endianness == PCT_BIG_ENDIAN )
```
   return;[](#l10.36)
```

- byteCount /= sizeof(*buffer); while (byteCount--) { value = *buffer; @@ -73,10 +63,10 @@ static void longReverse(SHA_INT32 *buffe *buffer++ = ( value << 16 ) | ( value >> 16 ); } } +#endif static void SHAcopy(SHAobject *src, SHAobject *dest) {

dest->Endianness = src->Endianness; dest->local = src->local; dest->digestsize = src->digestsize; dest->count_lo = src->count_lo; @@ -131,7 +121,9 @@ sha_transform(SHAobject *sha_info) SHA_INT32 S[8], W[64], t0, t1;

memcpy(W, sha_info->data, sizeof(sha_info->data));

longReverse(W, (int)sizeof(sha_info->data), sha_info->Endianness);

+#if PY_LITTLE_ENDIAN

longReverse(W, (int)sizeof(sha_info->data));

+#endif for (i = 16; i < 64; ++i) { W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]; @@ -228,7 +220,6 @@ sha_transform(SHAobject *sha_info) static void sha_init(SHAobject *sha_info) {

TestEndianness(sha_info->Endianness) sha_info->digest[0] = 0x6A09E667L; sha_info->digest[1] = 0xBB67AE85L; sha_info->digest[2] = 0x3C6EF372L; @@ -246,7 +237,6 @@ sha_init(SHAobject *sha_info) static void sha224_init(SHAobject *sha_info) {
TestEndianness(sha_info->Endianness) sha_info->digest[0] = 0xc1059ed8L; sha_info->digest[1] = 0x367cd507L; sha_info->digest[2] = 0x3070dd17L;

--- a/Modules/sha512module.c +++ b/Modules/sha512module.c @@ -22,13 +22,6 @@ #ifdef PY_LONG_LONG /* If no PY_LONG_LONG, don't compile anything! / -/ Endianness testing and definitions */ -#define TestEndianness(variable) {int i=1; variable=PCT_BIG_ENDIAN;[](#l11.8)

   if (*((char*)&i)==1) variable=PCT_LITTLE_ENDIAN;}[](#l11.9)

- -#define PCT_LITTLE_ENDIAN 1 -#define PCT_BIG_ENDIAN 0 - /* Some useful types / typedef unsigned char SHA_BYTE; @@ -52,7 +45,6 @@ typedef struct { SHA_INT64 digest[8]; / Message digest / SHA_INT32 count_lo, count_hi; / 64-bit bit count / SHA_BYTE data[SHA_BLOCKSIZE]; / SHA data buffer */

int Endianness; int local; /* unprocessed amount in data / int digestsize; } SHAobject; @@ -60,13 +52,11 @@ typedef struct { / When run on a little-endian CPU we need to perform byte reversal on an array of longwords. */ -static void longReverse(SHA_INT64 *buffer, int byteCount, int Endianness) +#if PY_LITTLE_ENDIAN +static void longReverse(SHA_INT64 *buffer, int byteCount) { SHA_INT64 value;
if ( Endianness == PCT_BIG_ENDIAN )
```
   return;[](#l11.36)
```

- byteCount /= sizeof(*buffer); while (byteCount--) { value = *buffer; @@ -83,10 +73,10 @@ static void longReverse(SHA_INT64 *buffe buffer++; } } +#endif static void SHAcopy(SHAobject *src, SHAobject *dest) {

dest->Endianness = src->Endianness; dest->local = src->local; dest->digestsize = src->digestsize; dest->count_lo = src->count_lo; @@ -141,7 +131,9 @@ sha512_transform(SHAobject *sha_info) SHA_INT64 S[8], W[80], t0, t1; memcpy(W, sha_info->data, sizeof(sha_info->data));
longReverse(W, (int)sizeof(sha_info->data), sha_info->Endianness);

+#if PY_LITTLE_ENDIAN

longReverse(W, (int)sizeof(sha_info->data));

+#endif for (i = 16; i < 80; ++i) { W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]; @@ -254,7 +246,6 @@ sha512_transform(SHAobject *sha_info) static void sha512_init(SHAobject *sha_info) {

TestEndianness(sha_info->Endianness) sha_info->digest[0] = Py_ULL(0x6a09e667f3bcc908); sha_info->digest[1] = Py_ULL(0xbb67ae8584caa73b); sha_info->digest[2] = Py_ULL(0x3c6ef372fe94f82b); @@ -272,7 +263,6 @@ sha512_init(SHAobject *sha_info) static void sha384_init(SHAobject *sha_info) {
TestEndianness(sha_info->Endianness) sha_info->digest[0] = Py_ULL(0xcbbb9d5dc1059ed8); sha_info->digest[1] = Py_ULL(0x629a292a367cd507); sha_info->digest[2] = Py_ULL(0x9159015a3070dd17);

--- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -988,7 +988,6 @@ PyLong_AsVoidPtr(PyObject *vv)

rewritten to use the newer PyLong_{As,From}ByteArray API. */

-#define IS_LITTLE_ENDIAN (int)(unsigned char)&one #define PY_ABS_LLONG_MIN (0-(unsigned PY_LONG_LONG)PY_LLONG_MIN) /* Create a new long int object from a C PY_LONG_LONG int. */ @@ -1141,7 +1140,6 @@ PyLong_AsLongLong(PyObject *vv) { PyLongObject *v; PY_LONG_LONG bytes;

int one = 1; int res; if (vv == NULL) { @@ -1176,7 +1174,7 @@ PyLong_AsLongLong(PyObject *vv) case 1: return v->ob_digit[0]; } res = _PyLong_AsByteArray((PyLongObject *)vv, (unsigned char *)&bytes,

                         SIZEOF_LONG_LONG, IS_LITTLE_ENDIAN, 1);[](#l12.23)

                         SIZEOF_LONG_LONG, PY_LITTLE_ENDIAN, 1);[](#l12.24)

/* Plan 9 can't handle PY_LONG_LONG in ? : expressions */ if (res < 0) @@ -1193,7 +1191,6 @@ PyLong_AsUnsignedLongLong(PyObject *vv) { PyLongObject *v; unsigned PY_LONG_LONG bytes;

int one = 1; int res; if (vv == NULL) { @@ -1212,7 +1209,7 @@ PyLong_AsUnsignedLongLong(PyObject *vv) } res = _PyLong_AsByteArray((PyLongObject *)vv, (unsigned char *)&bytes,

                         SIZEOF_LONG_LONG, IS_LITTLE_ENDIAN, 0);[](#l12.40)

                         SIZEOF_LONG_LONG, PY_LITTLE_ENDIAN, 0);[](#l12.41)

/* Plan 9 can't handle PY_LONG_LONG in ? : expressions / if (res < 0) @@ -1288,7 +1285,6 @@ PyLong_AsUnsignedLongLongMask(register P return (unsigned PY_LONG_LONG)-1; } } -#undef IS_LITTLE_ENDIAN / Get a C long long int from a long int object or any object that has an int method.

--- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -47,7 +47,7 @@ STRINGLIB(utf8_decode)(const char **inpt unsigned long value = *(unsigned long *) _s; if (value & ASCII_CHAR_MASK) break; -#ifdef BYTEORDER_IS_LITTLE_ENDIAN +#if PY_LITTLE_ENDIAN _p[0] = (STRINGLIB_CHAR)(value & 0xFFu); _p[1] = (STRINGLIB_CHAR)((value >> 8) & 0xFFu); _p[2] = (STRINGLIB_CHAR)((value >> 16) & 0xFFu); @@ -454,7 +454,7 @@ STRINGLIB(utf16_decode)(const unsigned c const unsigned char *q = *inptr; STRINGLIB_CHAR *p = dest + outpos; / Offsets from q for retrieving byte pairs in the right order. */ -#ifdef BYTEORDER_IS_LITTLE_ENDIAN +#if PY_LITTLE_ENDIAN int ihi = !!native_ordering, ilo = !native_ordering; #else int ihi = !native_ordering, ilo = !!native_ordering; @@ -485,7 +485,7 @@ STRINGLIB(utf16_decode)(const unsigned c block = SWAB(block); #endif } -#ifdef BYTEORDER_IS_LITTLE_ENDIAN +#if PY_LITTLE_ENDIAN

if SIZEOF_LONG == 4

             p[0] = (STRINGLIB_CHAR)(block & 0xFFFFu);[](#l13.28)
             p[1] = (STRINGLIB_CHAR)(block >> 16);[](#l13.29)

--- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -47,14 +47,6 @@ OF OR IN CONNECTION WITH THE USE OR PERF #include <windows.h> #endif -/* Endianness switches; defaults to little endian / - -#ifdef WORDS_BIGENDIAN -# define BYTEORDER_IS_BIG_ENDIAN -#else -# define BYTEORDER_IS_LITTLE_ENDIAN -#endif - / --- Globals ------------------------------------------------------------ The globals are initialized by the _PyUnicode_Init() API and should @@ -4813,7 +4805,7 @@ PyUnicode_DecodeUTF32Stateful(const char int bo = 0; /* assume native ordering by default */ const char errmsg = ""; / Offsets from q for retrieving bytes in the right order. */ -#ifdef BYTEORDER_IS_LITTLE_ENDIAN +#if PY_LITTLE_ENDIAN int iorder[] = {0, 1, 2, 3}; #else int iorder[] = {3, 2, 1, 0}; @@ -4835,7 +4827,7 @@ PyUnicode_DecodeUTF32Stateful(const char if (size >= 4) { const Py_UCS4 bom = (q[iorder[3]] << 24) | (q[iorder[2]] << 16) | (q[iorder[1]] << 8) | q[iorder[0]]; -#ifdef BYTEORDER_IS_LITTLE_ENDIAN +#if PY_LITTLE_ENDIAN if (bom == 0x0000FEFF) { q += 4; bo = -1; @@ -4949,7 +4941,7 @@ PyObject * unsigned char *p; Py_ssize_t nsize, i; /* Offsets from p for storing byte pairs in the right order. */ -#ifdef BYTEORDER_IS_LITTLE_ENDIAN +#if PY_LITTLE_ENDIAN int iorder[] = {0, 1, 2, 3}; #else int iorder[] = {3, 2, 1, 0}; @@ -5092,7 +5084,7 @@ PyUnicode_DecodeUTF16Stateful(const char return unicode_empty; } -#ifdef BYTEORDER_IS_LITTLE_ENDIAN +#if PY_LITTLE_ENDIAN native_ordering = bo <= 0; #else native_ordering = bo >= 0; @@ -5209,7 +5201,7 @@ PyObject * unsigned short *out; Py_ssize_t bytesize; Py_ssize_t pairs; -#ifdef WORDS_BIGENDIAN +#if PY_BIG_ENDIAN int native_ordering = byteorder >= 0; #else int native_ordering = byteorder <= 0;

--- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1561,7 +1561,6 @@ PyObject * _PySys_Init(void) { PyObject *m, *v, *sysdict, *version_info;

char *s;

m = PyModule_Create(&sysmodule); if (m == NULL) @@ -1638,20 +1637,14 @@ PyObject * PyLong_FromLong(0x10FFFF)); SET_SYS_FROM_STRING("builtin_module_names", list_builtin_module_names());

{

   /* Assumes that longs are at least 2 bytes long.[](#l15.16)

```
      Should be safe! */[](#l15.17)
```
```
   unsigned long number = 1;[](#l15.18)
```
```
   char *value;[](#l15.19)
```

+#if PY_BIG_ENDIAN

SET_SYS_FROM_STRING("byteorder",

                   PyUnicode_FromString("big"));[](#l15.22)

+#else

SET_SYS_FROM_STRING("byteorder",

                   PyUnicode_FromString("little"));[](#l15.25)

+#endif

```
   s = (char *) &number;[](#l15.28)
```
```
   if (s[0] == 0)[](#l15.29)
```
```
       value = "big";[](#l15.30)
```
```
   else[](#l15.31)
```
```
       value = "little";[](#l15.32)
```

   SET_SYS_FROM_STRING("byteorder",[](#l15.33)

                       PyUnicode_FromString(value));[](#l15.34)

}

#ifdef MS_COREDLL SET_SYS_FROM_STRING("dllhandle", PyLong_FromVoidPtr(PyWin_DLLhModule));