cpython: 519bde9db8e0 (original) (raw)
Mercurial > cpython
changeset 103022:519bde9db8e0
Issue #11734: Add support for IEEE 754 half-precision floats to the struct module. Original patch by Eli Stevens. [#11734]
Mark Dickinson dickinsm@gmail.com | |
---|---|
date | Sat, 03 Sep 2016 17:21:29 +0100 |
parents | 85f9f8bf2ec8 |
children | fd4e4fa75260 |
files | Doc/library/struct.rst Include/floatobject.h Lib/test/test_struct.py Misc/ACKS Misc/NEWS Modules/_struct.c Objects/floatobject.c |
diffstat | 7 files changed, 393 insertions(+), 11 deletions(-)[+] [-] Doc/library/struct.rst 23 Include/floatobject.h 10 Lib/test/test_struct.py 107 Misc/ACKS 1 Misc/NEWS 3 Modules/_struct.c 76 Objects/floatobject.c 184 |
line wrap: on
line diff
--- a/Doc/library/struct.rst
+++ b/Doc/library/struct.rst
@@ -216,6 +216,8 @@ platform-dependent.
+--------+--------------------------+--------------------+----------------+------------+
| N
| :c:type:size_t
| integer | | (4) |
+--------+--------------------------+--------------------+----------------+------------+
+| e
| (7) | float | 2 | (5) |
++--------+--------------------------+--------------------+----------------+------------+
| f
| :c:type:float
| float | 4 | (5) |
+--------+--------------------------+--------------------+----------------+------------+
| d
| :c:type:double
| float | 8 | (5) |
@@ -257,9 +259,10 @@ Notes:
fits your application.
(5)
- For the
'f'
and'd'
conversion codes, the packed representation uses - the IEEE 754 binary32 (for
'f'
) or binary64 (for'd'
) format, - regardless of the floating-point format used by the platform.
- For the
'f'
,'d'
and'e'
conversion codes, the packed - representation uses the IEEE 754 binary32, binary64 or binary16 format (for
'f'
,'d'
or'e'
respectively), regardless of the floating-point- format used by the platform.
(6)
The 'P'
format character is only available for the native byte ordering
@@ -268,6 +271,16 @@ Notes:
on the host system. The struct module does not interpret this as native
ordering, so the 'P'
format is not available.
+(7)
- The IEEE 754 binary16 "half precision" type was introduced in the 2008
- revision of the
IEEE 754 standard <ieee 754 standard_>
_. It has a sign - bit, a 5-bit exponent and 11-bit precision (with 10 bits explicitly stored),
- and can represent numbers between approximately
6.1e-05
and6.5e+04
- at full precision. This type is not widely supported by C compilers: on a
- typical machine, an unsigned short can be used for storage, but not for math
- operations. See the Wikipedia page on the `half-precision floating-point
- format `_ for more information. +
A format character may be preceded by an integral repeat count. For example,
the format string '4h'
means exactly the same as 'hhhh'
.
@@ -430,3 +443,7 @@ The :mod:struct
module also defines th
The calculated size of the struct (and hence of the bytes object produced
by the :meth:pack
method) corresponding to :attr:format
.
+
+.. _half precision format: https://en.wikipedia.org/wiki/Half-precision_floating-point_format[](#l1.48)
+
+.. _ieee 754 standard: https://en.wikipedia.org/wiki/IEEE_floating_point#IEEE_754-2008[](#l1.50)
--- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -74,9 +74,9 @@ PyAPI_FUNC(double) PyFloat_AsDouble(PyOb
-/* The pack routines write 4 or 8 bytes, starting at p. le is a bool +/* The pack routines write 2, 4 or 8 bytes, starting at p. le is a bool
- last, at p+1, p+3 or p+7), false if you want big-endian format (exponent
- first, at p).
- Return value: 0 if all is OK, -1 if error (and an exception is
- set, most likely OverflowError). @@ -84,6 +84,7 @@ PyAPI_FUNC(double) PyFloat_AsDouble(PyOb
- 1): What this does is undefined if x is a NaN or infinity.
- 2): -0.0 and +0.0 produce the same string. */ +PyAPI_FUNC(int) _PyFloat_Pack2(double x, unsigned char *p, int le); PyAPI_FUNC(int) _PyFloat_Pack4(double x, unsigned char *p, int le); PyAPI_FUNC(int) _PyFloat_Pack8(double x, unsigned char *p, int le); @@ -96,14 +97,15 @@ PyAPI_FUNC(int) _PyFloat_Repr(double x, PyAPI_FUNC(int) _PyFloat_Digits(char buf, double v, int signum); PyAPI_FUNC(void) _PyFloat_DigitsInit(void); -/ The unpack routines read 4 or 8 bytes, starting at p. le is a bool +/ The unpack routines read 2, 4 or 8 bytes, starting at p. le is a bool
- argument, true if the string is in little-endian format (exponent
- last, at p+1, p+3 or p+7), false if big-endian (exponent first, at p).
- Return value: The unpacked double. On error, this is -1.0 and
- PyErr_Occurred() is true (and an exception is set, most likely
- OverflowError). Note that on a non-IEEE platform this will refuse
- to unpack a string that represents a NaN or infinity. */ +PyAPI_FUNC(double) _PyFloat_Unpack2(const unsigned char *p, int le); PyAPI_FUNC(double) _PyFloat_Unpack4(const unsigned char *p, int le); PyAPI_FUNC(double) _PyFloat_Unpack8(const unsigned char *p, int le);
--- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -1,5 +1,6 @@ from collections import abc import array +import math import operator import unittest import struct @@ -366,8 +367,6 @@ class StructTest(unittest.TestCase): # SF bug 705836. "<f" and ">f" had a severe rounding bug, where a carry # from the low-order discarded bits could propagate into the exponent # field, causing the result to be wrong by a factor of 2.
import math[](#l3.14)
- for base in range(1, 33): # smaller <- largest representable float less than base. delta = 0.5 @@ -659,6 +658,110 @@ class UnpackIteratorTest(unittest.TestCa self.assertRaises(StopIteration, next, it) self.assertRaises(StopIteration, next, it)
- def test_half_float(self):
# Little-endian examples from:[](#l3.24)
# http://en.wikipedia.org/wiki/Half_precision_floating-point_format[](#l3.25)
format_bits_float__cleanRoundtrip_list = [[](#l3.26)
(b'\x00\x3c', 1.0),[](#l3.27)
(b'\x00\xc0', -2.0),[](#l3.28)
(b'\xff\x7b', 65504.0), # (max half precision)[](#l3.29)
(b'\x00\x04', 2**-14), # ~= 6.10352 * 10**-5 (min pos normal)[](#l3.30)
(b'\x01\x00', 2**-24), # ~= 5.96046 * 10**-8 (min pos subnormal)[](#l3.31)
(b'\x00\x00', 0.0),[](#l3.32)
(b'\x00\x80', -0.0),[](#l3.33)
(b'\x00\x7c', float('+inf')),[](#l3.34)
(b'\x00\xfc', float('-inf')),[](#l3.35)
(b'\x55\x35', 0.333251953125), # ~= 1/3[](#l3.36)
][](#l3.37)
for le_bits, f in format_bits_float__cleanRoundtrip_list:[](#l3.39)
be_bits = le_bits[::-1][](#l3.40)
self.assertEqual(f, struct.unpack('<e', le_bits)[0])[](#l3.41)
self.assertEqual(le_bits, struct.pack('<e', f))[](#l3.42)
self.assertEqual(f, struct.unpack('>e', be_bits)[0])[](#l3.43)
self.assertEqual(be_bits, struct.pack('>e', f))[](#l3.44)
if sys.byteorder == 'little':[](#l3.45)
self.assertEqual(f, struct.unpack('e', le_bits)[0])[](#l3.46)
self.assertEqual(le_bits, struct.pack('e', f))[](#l3.47)
else:[](#l3.48)
self.assertEqual(f, struct.unpack('e', be_bits)[0])[](#l3.49)
self.assertEqual(be_bits, struct.pack('e', f))[](#l3.50)
# Check for NaN handling:[](#l3.52)
format_bits__nan_list = [[](#l3.53)
('<e', b'\x01\xfc'),[](#l3.54)
('<e', b'\x00\xfe'),[](#l3.55)
('<e', b'\xff\xff'),[](#l3.56)
('<e', b'\x01\x7c'),[](#l3.57)
('<e', b'\x00\x7e'),[](#l3.58)
('<e', b'\xff\x7f'),[](#l3.59)
][](#l3.60)
for formatcode, bits in format_bits__nan_list:[](#l3.62)
self.assertTrue(math.isnan(struct.unpack('<e', bits)[0]))[](#l3.63)
self.assertTrue(math.isnan(struct.unpack('>e', bits[::-1])[0]))[](#l3.64)
# Check that packing produces a bit pattern representing a quiet NaN:[](#l3.66)
# all exponent bits and the msb of the fraction should all be 1.[](#l3.67)
packed = struct.pack('<e', math.nan)[](#l3.68)
self.assertEqual(packed[1] & 0x7e, 0x7e)[](#l3.69)
packed = struct.pack('<e', -math.nan)[](#l3.70)
self.assertEqual(packed[1] & 0x7e, 0x7e)[](#l3.71)
# Checks for round-to-even behavior[](#l3.73)
format_bits_float__rounding_list = [[](#l3.74)
('>e', b'\x00\x01', 2.0**-25 + 2.0**-35), # Rounds to minimum subnormal[](#l3.75)
('>e', b'\x00\x00', 2.0**-25), # Underflows to zero (nearest even mode)[](#l3.76)
('>e', b'\x00\x00', 2.0**-26), # Underflows to zero[](#l3.77)
('>e', b'\x03\xff', 2.0**-14 - 2.0**-24), # Largest subnormal.[](#l3.78)
('>e', b'\x03\xff', 2.0**-14 - 2.0**-25 - 2.0**-65),[](#l3.79)
('>e', b'\x04\x00', 2.0**-14 - 2.0**-25),[](#l3.80)
('>e', b'\x04\x00', 2.0**-14), # Smallest normal.[](#l3.81)
('>e', b'\x3c\x01', 1.0+2.0**-11 + 2.0**-16), # rounds to 1.0+2**(-10)[](#l3.82)
('>e', b'\x3c\x00', 1.0+2.0**-11), # rounds to 1.0 (nearest even mode)[](#l3.83)
('>e', b'\x3c\x00', 1.0+2.0**-12), # rounds to 1.0[](#l3.84)
('>e', b'\x7b\xff', 65504), # largest normal[](#l3.85)
('>e', b'\x7b\xff', 65519), # rounds to 65504[](#l3.86)
('>e', b'\x80\x01', -2.0**-25 - 2.0**-35), # Rounds to minimum subnormal[](#l3.87)
('>e', b'\x80\x00', -2.0**-25), # Underflows to zero (nearest even mode)[](#l3.88)
('>e', b'\x80\x00', -2.0**-26), # Underflows to zero[](#l3.89)
('>e', b'\xbc\x01', -1.0-2.0**-11 - 2.0**-16), # rounds to 1.0+2**(-10)[](#l3.90)
('>e', b'\xbc\x00', -1.0-2.0**-11), # rounds to 1.0 (nearest even mode)[](#l3.91)
('>e', b'\xbc\x00', -1.0-2.0**-12), # rounds to 1.0[](#l3.92)
('>e', b'\xfb\xff', -65519), # rounds to 65504[](#l3.93)
][](#l3.94)
for formatcode, bits, f in format_bits_float__rounding_list:[](#l3.96)
self.assertEqual(bits, struct.pack(formatcode, f))[](#l3.97)
# This overflows, and so raises an error[](#l3.99)
format_bits_float__roundingError_list = [[](#l3.100)
# Values that round to infinity.[](#l3.101)
('>e', 65520.0),[](#l3.102)
('>e', 65536.0),[](#l3.103)
('>e', 1e300),[](#l3.104)
('>e', -65520.0),[](#l3.105)
('>e', -65536.0),[](#l3.106)
('>e', -1e300),[](#l3.107)
('<e', 65520.0),[](#l3.108)
('<e', 65536.0),[](#l3.109)
('<e', 1e300),[](#l3.110)
('<e', -65520.0),[](#l3.111)
('<e', -65536.0),[](#l3.112)
('<e', -1e300),[](#l3.113)
][](#l3.114)
for formatcode, f in format_bits_float__roundingError_list:[](#l3.116)
self.assertRaises(OverflowError, struct.pack, formatcode, f)[](#l3.117)
# Double rounding[](#l3.119)
format_bits_float__doubleRoundingError_list = [[](#l3.120)
('>e', b'\x67\xff', 0x1ffdffffff * 2**-26), # should be 2047, if double-rounded 64>32>16, becomes 2048[](#l3.121)
][](#l3.122)
for formatcode, bits, f in format_bits_float__doubleRoundingError_list:[](#l3.124)
self.assertEqual(bits, struct.pack(formatcode, f))[](#l3.125)
+ if name == 'main': unittest.main()
--- a/Misc/ACKS +++ b/Misc/ACKS @@ -1435,6 +1435,7 @@ Greg Stein Marek Stepniowski Baruch Sterin Chris Stern +Eli Stevens Alex Stewart Victor Stinner Richard Stoakley
--- a/Misc/NEWS +++ b/Misc/NEWS @@ -69,6 +69,9 @@ Core and Builtins Library ------- +- Issue #11734: Add support for IEEE 754 half-precision floats to the
--- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -267,6 +267,33 @@ get_size_t(PyObject *v, size_t p) / Floating point helpers */ static PyObject * +unpack_halffloat(const char p, / start of 2-byte string */
int le) /* true for little-endian, false for big-endian */[](#l6.8)
- x = _PyFloat_Unpack2((unsigned char *)p, le);
- if (x == -1.0 && PyErr_Occurred()) {
return NULL;[](#l6.14)
- }
- return PyFloat_FromDouble(x);
+} + +static int +pack_halffloat(char p, / start of 2-byte string */
PyObject *v, /* value to pack */[](#l6.21)
int le) /* true for little-endian, false for big-endian */[](#l6.22)
- double x = PyFloat_AsDouble(v);
- if (x == -1.0 && PyErr_Occurred()) {
PyErr_SetString(StructError,[](#l6.26)
"required argument is not a float");[](#l6.27)
return -1;[](#l6.28)
- }
- return _PyFloat_Pack2(x, (unsigned char *)p, le);
+} + +static PyObject * unpack_float(const char p, / start of 4-byte string / int le) / true for little-endian, false for big-endian */ { @@ -470,6 +497,16 @@ nu_bool(const char *p, const formatdef * static PyObject * +nu_halffloat(const char *p, const formatdef *f) +{ +#if PY_LITTLE_ENDIAN
+#endif +} + +static PyObject * nu_float(const char *p, const formatdef *f) { float x; @@ -681,6 +718,16 @@ np_bool(char *p, PyObject *v, const form } static int +np_halffloat(char *p, PyObject *v, const formatdef *f) +{ +#if PY_LITTLE_ENDIAN
+#endif +} + +static int np_float(char *p, PyObject *v, const formatdef *f) { float x = (float)PyFloat_AsDouble(v); @@ -743,6 +790,7 @@ static const formatdef native_table[] = {'Q', sizeof(PY_LONG_LONG), LONG_LONG_ALIGN, nu_ulonglong,np_ulonglong}, #endif {'?', sizeof(BOOL_TYPE), BOOL_ALIGN, nu_bool, np_bool},
- {'e', sizeof(short), SHORT_ALIGN, nu_halffloat, np_halffloat}, {'f', sizeof(float), FLOAT_ALIGN, nu_float, np_float}, {'d', sizeof(double), DOUBLE_ALIGN, nu_double, np_double}, {'P', sizeof(void *), VOID_P_ALIGN, nu_void_p, np_void_p}, @@ -826,6 +874,12 @@ bu_ulonglong(const char *p, const format }
static PyObject * +bu_halffloat(const char *p, const formatdef *f) +{
+} + +static PyObject * bu_float(const char *p, const formatdef *f) { return unpack_float(p, 0); @@ -922,6 +976,12 @@ bp_ulonglong(char *p, PyObject *v, const } static int +bp_halffloat(char *p, PyObject *v, const formatdef *f) +{
+} + +static int bp_float(char *p, PyObject *v, const formatdef *f) { double x = PyFloat_AsDouble(v); @@ -972,6 +1032,7 @@ static formatdef bigendian_table[] = { {'q', 8, 0, bu_longlong, bp_longlong}, {'Q', 8, 0, bu_ulonglong, bp_ulonglong}, {'?', 1, 0, bu_bool, bp_bool},
- {'e', 2, 0, bu_halffloat, bp_halffloat}, {'f', 4, 0, bu_float, bp_float}, {'d', 8, 0, bu_double, bp_double}, {0} @@ -1054,6 +1115,12 @@ lu_ulonglong(const char *p, const format }
static PyObject * +lu_halffloat(const char *p, const formatdef *f) +{
+} + +static PyObject * lu_float(const char *p, const formatdef *f) { return unpack_float(p, 1); @@ -1142,6 +1209,12 @@ lp_ulonglong(char *p, PyObject *v, const } static int +lp_halffloat(char *p, PyObject *v, const formatdef *f) +{
+} + +static int lp_float(char *p, PyObject *v, const formatdef f) { double x = PyFloat_AsDouble(v); @@ -1182,6 +1255,7 @@ static formatdef lilendian_table[] = { {'Q', 8, 0, lu_ulonglong, lp_ulonglong}, {'?', 1, 0, bu_bool, bp_bool}, / Std rep not endian dep, but potentially different from native rep -- reuse bx_bool funcs. */
- {'e', 2, 0, lu_halffloat, lp_halffloat}, {'f', 4, 0, lu_float, lp_float}, {'d', 8, 0, lu_double, lp_double}, {0} @@ -2239,7 +2313,7 @@ these can be preceded by a decimal repea x: pad byte (no data); c:char; b:signed byte; B:unsigned byte;\n[](#l6.148) ?: _Bool (requires C99; if not available, char is used instead)\n[](#l6.149) h:short; H:unsigned short; i:int; I:unsigned int;\n[](#l6.150)
- l:long; L:unsigned long; f:float; d:double.\n[](#l6.151)
- l:long; L:unsigned long; f:float; d:double; e:half-float.\n[](#l6.152) Special cases (preceding decimal count indicates length):\n[](#l6.153) s:string (array of char); p: pascal string (with count byte).\n[](#l6.154) Special cases (only available in native format):\n[](#l6.155)
--- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1975,8 +1975,120 @@ void /*----------------------------------------------------------------------------
bits = (unsigned short)f; Note the truncation[](#l7.12)
if ((f - bits > 0.5) || (f - bits == 0.5 && bits % 2)) {[](#l7.13)
bits++;[](#l7.14)
}[](#l7.15)
*/ + +int +_PyFloat_Pack2(double x, unsigned char *p, int le) +{
- unsigned char sign;
- int e;
- double f;
- unsigned short bits;
- int incr = 1;
- if (x == 0.0) {
sign = (copysign(1.0, x) == -1.0);[](#l7.28)
e = 0;[](#l7.29)
bits = 0;[](#l7.30)
- }
- else if (Py_IS_INFINITY(x)) {
sign = (x < 0.0);[](#l7.33)
e = 0x1f;[](#l7.34)
bits = 0;[](#l7.35)
- }
- else if (Py_IS_NAN(x)) {
/* There are 2046 distinct half-precision NaNs (1022 signaling and[](#l7.38)
1024 quiet), but there are only two quiet NaNs that don't arise by[](#l7.39)
quieting a signaling NaN; we get those by setting the topmost bit[](#l7.40)
of the fraction field and clearing all other fraction bits. We[](#l7.41)
choose the one with the appropriate sign. */[](#l7.42)
sign = (copysign(1.0, x) == -1.0);[](#l7.43)
e = 0x1f;[](#l7.44)
bits = 512;[](#l7.45)
- }
- else {
sign = (x < 0.0);[](#l7.48)
if (sign) {[](#l7.49)
x = -x;[](#l7.50)
}[](#l7.51)
f = frexp(x, &e);[](#l7.53)
if (f < 0.5 || f >= 1.0) {[](#l7.54)
PyErr_SetString(PyExc_SystemError,[](#l7.55)
"frexp() result out of range");[](#l7.56)
return -1;[](#l7.57)
}[](#l7.58)
/* Normalize f to be in the range [1.0, 2.0) */[](#l7.60)
f *= 2.0;[](#l7.61)
e--;[](#l7.62)
if (e >= 16) {[](#l7.64)
goto Overflow;[](#l7.65)
}[](#l7.66)
else if (e < -25) {[](#l7.67)
/* |x| < 2**-25. Underflow to zero. */[](#l7.68)
f = 0.0;[](#l7.69)
e = 0;[](#l7.70)
}[](#l7.71)
else if (e < -14) {[](#l7.72)
/* |x| < 2**-14. Gradual underflow */[](#l7.73)
f = ldexp(f, 14 + e);[](#l7.74)
e = 0;[](#l7.75)
}[](#l7.76)
else /* if (!(e == 0 && f == 0.0)) */ {[](#l7.77)
e += 15;[](#l7.78)
f -= 1.0; /* Get rid of leading 1 */[](#l7.79)
}[](#l7.80)
f *= 1024.0; /* 2**10 */[](#l7.82)
/* Round to even */[](#l7.83)
bits = (unsigned short)f; /* Note the truncation */[](#l7.84)
assert(bits < 1024);[](#l7.85)
assert(e < 31);[](#l7.86)
if ((f - bits > 0.5) || ((f - bits == 0.5) && (bits % 2 == 1))) {[](#l7.87)
++bits;[](#l7.88)
if (bits == 1024) {[](#l7.89)
/* The carry propagated out of a string of 10 1 bits. */[](#l7.90)
bits = 0;[](#l7.91)
++e;[](#l7.92)
if (e == 31)[](#l7.93)
goto Overflow;[](#l7.94)
}[](#l7.95)
}[](#l7.96)
- }
- Overflow:
- PyErr_SetString(PyExc_OverflowError,
"float too large to pack with e format");[](#l7.118)
- return -1;
+} + int _PyFloat_Pack4(double x, unsigned char *p, int le) { @@ -2212,6 +2324,76 @@ int } double +_PyFloat_Unpack2(const unsigned char *p, int le) +{
+#ifdef PY_NO_SHORT_FLOAT_REPR
if (f == 0) {[](#l7.153)
/* Infinity */[](#l7.154)
return sign ? -Py_HUGE_VAL : Py_HUGE_VAL;[](#l7.155)
}[](#l7.156)
else {[](#l7.157)
/* NaN */[](#l7.158)
return sign ? -Py_NAN : Py_NAN;[](#l7.160)
PyErr_SetString([](#l7.162)
PyExc_ValueError,[](#l7.163)
"can't unpack IEEE 754 NaN "[](#l7.164)
"on platform that does not support NaNs");[](#l7.165)
return -1;[](#l7.166)
}[](#l7.168)
if (f == 0) {[](#l7.170)
/* Infinity */[](#l7.171)
return _Py_dg_infinity(sign);[](#l7.172)
}[](#l7.173)
else {[](#l7.174)
/* NaN */[](#l7.175)
return _Py_dg_stdnan(sign);[](#l7.176)
}[](#l7.177)
+#endif /* #ifdef PY_NO_SHORT_FLOAT_REPR */
- if (e == 0) {
e = -14;[](#l7.184)
- }
- else {
x += 1.0;[](#l7.187)
e -= 15;[](#l7.188)
- }
- x = ldexp(x, e);
+} + +double _PyFloat_Unpack4(const unsigned char *p, int le) { if (float_format == unknown_format) {