[Python-Dev] cpython: Issue #16148: implemented PEP 424 (original) (raw)

Georg Brandl g.brandl at gmx.net
Sat Oct 6 14:35:24 CEST 2012

Am 06.10.2012 14:12, schrieb armin.ronacher:

http://hg.python.org/cpython/rev/a7ec0a1b0f7c changeset: 79511:a7ec0a1b0f7c parent: 79507:3c1df1ede882 user: Armin Ronacher <armin.ronacher at active-4.com> date: Sat Oct 06 14:03:24 2012 +0200 summary: Issue #16148: implemented PEP 424

files: Doc/c-api/object.rst | 7 ++ Doc/library/operator.rst | 6 + Include/abstract.h | 5 +- Lib/test/testenumerate.py | 9 +- Lib/test/testiterlen.py | 62 +++++++++---------- Lib/test/testitertools.py | 5 +- Lib/test/testoperator.py | 25 ++++++++ Lib/test/testset.py | 2 - Modules/operator.c | 27 ++++++++ Objects/abstract.c | 80 +++++++++++++++---------- Objects/bytearrayobject.c | 2 +- Objects/bytesobject.c | 2 +- Objects/iterobject.c | 11 ++- Objects/listobject.c | 2 +- 14 files changed, 162 insertions(+), 83 deletions(-)

diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -342,6 +342,13 @@ returned. This is the equivalent to the Python expression len(o). +.. c:function:: Pyssizet PyObjectLengthHint(PyObject *o, Pyssizet default) + + Return an estimated length for the object o. First trying to return its + actual length, then an estimate using _lengthhint_, and finally + returning the default value. On error -1 is returned. This is the + equivalent to the Python expression operator.lengthhint(o, default).

Needs a versionadded.

Since length_hint is now official, it needs an entry in Doc/reference/datamodel.rst (which you can link to here.)

.. c:function:: PyObject* PyObjectGetItem(PyObject *o, PyObject *key)

Return element of o corresponding to the object key or NULL on failure. diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -235,6 +235,12 @@ .. XXX: find a better, readable, example +.. function:: lengthhint(obj, default=0) + + Return an estimated length for the object o. First trying to return its + actual length, then an estimate using _lengthhint_, and finally + returning the default value.

This one also needs versionadded and a link to length_hint.

The :mod:operator module also defines tools for generalized attribute and item lookups. These are useful for making fast field extractors as arguments for :func:map, :func:sorted, :meth:itertools.groupby, or other functions that diff --git a/Include/abstract.h b/Include/abstract.h --- a/Include/abstract.h +++ b/Include/abstract.h @@ -403,9 +403,8 @@ PyAPIFUNC(Pyssizet) PyObjectLength(PyObject *o); #define PyObjectLength PyObjectSize

-#ifndef PyLIMITEDAPI - PyAPIFUNC(Pyssizet) PyObjectLengthHint(PyObject *o, Pyssizet); -#endif +PyAPIFUNC(int) PyObjectHasLen(PyObject *o); +PyAPIFUNC(Pyssizet) PyObjectLengthHint(PyObject *o, Pyssizet);

Not sure if new functions should be included in the limited API. I seem to recall some discussion about giving Py_LIMITED_API a numeric value for the required API version? PEP 384 is silent about it.

/* Guess the size of object o using len(o) or o.lengthhint(). diff --git a/Lib/test/testenumerate.py b/Lib/test/testenumerate.py --- a/Lib/test/testenumerate.py +++ b/Lib/test/testenumerate.py @@ -1,4 +1,5 @@ import unittest +import operator import sys import pickle

@@ -168,15 +169,13 @@ x = range(1) self.assertEqual(type(reversed(x)), type(iter(x))) - @support.cpythononly def testlen(self): # This is an implementation detail, not an interface requirement

If it's not cpython_only anymore, this comment should also vanish?

- from test.testiterlen import len for s in ('hello', tuple('hello'), list('hello'), range(5)): - self.assertEqual(len(reversed(s)), len(s)) + self.assertEqual(operator.lengthhint(reversed(s)), len(s)) r = reversed(s) list(r) - self.assertEqual(len(r), 0) + self.assertEqual(operator.lengthhint(r), 0) class SeqWithWeirdLen: called = False def len(self): @@ -187,7 +186,7 @@ def getitem(self, index): return index r = reversed(SeqWithWeirdLen()) - self.assertRaises(ZeroDivisionError, len, r) + self.assertRaises(ZeroDivisionError, operator.lengthhint, r)

def testgc(self): diff --git a/Lib/test/testiterlen.py b/Lib/test/testiterlen.py --- a/Lib/test/testiterlen.py +++ b/Lib/test/testiterlen.py @@ -45,31 +45,21 @@ from test import support from itertools import repeat from collections import deque -from builtins import len as len +from operator import lengthhint

## ------- Concrete Type Tests -------

@@ -92,10 +82,6 @@ def setUp(self): self.it = repeat(None, n) - def testnolenforinfiniterepeat(self): - # The repeat() object can also be infinite - self.assertRaises(TypeError, len, repeat(None))

Why is this removed? I can see it was duplicated in test_itertools (below), but you removed both instances.

diff --git a/Lib/test/testitertools.py b/Lib/test/testitertools.py --- a/Lib/test/testitertools.py +++ b/Lib/test/testitertools.py @@ -1723,9 +1723,8 @@ class LengthTransparency(unittest.TestCase):

def testrepeat(self): - from test.testiterlen import len - self.assertEqual(len(repeat(None, 50)), 50) - self.assertRaises(TypeError, len, repeat(None)) + self.assertEqual(operator.lengthhint(repeat(None, 50)), 50) + self.assertEqual(operator.lengthhint(repeat(None), 12), 12) class RegressionTests(unittest.TestCase): diff --git a/Lib/test/testoperator.py b/Lib/test/testoperator.py --- a/Lib/test/testoperator.py +++ b/Lib/test/testoperator.py @@ -410,6 +410,31 @@ self.assertEqual(operator.ixor (c, 5), "ixor") self.assertEqual(operator.iconcat (c, c), "iadd") + def testlengthhint(self): + class X(object): + def init(self, value): + self.value = value + + def lengthhint(self): + if type(self.value) is type: + raise self.value + else: + return self.value + + self.assertEqual(operator.lengthhint([], 2), 0) + self.assertEqual(operator.lengthhint(iter([1, 2, 3])), 3) + + self.assertEqual(operator.lengthhint(X(2)), 2) + self.assertEqual(operator.lengthhint(X(NotImplemented), 4), 4) + self.assertEqual(operator.lengthhint(X(TypeError), 12), 12) + with self.assertRaises(TypeError): + operator.lengthhint(X("abc")) + with self.assertRaises(ValueError): + operator.lengthhint(X(-2)) + with self.assertRaises(LookupError): + operator.lengthhint(X(LookupError)) + + def testmain(verbose=None): import sys testclasses = ( diff --git a/Lib/test/testset.py b/Lib/test/testset.py --- a/Lib/test/testset.py +++ b/Lib/test/testset.py @@ -848,8 +848,6 @@ for v in self.set: self.assertIn(v, self.values) setiter = iter(self.set) - # note: lengthhint is an internal undocumented API, - # don't rely on it in your own programs self.assertEqual(setiter.lengthhint(), len(self.set)) def testpickling(self): diff --git a/Modules/operator.c b/Modules/operator.c --- a/Modules/operator.c +++ b/Modules/operator.c @@ -208,6 +208,31 @@ return (result == 0); } +PyDocSTRVAR(lengthhint_doc,_ +"lengthhint(obj, default=0) -> int\n" +"Return an estimate of the number of items in obj.\n" +"This is useful for presizing containers when building from an\n" +"iterable.\n" +"\n" +"If the object supports len(), the result will be\n" +"exact. Otherwise, it may over- or under-estimate by an\n" +"arbitrary amount. The result will be an integer >= 0."); + +static PyObject *lengthhint(PyObject *self, PyObject *args) +{ + PyObject *obj; + Pyssizet defaultvalue = 0, res; + if (!PyArgParseTuple(args, "O|n:lengthhint", &obj, &defaultvalue)) { + return NULL; + } + res = PyObjectLengthHint(obj, defaultvalue); + if (res == -1 && PyErrOccurred()) { + return NULL; + } + return PyLongFromSsizet(res); +} + + PyDocSTRVAR(comparedigest_doc,_ "comparedigest(a, b) -> bool\n" "\n" @@ -366,6 +391,8 @@ {"comparedigest", (PyCFunction)comparedigest, METHVARARGS, comparedigest_doc},_ + {"lengthhint", (PyCFunction)lengthhint, METHVARARGS, + lengthhint_doc},_ {NULL, NULL} /* sentinel */ }; diff --git a/Objects/abstract.c b/Objects/abstract.c --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -64,49 +64,67 @@ } #define PyObjectLength PyObjectSize +int +PyObjectHasLen(PyObject *o) { + return (PyTYPE(o)->tpassequence && PyTYPE(o)->tpassequence->sqlength) || + (PyTYPE(o)->tpasmapping && PyTYPE(o)->tpasmapping->mplength); +} /* The length hint function returns a non-negative value from o.len() - or o.lengthhint(). If those methods aren't found or return a negative - value, then the defaultvalue is returned. If one of the calls fails, - this function returns -1. + or o.lengthhint(). If those methods aren't found. If one of the calls ^^^^^ Sentence incomplete.

+ fails this function returns -1. */

Pyssizet -PyObjectLengthHint(PyObject *o, Pyssizet defaultvalue) +PyObjectLengthHint(PyObject *o, Pyssizet defaultvalue)

Should we put "#define _PyObject_LengthHint PyObject_LengthHint" in some header to make porting easier?

{ PyIDENTIFIER(lengthhint); - PyObject *ro, *hintmeth; - Pyssizet rv; - - /* try o.len() */ - rv = PyObjectSize(o); - if (rv >= 0) - return rv; - if (PyErrOccurred()) { - if (!PyErrExceptionMatches(PyExcTypeError)) + Pyssizet res = PyObjectLength(o); + if (res < 0 && PyErrOccurred()) { + if (!PyErrExceptionMatches(PyExcTypeError)) { return -1; + } PyErrClear(); } - - /* try o.lengthhint() */ - hintmeth = PyObjectLookupSpecial(o, &PyId_lengthhint);_ - if (hintmeth == NULL) { - if (PyErrOccurred()) + else { + return res; + } + PyObject *hint = PyObjectLookupSpecial(o, &PyId_lengthhint);_ + if (hint == NULL) { + if (PyErrOccurred()) { return -1; - else - return defaultvalue; - } - ro = PyObjectCallFunctionObjArgs(hintmeth, NULL); - PyDECREF(hintmeth); - if (ro == NULL) { - if (!PyErrExceptionMatches(PyExcTypeError)) - return -1; - PyErrClear(); + } return defaultvalue; } - rv = PyLongCheck(ro) ? PyLongAsSsizet(ro) : defaultvalue; - PyDECREF(ro); - return rv; + PyObject *result = PyObjectCallFunctionObjArgs(hint, NULL); + PyDECREF(hint); + if (result == NULL) { + if (PyErrExceptionMatches(PyExcTypeError)) { + PyErrClear(); + return defaultvalue; + } + return -1; + } + else if (result == PyNotImplemented) { + PyDECREF(result); + return defaultvalue; + } + if (!PyLongCheck(result)) { + PyErrFormat(PyExcTypeError, "Length hint must be an integer, not %s",

We usually limit the string size here, e.g. "%.100s".

+ PyTYPE(result)->tpname); + PyDECREF(result); + return -1; + } + defaultvalue = PyLongAsSsizet(result);

Not sure the micro-optimization is worth the confusion of reassigning defaultvalue.

+ PyDECREF(result); + if (defaultvalue < 0 && PyErrOccurred()) {_ _+ return -1;_ _+ }_ _+ if (defaultvalue < 0) {_ _+ PyErrFormat(PyExcValueError, "_lengthhint_() should return >= 0");

Exception message is inconsistent with above: "Length hint" vs "length_hint()".

+ return -1; + } + return defaultvalue; }

diff --git a/Objects/iterobject.c b/Objects/iterobject.c --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -76,9 +76,14 @@ Pyssizet seqsize, len;

if (it->itseq) { - seqsize = PySequenceSize(it->itseq); - if (seqsize == -1) - return NULL; + if (PyObjectHasLen(it->itseq)) { + seqsize = PySequenceSize(it->itseq); + if (seqsize == -1) + return NULL; + } + else { + return PyNotImplemented;

An INCREF is missing here, as discussed on IRC.

+ } len = seqsize - it->itindex; if (len >= 0) return PyLongFromSsizet(len);

More information about the Python-Dev mailing list