(original) (raw)

diff -r ac6d33317eda Include/Python-ast.h --- a/Include/Python-ast.h Mon Jan 18 12:15:08 2016 +0100 +++ b/Include/Python-ast.h Mon Jan 18 17:50:20 2016 +0100 @@ -202,9 +202,9 @@ enum _expr_kind {BoolOp_kind=1, BinOp_ki Await_kind=12, Yield_kind=13, YieldFrom_kind=14, Compare_kind=15, Call_kind=16, Num_kind=17, Str_kind=18, FormattedValue_kind=19, JoinedStr_kind=20, Bytes_kind=21, - NameConstant_kind=22, Ellipsis_kind=23, Attribute_kind=24, - Subscript_kind=25, Starred_kind=26, Name_kind=27, - List_kind=28, Tuple_kind=29}; + NameConstant_kind=22, Ellipsis_kind=23, Constant_kind=24, + Attribute_kind=25, Subscript_kind=26, Starred_kind=27, + Name_kind=28, List_kind=29, Tuple_kind=30}; struct _expr { enum _expr_kind kind; union { @@ -316,6 +316,10 @@ struct _expr { } NameConstant; struct { + constant value; + } Constant; + + struct { expr_ty value; identifier attr; expr_context_ty ctx; @@ -567,6 +571,9 @@ expr_ty _Py_NameConstant(singleton value *arena); #define Ellipsis(a0, a1, a2) _Py_Ellipsis(a0, a1, a2) expr_ty _Py_Ellipsis(int lineno, int col_offset, PyArena *arena); +#define Constant(a0, a1, a2, a3) _Py_Constant(a0, a1, a2, a3) +expr_ty _Py_Constant(constant value, int lineno, int col_offset, PyArena + *arena); #define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5) expr_ty _Py_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int col_offset, PyArena *arena); diff -r ac6d33317eda Include/asdl.h --- a/Include/asdl.h Mon Jan 18 12:15:08 2016 +0100 +++ b/Include/asdl.h Mon Jan 18 17:50:20 2016 +0100 @@ -6,6 +6,7 @@ typedef PyObject * string; typedef PyObject * bytes; typedef PyObject * object; typedef PyObject * singleton; +typedef PyObject * constant; /* It would be nice if the code generated by asdl_c.py was completely independent of Python, but it is a goal the requires too much work diff -r ac6d33317eda Lib/ast.py --- a/Lib/ast.py Mon Jan 18 12:15:08 2016 +0100 +++ b/Lib/ast.py Mon Jan 18 17:50:20 2016 +0100 @@ -47,7 +47,9 @@ def literal_eval(node_or_string): if isinstance(node_or_string, Expression): node_or_string = node_or_string.body def _convert(node): - if isinstance(node, (Str, Bytes)): + if isinstance(node, Constant): + return node.value + elif isinstance(node, (Str, Bytes)): return node.s elif isinstance(node, Num): return node.n @@ -64,22 +66,25 @@ def literal_eval(node_or_string): return node.value elif isinstance(node, UnaryOp) and \ isinstance(node.op, (UAdd, USub)) and \ - isinstance(node.operand, (Num, UnaryOp, BinOp)): + isinstance(node.operand, (Num, Constant, UnaryOp, BinOp)): operand = _convert(node.operand) - if isinstance(node.op, UAdd): - return + operand - else: - return - operand + if isinstance(operand, (int, float, complex)): + if isinstance(node.op, UAdd): + return + operand + else: + return - operand elif isinstance(node, BinOp) and \ isinstance(node.op, (Add, Sub)) and \ - isinstance(node.right, (Num, UnaryOp, BinOp)) and \ - isinstance(node.left, (Num, UnaryOp, BinOp)): + isinstance(node.right, (Num, Constant, UnaryOp, BinOp)) and \ + isinstance(node.left, (Num, Constant, UnaryOp, BinOp)): left = _convert(node.left) right = _convert(node.right) - if isinstance(node.op, Add): - return left + right - else: - return left - right + if (isinstance(left, (int, float, complex)) + and isinstance(right, (int, float, complex))): + if isinstance(node.op, Add): + return left + right + else: + return left - right raise ValueError('malformed node or string: ' + repr(node)) return _convert(node_or_string) @@ -196,12 +201,19 @@ def get_docstring(node, clean=True): """ if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)): raise TypeError("%r can't have docstrings" % node.__class__.__name__) - if node.body and isinstance(node.body[0], Expr) and \ - isinstance(node.body[0].value, Str): - if clean: - import inspect - return inspect.cleandoc(node.body[0].value.s) - return node.body[0].value.s + if not(node.body and isinstance(node.body[0], Expr)): + return + node = node.body[0].value + if isinstance(node, Str): + text = node.s + elif isinstance(node, Constant): + text = node.value + else: + return + if clean: + import inspect + text = inspect.cleandoc(text) + return text def walk(node): diff -r ac6d33317eda Parser/Python.asdl --- a/Parser/Python.asdl Mon Jan 18 12:15:08 2016 +0100 +++ b/Parser/Python.asdl Mon Jan 18 17:50:20 2016 +0100 @@ -1,4 +1,5 @@ --- ASDL's six builtin types are identifier, int, string, bytes, object, singleton +-- ASDL's 7 builtin types are: +-- identifier, int, string, bytes, object, singleton, constant module Python { @@ -76,6 +77,7 @@ module Python | Bytes(bytes s) | NameConstant(singleton value) | Ellipsis + | Constant(constant value) -- the following expression can appear in assignment context | Attribute(expr value, identifier attr, expr_context ctx) diff -r ac6d33317eda Parser/asdl.py --- a/Parser/asdl.py Mon Jan 18 12:15:08 2016 +0100 +++ b/Parser/asdl.py Mon Jan 18 17:50:20 2016 +0100 @@ -33,7 +33,8 @@ import re # See the EBNF at the top of the file to understand the logical connection # between the various node types. -builtin_types = {'identifier', 'string', 'bytes', 'int', 'object', 'singleton'} +builtin_types = {'identifier', 'string', 'bytes', 'int', 'object', 'singleton', + 'constant'} class AST: def __repr__(self): diff -r ac6d33317eda Parser/asdl_c.py --- a/Parser/asdl_c.py Mon Jan 18 12:15:08 2016 +0100 +++ b/Parser/asdl_c.py Mon Jan 18 17:50:20 2016 +0100 @@ -834,6 +834,7 @@ static PyObject* ast2obj_object(void *o) return (PyObject*)o; } #define ast2obj_singleton ast2obj_object +#define ast2obj_constant ast2obj_object #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object #define ast2obj_bytes ast2obj_object @@ -871,6 +872,24 @@ static int obj2ast_object(PyObject* obj, return 0; } +static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena) +{ + if (obj == Py_None || obj == Py_True || obj == Py_False) { + *out = obj; + return 0; + } + + if (obj) { + if (PyArena_AddPyObject(arena, obj) < 0) { + *out = NULL; + return -1; + } + Py_INCREF(obj); + } + *out = obj; + return 0; +} + static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && obj != Py_None) { diff -r ac6d33317eda Python/Python-ast.c --- a/Python/Python-ast.c Mon Jan 18 12:15:08 2016 +0100 +++ b/Python/Python-ast.c Mon Jan 18 17:50:20 2016 +0100 @@ -306,6 +306,10 @@ static char *NameConstant_fields[]={ "value", }; static PyTypeObject *Ellipsis_type; +static PyTypeObject *Constant_type; +static char *Constant_fields[]={ + "value", +}; static PyTypeObject *Attribute_type; _Py_IDENTIFIER(attr); _Py_IDENTIFIER(ctx); @@ -709,6 +713,7 @@ static PyObject* ast2obj_object(void *o) return (PyObject*)o; } #define ast2obj_singleton ast2obj_object +#define ast2obj_constant ast2obj_object #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object #define ast2obj_bytes ast2obj_object @@ -746,6 +751,24 @@ static int obj2ast_object(PyObject* obj, return 0; } +static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena) +{ + if (obj == Py_None || obj == Py_True || obj == Py_False) { + *out = obj; + return 0; + } + + if (obj) { + if (PyArena_AddPyObject(arena, obj) < 0) { + *out = NULL; + return -1; + } + Py_INCREF(obj); + } + *out = obj; + return 0; +} + static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && obj != Py_None) { @@ -941,6 +964,8 @@ static int init_types(void) if (!NameConstant_type) return 0; Ellipsis_type = make_type("Ellipsis", expr_type, NULL, 0); if (!Ellipsis_type) return 0; + Constant_type = make_type("Constant", expr_type, Constant_fields, 1); + if (!Constant_type) return 0; Attribute_type = make_type("Attribute", expr_type, Attribute_fields, 3); if (!Attribute_type) return 0; Subscript_type = make_type("Subscript", expr_type, Subscript_fields, 3); @@ -2167,6 +2192,25 @@ Ellipsis(int lineno, int col_offset, PyA } expr_ty +Constant(constant value, int lineno, int col_offset, PyArena *arena) +{ + expr_ty p; + if (!value) { + PyErr_SetString(PyExc_ValueError, + "field value is required for Constant"); + return NULL; + } + p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = Constant_kind; + p->v.Constant.value = value; + p->lineno = lineno; + p->col_offset = col_offset; + return p; +} + +expr_ty Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int col_offset, PyArena *arena) { @@ -3267,6 +3311,15 @@ ast2obj_expr(void* _o) result = PyType_GenericNew(Ellipsis_type, NULL, NULL); if (!result) goto failed; break; + case Constant_kind: + result = PyType_GenericNew(Constant_type, NULL, NULL); + if (!result) goto failed; + value = ast2obj_constant(o->v.Constant.value); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) + goto failed; + Py_DECREF(value); + break; case Attribute_kind: result = PyType_GenericNew(Attribute_type, NULL, NULL); if (!result) goto failed; @@ -6240,6 +6293,28 @@ obj2ast_expr(PyObject* obj, expr_ty* out if (*out == NULL) goto failed; return 0; } + isinstance = PyObject_IsInstance(obj, (PyObject*)Constant_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + constant value; + + if (_PyObject_HasAttrId(obj, &PyId_value)) { + int res; + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp == NULL) goto failed; + res = obj2ast_constant(tmp, &value, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } else { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Constant"); + return 1; + } + *out = Constant(value, lineno, col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } isinstance = PyObject_IsInstance(obj, (PyObject*)Attribute_type); if (isinstance == -1) { return 1; @@ -7517,6 +7592,8 @@ PyInit__ast(void) 0) return NULL; if (PyDict_SetItemString(d, "Ellipsis", (PyObject*)Ellipsis_type) < 0) return NULL; + if (PyDict_SetItemString(d, "Constant", (PyObject*)Constant_type) < 0) + return NULL; if (PyDict_SetItemString(d, "Attribute", (PyObject*)Attribute_type) < 0) return NULL; if (PyDict_SetItemString(d, "Subscript", (PyObject*)Subscript_type) < 0) diff -r ac6d33317eda Python/ast.c --- a/Python/ast.c Mon Jan 18 12:15:08 2016 +0100 +++ b/Python/ast.c Mon Jan 18 17:50:20 2016 +0100 @@ -132,6 +132,16 @@ validate_arguments(arguments_ty args) } static int +validate_constant_primitive(PyObject *value) +{ + return (PyLong_CheckExact(value) + || PyFloat_CheckExact(value) + || !PyComplex_CheckExact(value) + || !PyUnicode_CheckExact(value) + || !PyBytes_CheckExact(value)); +} + +static int validate_expr(expr_ty exp, expr_context_ty ctx) { int check_ctx = 1; @@ -240,6 +250,29 @@ validate_expr(expr_ty exp, expr_context_ return validate_expr(exp->v.Call.func, Load) && validate_exprs(exp->v.Call.args, Load, 0) && validate_keywords(exp->v.Call.keywords); + case Constant_kind: { + PyObject *value = exp->v.Constant.value; + int valid; + if (PyTuple_CheckExact(value)) { + Py_ssize_t i, len; + len = PyTuple_GET_SIZE(value); + valid = 1; /* an empty tuple is valid */ + for (i=0; i < len; i++) { + PyObject *item = PyTuple_GET_ITEM(value, i); + valid = validate_constant_primitive(item); + if (!valid) + break; + } + } + else + valid = validate_constant_primitive(value); + + if (!valid) { + PyErr_SetString(PyExc_TypeError, "invalid type in Constant"); + return 0; + } + return 1; + } case Num_kind: { PyObject *n = exp->v.Num.n; if (!PyLong_CheckExact(n) && !PyFloat_CheckExact(n) && @@ -987,6 +1020,15 @@ set_context(struct compiling *c, expr_ty case DictComp_kind: expr_name = "dict comprehension"; break; + case Constant_kind: + { + PyObject *value = e->v.Constant.value; + if (value == Py_None || value == Py_True || value == Py_False) + expr_name = "keyword"; + else + expr_name = "literal"; + break; + } case Dict_kind: case Set_kind: case Num_kind: diff -r ac6d33317eda Python/compile.c --- a/Python/compile.c Mon Jan 18 12:15:08 2016 +0100 +++ b/Python/compile.c Mon Jan 18 17:50:20 2016 +0100 @@ -1353,7 +1353,11 @@ compiler_isdocstring(stmt_ty s) { if (s->kind != Expr_kind) return 0; - return s->v.Expr.value->kind == Str_kind; + if (s->v.Expr.value->kind == Str_kind) + return 1; + if (s->v.Expr.value->kind == Constant_kind) + return PyUnicode_CheckExact(s->v.Expr.value->v.Constant.value); + return 0; } /* Compile a sequence of statements, checking for a docstring. */ @@ -1727,8 +1731,12 @@ compiler_function(struct compiler *c, st st = (stmt_ty)asdl_seq_GET(body, 0); docstring = compiler_isdocstring(st); - if (docstring && c->c_optimize < 2) - first_const = st->v.Expr.value->v.Str.s; + if (docstring && c->c_optimize < 2) { + if (st->v.Expr.value->kind == Constant_kind) + first_const = st->v.Expr.value->v.Constant.value; + else + first_const = st->v.Expr.value->v.Str.s; + } if (compiler_add_o(c, c->u->u_consts, first_const) < 0) { compiler_exit_scope(c); return 0; @@ -2712,8 +2720,13 @@ compiler_visit_stmt(struct compiler *c, VISIT(c, expr, s->v.Expr.value); ADDOP(c, PRINT_EXPR); } - else if (s->v.Expr.value->kind != Str_kind && - s->v.Expr.value->kind != Num_kind) { + else if (!(s->v.Expr.value->kind == Str_kind || + s->v.Expr.value->kind == Num_kind || + (s->v.Expr.value->kind == Constant_kind + && (PyUnicode_CheckExact(s->v.Expr.value->v.Constant.value) + || PyLong_CheckExact(s->v.Expr.value->v.Constant.value) + || PyFloat_CheckExact(s->v.Expr.value->v.Constant.value) + || PyComplex_CheckExact(s->v.Expr.value->v.Constant.value))))) { VISIT(c, expr, s->v.Expr.value); ADDOP(c, POP_TOP); } @@ -3664,6 +3677,8 @@ expr_constant(struct compiler *c, expr_t switch (e->kind) { case Ellipsis_kind: return 1; + case Constant_kind: + return PyObject_IsTrue(e->v.Constant.value); case Num_kind: return PyObject_IsTrue(e->v.Num.n); case Str_kind: @@ -3951,6 +3966,9 @@ compiler_visit_expr(struct compiler *c, return compiler_compare(c, e); case Call_kind: return compiler_call(c, e); + case Constant_kind: + ADDOP_O(c, LOAD_CONST, e->v.Constant.value, consts); + break; case Num_kind: ADDOP_O(c, LOAD_CONST, e->v.Num.n, consts); break; diff -r ac6d33317eda Python/future.c --- a/Python/future.c Mon Jan 18 12:15:08 2016 +0100 +++ b/Python/future.c Mon Jan 18 17:50:20 2016 +0100 @@ -79,7 +79,10 @@ future_parse(PyFutureFeatures *ff, mod_t i = 0; first = (stmt_ty)asdl_seq_GET(mod->v.Module.body, i); - if (first->kind == Expr_kind && first->v.Expr.value->kind == Str_kind) + if (first->kind == Expr_kind + && (first->v.Expr.value->kind == Str_kind + || (first->v.Expr.value->kind == Constant_kind + && PyUnicode_CheckExact(first->v.Expr.value->v.Constant.value)))) i++; diff -r ac6d33317eda Python/symtable.c --- a/Python/symtable.c Mon Jan 18 12:15:08 2016 +0100 +++ b/Python/symtable.c Mon Jan 18 17:50:20 2016 +0100 @@ -1455,6 +1455,7 @@ symtable_visit_expr(struct symtable *st, case JoinedStr_kind: VISIT_SEQ(st, expr, e->v.JoinedStr.values); break; + case Constant_kind: case Num_kind: case Str_kind: case Bytes_kind: diff -r ac6d33317eda Tools/parser/unparse.py --- a/Tools/parser/unparse.py Mon Jan 18 12:15:08 2016 +0100 +++ b/Tools/parser/unparse.py Mon Jan 18 17:50:20 2016 +0100 @@ -343,6 +343,11 @@ class Unparser: value = t.s.replace("{", "{{").replace("}", "}}") write(value) + def _fstring_Constant(self, t, write): + assert isinstance(t.value, str) + value = t.value.replace("{", "{{").replace("}", "}}") + write(value) + def _fstring_FormattedValue(self, t, write): write("{") expr = io.StringIO() @@ -364,6 +369,25 @@ class Unparser: def _Name(self, t): self.write(t.id) + def _write_constant(self, value): + if isinstance(value, (float, complex)): + self.write(repr(value).replace("inf", INFSTR)) + else: + self.write(repr(value)) + + def _Constant(self, t): + value = t.value + if isinstance(value, tuple): + self.write("(") + if len(value) == 1: + self._write_constant(value[0]) + self.write(",") + else: + interleave(lambda: self.write(", "), self._write_constant, value) + self.write(")") + else: + self._write_constant(t.value) + def _NameConstant(self, t): self.write(repr(t.value)) @@ -443,7 +467,7 @@ class Unparser: def _Tuple(self, t): self.write("(") if len(t.elts) == 1: - (elt,) = t.elts + elt = t.elts[0] self.dispatch(elt) self.write(",") else: @@ -490,7 +514,9 @@ class Unparser: # Special case: 3.__abs__() is a syntax error, so if t.value # is an integer literal then we need to either parenthesize # it or add an extra space to get 3 .__abs__(). - if isinstance(t.value, ast.Num) and isinstance(t.value.n, int): + if ((isinstance(t.value, ast.Num) and isinstance(t.value.n, int)) + or (isinstance(t.value, ast.Constant) + and isinstance(t.value.value, int))): self.write(" ") self.write(".") self.write(t.attr)