cpython: 6b7704fe1be1 (original) (raw)

Mercurial > cpython

changeset 75100:6b7704fe1be1 2.6

- Issue #13703: oCERT-2011-003: add -R command-line option and PYTHONHASHSEED environment variable, to provide an opt-in way to protect against denial of service attacks due to hash collisions within the dict and set types. Patch by David Malcolm, based on work by Victor Stinner. [#13703]

Barry Warsaw barry@python.org
date Mon, 20 Feb 2012 20:42:21 -0500
parents 90ec0bc01f3b
children 19e6e55f09f3
files Doc/library/sys.rst Doc/reference/datamodel.rst Doc/using/cmdline.rst Include/object.h Include/pydebug.h Include/pythonrun.h Lib/os.py Lib/test/test_cmd_line.py Lib/test/test_hash.py Lib/test/test_os.py Lib/test/test_set.py Lib/test/test_support.py Lib/test/test_symtable.py Lib/test/test_sys.py Makefile.pre.in Misc/NEWS Misc/python.man Modules/main.c Modules/posixmodule.c Objects/bufferobject.c Objects/object.c Objects/stringobject.c Objects/unicodeobject.c PCbuild/pythoncore.vcproj Python/pythonrun.c Python/random.c Python/sysmodule.c
diffstat 27 files changed, 711 insertions(+), 157 deletions(-)[+] [-] Doc/library/sys.rst 5 Doc/reference/datamodel.rst 2 Doc/using/cmdline.rst 46 Include/object.h 6 Include/pydebug.h 1 Include/pythonrun.h 2 Lib/os.py 19 Lib/test/test_cmd_line.py 14 Lib/test/test_hash.py 100 Lib/test/test_os.py 54 Lib/test/test_set.py 52 Lib/test/test_support.py 12 Lib/test/test_symtable.py 7 Lib/test/test_sys.py 2 Makefile.pre.in 3 Misc/NEWS 5 Misc/python.man 29 Modules/main.c 16 Modules/posixmodule.c 135 Objects/bufferobject.c 12 Objects/object.c 2 Objects/stringobject.c 12 Objects/unicodeobject.c 12 PCbuild/pythoncore.vcproj 4 Python/pythonrun.c 8 Python/random.c 302 Python/sysmodule.c 6

line wrap: on

line diff

--- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -289,6 +289,11 @@ always available. +------------------------------+------------------------------------------+ | :const:bytes_warning | -b | +------------------------------+------------------------------------------+

--- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1273,6 +1273,8 @@ Basic customization modules are still available at the time when the :meth:__del__ method is called.

.. method:: object.repr(self)

--- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -21,7 +21,7 @@ Command line When invoking Python, you may specify any of these options::

The most common use case is, of course, a simple invocation of a script:: @@ -239,6 +239,29 @@ Miscellaneous options :pep:238 -- Changing the division operator +.. cmdoption:: -R +

+ .. cmdoption:: -s Don't add user site directory to sys.path @@ -501,6 +524,27 @@ These environment variables influence Py .. versionadded:: 2.6 +.. envvar:: PYTHONHASHSEED +

+ .. envvar:: PYTHONIOENCODING Overrides the encoding used for stdin/stdout/stderr, in the syntax

--- a/Include/object.h +++ b/Include/object.h @@ -506,6 +506,12 @@ PyAPI_FUNC(void) Py_ReprLeave(PyObject PyAPI_FUNC(long) _Py_HashDouble(double); PyAPI_FUNC(long) _Py_HashPointer(void); +typedef struct {

+} _Py_HashSecret_t; +PyAPI_DATA(_Py_HashSecret_t) _Py_HashSecret; + /* Helper for passing objects to printf and the like */ #define PyObject_REPR(obj) PyString_AS_STRING(PyObject_Repr(obj))

--- a/Include/pydebug.h +++ b/Include/pydebug.h @@ -26,6 +26,7 @@ PyAPI_DATA(int) Py_NoUserSiteDirectory; PyAPI_DATA(int) _Py_QnewFlag; /* Warn about 3.x issues / PyAPI_DATA(int) Py_Py3kWarningFlag; +PyAPI_DATA(int) Py_HashRandomizationFlag; / this is a wrapper around getenv() that pays attention to Py_IgnoreEnvironmentFlag. It should be used for getting variables like

--- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -168,6 +168,8 @@ typedef void (PyOS_sighandler_t)(int); PyAPI_FUNC(PyOS_sighandler_t) PyOS_getsig(int); PyAPI_FUNC(PyOS_sighandler_t) PyOS_setsig(int, PyOS_sighandler_t); +/ Random */ +PyAPI_FUNC(int) _PyOS_URandom (void *buffer, Py_ssize_t size); #ifdef __cplusplus }

--- a/Lib/os.py +++ b/Lib/os.py @@ -742,22 +742,3 @@ try: _make_statvfs_result) except NameError: # statvfs_result may not exist pass - -if not _exists("urandom"):

-

-

--- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -103,6 +103,20 @@ class CmdLineTest(unittest.TestCase): self.exit_code('-c', 'pass'), 0)

+

def test_main(): test.test_support.run_unittest(CmdLineTest)

--- a/Lib/test/test_hash.py +++ b/Lib/test/test_hash.py @@ -3,10 +3,18 @@ #

Also test that hash implementations are inherited as expected

+import os +import sys +import struct +import datetime import unittest +import subprocess + from test import test_support from collections import Hashable +IS_64BIT = (struct.calcsize('l') == 8) + class HashEqualityTestCase(unittest.TestCase): @@ -133,10 +141,100 @@ class HashBuiltinsTestCase(unittest.Test for obj in self.hashes_to_check: self.assertEqual(hash(obj), _default_hash(obj)) +class HashRandomizationTests(unittest.TestCase): +

+

+

+

+ +class StringlikeHashRandomizationTests(HashRandomizationTests):

+

+

+

+ +class StrHashRandomizationTests(StringlikeHashRandomizationTests):

+

+ +class UnicodeHashRandomizationTests(StringlikeHashRandomizationTests):

+

+ +class BufferHashRandomizationTests(StringlikeHashRandomizationTests):

+

+ +class DatetimeTests(HashRandomizationTests):

+ +class DatetimeDateTests(DatetimeTests):

+ +class DatetimeDatetimeTests(DatetimeTests):

+ +class DatetimeTimeTests(DatetimeTests):

+ + def test_main(): test_support.run_unittest(HashEqualityTestCase, HashInheritanceTestCase,

if name == "main":

--- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -6,6 +6,8 @@ import os import unittest import warnings import sys +import subprocess + from test import test_support warnings.filterwarnings("ignore", "tempnam", RuntimeWarning, name) @@ -499,18 +501,46 @@ class DevNullTests (unittest.TestCase): class URandomTests (unittest.TestCase): def test_urandom(self):

+

+

+

+

class Win32ErrorTests(unittest.TestCase): def test_rename(self):

--- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -6,7 +6,6 @@ import weakref import operator import copy import pickle -import os from random import randrange, shuffle import sys import collections @@ -688,6 +687,17 @@ class TestBasicOps(unittest.TestCase): if self.repr is not None: self.assertEqual(repr(self.set), self.repr)

+

+ def test_print(self): fo = open(test_support.TESTFN, "wb") try: @@ -837,6 +847,46 @@ class TestBasicOpsTriple(TestBasicOps): self.length = 3 self.repr = None +#------------------------------------------------------------------------------ + +class TestBasicOpsString(TestBasicOps):

+

+ +#------------------------------------------------------------------------------ + +class TestBasicOpsUnicode(TestBasicOps):

+

+ +#------------------------------------------------------------------------------ + +class TestBasicOpsMixedStringUnicode(TestBasicOps):

+

+ #============================================================================== def baditer():

--- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -24,7 +24,7 @@ import re "captured_stdout", "TransientResource", "transient_internet", "run_with_locale", "set_memlimit", "bigmemtest", "bigaddrspacetest", "BasicTestRunner", "run_unittest", "run_doctest", "threading_setup",

class Error(Exception): """Base class for regression test exceptions.""" @@ -893,3 +893,13 @@ def reap_children(): break except: break + +def strip_python_stderr(stderr):

+

--- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -105,10 +105,11 @@ class SymtableTest(unittest.TestCase): def test_function_info(self): func = self.spam

def test_globals(self):

--- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -384,7 +384,7 @@ class SysModuleTest(unittest.TestCase): attrs = ("debug", "py3k_warning", "division_warning", "division_new", "inspect", "interactive", "optimize", "dont_write_bytecode", "no_site", "ignore_environment", "tabcheck", "verbose",

--- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -280,6 +280,7 @@ PYTHON_OBJS= [](#l15.3) Python/pymath.o [](#l15.4) Python/pystate.o [](#l15.5) Python/pythonrun.o [](#l15.6)

@@ -708,7 +709,7 @@ buildbottest: all platform -@if which pybuildbot.identify >/dev/null 2>&1; then [](#l15.12) pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; [](#l15.13) fi - (TESTPYTHON)(TESTPYTHON) (TESTPYTHON)(TESTPROG) -uall -rw $(TESTOPTS) + (TESTPYTHON)−R(TESTPYTHON) -R (TESTPYTHON)R(TESTPROG) -uall -rw $(TESTOPTS) QUICKTESTOPTS= $(TESTOPTS) -x test_thread test_signal test_strftime [](#l15.18) test_unicodedata test_re test_sre test_select test_poll [](#l15.19)

--- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,11 @@ What's New in Python 2.6.8 rc 1? Core and Builtins ----------------- +- Issue #13703: oCERT-2011-003: add -R command-line option and PYTHONHASHSEED

--- a/Misc/python.man +++ b/Misc/python.man @@ -34,6 +34,9 @@ python - an interpreted, interactive, o .B -O0 ] [ +.B -R +] +[ .B -Q .I argument ] @@ -151,6 +154,18 @@ to \fI.pyo\fP. Given twice, causes docs .B -O0 Discard docstrings in addition to the \fB-O\fP optimizations. .TP +.B -R +Turn on "hash randomization", so that the hash() values of str, bytes and +datetime objects are "salted" with an unpredictable pseudo-random value. +Although they remain constant within an individual Python process, they are +not predictable between repeated invocations of Python. +.IP +This is intended to provide protection against a denial of service +caused by carefully-chosen inputs that exploit the worst case performance +of a dict insertion, O(n^2) complexity. See +http://www.ocert.org/advisories/ocert-2011-003.html[](#l17.26) +for details. +.TP .BI "-Q " argument Division control; see PEP 238. The argument must be one of "old" (the default, int/int and long/long return an int or long), "new" (new @@ -411,6 +426,20 @@ the \fB-u\fP option. If this is set to a non-empty string it is equivalent to specifying the \fB-v\fP option. If set to an integer, it is equivalent to specifying \fB-v\fP multiple times. +.IP PYTHONHASHSEED +If this variable is set to "random", the effect is the same as specifying +the \fB-R\fP option: a random value is used to seed the hashes of str, +bytes and datetime objects. + +If PYTHONHASHSEED is set to an integer value, it is used as a fixed seed for +generating the hash() of the types covered by the hash randomization. Its +purpose is to allow repeatable hashing, such as for selftests for the +interpreter itself, or to allow a cluster of python processes to share hash +values. + +The integer must be a decimal number in the range [0,4294967295]. Specifying +the value 0 will lead to the same hash values as when hash randomization is +disabled. .SH AUTHOR The Python Software Foundation: http://www.python.org/psf[](#l17.51) .SH INTERNET RESOURCES

--- a/Modules/main.c +++ b/Modules/main.c @@ -40,7 +40,7 @@ static char *orig_argv; static int orig_argc; / command line options */ -#define BASE_OPTS "3bBc:dEhiJm:OQ:sStuUvVW:xX?" +#define BASE_OPTS "3bBc:dEhiJm:OQ:RsStuUvVW:xX?" #ifndef RISCOS #define PROGRAM_OPTS BASE_OPTS @@ -71,6 +71,9 @@ static char *usage_2 = "[](#l18.12) -m mod : run library module as a script (terminates option list)\n[](#l18.13) -O : optimize generated bytecode slightly; also PYTHONOPTIMIZE=x\n[](#l18.14) -OO : remove doc-strings in addition to the -O optimizations\n[](#l18.15) +-R : use a pseudo-random salt to make hash() values of various types be\n[](#l18.16)

-Q arg : division options: -Qold (default), -Qwarn, -Qwarnall, -Qnew\n[](#l18.19) -s : don't add user site directory to sys.path; also PYTHONNOUSERSITE\n[](#l18.20) -S : don't imply 'import site' on initialization\n[](#l18.21) @@ -101,6 +104,12 @@ PYTHONHOME : alternate direct PYTHONCASEOK : ignore case in 'import' statements (Windows).\n[](#l18.23) PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n[](#l18.24) "; +static char *usage_6 = "[](#l18.26) +PYTHONHASHSEED: if this variable is set to random, the effect is the same \n[](#l18.27)

static int @@ -117,6 +126,7 @@ usage(int exitcode, char* program) fputs(usage_3, f); fprintf(f, usage_4, DELIM); fprintf(f, usage_5, DELIM, PYTHONHOMEHELP);

#if defined(__VMS) if (exitcode == 0) { @@ -388,6 +398,10 @@ Py_Main(int argc, char **argv) PySys_AddWarnOption(_PyOS_optarg); break;

+ /* This space reserved for other options */ default:

--- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8371,117 +8371,35 @@ posix_getloadavg(PyObject *self, PyObjec } #endif -#ifdef MS_WINDOWS - -PyDoc_STRVAR(win32_urandom__doc__, +PyDoc_STRVAR(posix_urandom__doc__, "urandom(n) -> str\n\n[](#l19.11) -Return a string of n random bytes suitable for cryptographic use."); - -typedef BOOL (WINAPI *CRYPTACQUIRECONTEXTA)(HCRYPTPROV *phProv,[](#l19.14)

-typedef BOOL (WINAPI *CRYPTGENRANDOM)(HCRYPTPROV hProv, DWORD dwLen,[](#l19.17)

- -static CRYPTGENRANDOM pCryptGenRandom = NULL; -/* This handle is never explicitly released. Instead, the operating

-static PyObject* -win32_urandom(PyObject *self, PyObject *args) -{

-

+Return n random bytes suitable for cryptographic use."); + +static PyObject * +posix_urandom(PyObject *self, PyObject *args) +{

+

-

-

-

-

-

-

+

-

-

-} -#endif static PyMethodDef posix_methods[] = { {"access", posix_access, METH_VARARGS, posix_access__doc__}, @@ -8787,12 +8705,7 @@ static PyMethodDef posix_methods[] = { #ifdef HAVE_GETLOADAVG {"getloadavg", posix_getloadavg, METH_NOARGS, posix_getloadavg__doc__}, #endif

--- a/Objects/bufferobject.c +++ b/Objects/bufferobject.c @@ -334,10 +334,20 @@ buffer_hash(PyBufferObject *self) return -1; p = (unsigned char *) ptr; len = size;

--- a/Objects/object.c +++ b/Objects/object.c @@ -1101,6 +1101,8 @@ PyObject_HashNotImplemented(PyObject *se return -1; } +_Py_HashSecret_t _Py_HashSecret; + long PyObject_Hash(PyObject *v) {

--- a/Objects/stringobject.c +++ b/Objects/stringobject.c @@ -1212,11 +1212,21 @@ string_hash(PyStringObject *a) if (a->ob_shash != -1) return a->ob_shash; len = Py_SIZE(a);

--- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6695,11 +6695,21 @@ unicode_hash(PyUnicodeObject *self) if (self->hash != -1) return self->hash; len = PyUnicode_GET_SIZE(self);

--- a/PCbuild/pythoncore.vcproj +++ b/PCbuild/pythoncore.vcproj @@ -1779,6 +1779,10 @@ > <File + RelativePath="..\Python\random.c" + > + + <File RelativePath="..\Python\structmember.c" >

--- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -67,6 +67,7 @@ static void call_sys_exitfunc(void); static void call_ll_exitfuncs(void); extern void _PyUnicode_Init(void); extern void _PyUnicode_Fini(void); +extern void _PyRandom_Init(void); #ifdef WITH_THREAD extern void _PyGILState_Init(PyInterpreterState *, PyThreadState ); @@ -89,6 +90,7 @@ int Py_IgnoreEnvironmentFlag; / e.g. PY true divisions (which they will be in 2.3). / int _Py_QnewFlag = 0; int Py_NoUserSiteDirectory = 0; / for -s and site.py / +int Py_HashRandomizationFlag = 0; / for -R and PYTHONHASHSEED / / PyModule_GetWarningsModule is no longer necessary as of 2.6 since _warnings is builtin. This API should not be used. */ @@ -166,6 +168,12 @@ Py_InitializeEx(int install_sigs) Py_OptimizeFlag = add_flag(Py_OptimizeFlag, p); if ((p = Py_GETENV("PYTHONDONTWRITEBYTECODE")) && *p != '\0') Py_DontWriteBytecodeFlag = add_flag(Py_DontWriteBytecodeFlag, p);

+

interp = PyInterpreterState_New(); if (interp == NULL)

new file mode 100644 --- /dev/null +++ b/Python/random.c @@ -0,0 +1,302 @@ +#include "Python.h" +#ifdef MS_WINDOWS +#include <windows.h> +#else +#include <fcntl.h> +#endif + +static int random_initialized = 0; + +#ifdef MS_WINDOWS +typedef BOOL (WINAPI *CRYPTACQUIRECONTEXTA)(HCRYPTPROV *phProv,[](#l26.15)

+typedef BOOL (WINAPI *CRYPTGENRANDOM)(HCRYPTPROV hProv, DWORD dwLen,[](#l26.18)

+ +static CRYPTGENRANDOM pCryptGenRandom = NULL; +/* This handle is never explicitly released. Instead, the operating

+static int +win32_urandom_init(int raise) +{

+

+

+

+

+

+ +error:

+} + +/* Fill buffer with size pseudo-random bytes generated by the Windows CryptoGen

+

+

+} +#endif /* MS_WINDOWS / + + +#ifdef __VMS +/ Use openssl random routine */ +#include <openssl/rand.h> +static int +vms_urandom(unsigned char *buffer, Py_ssize_t size, int raise) +{

+} +#endif /* __VMS / + + +#if !defined(MS_WINDOWS) && !defined(__VMS) + +/ Read size bytes from /dev/urandom into buffer.

+

+

+

+} + +/* Read size bytes from /dev/urandom into buffer.

+

+

+

+

+} +#endif /* !defined(MS_WINDOWS) && !defined(__VMS) / + +/ Fill buffer with pseudo-random bytes generated by a linear congruent

+

+

+} + +/* Fill buffer with size pseudo-random bytes, not suitable for cryptographic

+ +#ifdef MS_WINDOWS

+#else +# ifdef __VMS

+# else

+# endif +#endif +} + +void +_PyRandom_Init(void) +{

+

+

+

+

+#ifdef MS_WINDOWS

+#else /* #ifdef MS_WINDOWS */ +# ifdef __VMS

+# else

+# endif +#endif

+}

--- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1224,6 +1224,7 @@ static PyStructSequence_Field flags_fiel {"unicode", "-U"}, /* {"skip_first", "-x"}, */ {"bytes_warning", "-b"},

#ifdef RISCOS

#else

#endif }; @@ -1271,6 +1272,7 @@ make_flags(void) SetFlag(Py_UnicodeFlag); /* SetFlag(skipfirstline); */ SetFlag(Py_BytesWarningFlag);

#undef SetFlag if (PyErr_Occurred()) {