cpython: bc06f67234d0 (original) (raw)

Mercurial > cpython

changeset 89836:bc06f67234d0

Issue #19977: When the ``LC_TYPE`` locale is the POSIX locale (``C`` locale), :py:data:`sys.stdin` and :py:data:`sys.stdout` are now using the ``surrogateescape`` error handler, instead of the ``strict`` error handler. [#19977]

Victor Stinner victor.stinner@gmail.com
date	Tue, 18 Mar 2014 01🔞21 +0100
parents	daa6bf71170f
children	f6794a0fb2b3
files	Doc/whatsnew/3.5.rst Lib/test/test_sys.py Misc/NEWS Python/pythonrun.c
diffstat	4 files changed, 62 insertions(+), 2 deletions(-)[+] [-] Doc/whatsnew/3.5.rst 5 Lib/test/test_sys.py 44 Misc/NEWS 4 Python/pythonrun.c 11

line wrap: on

line diff

--- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -79,7 +79,10 @@ New built-in features: Implementation improvements: -* None yet. +* When the LC_TYPE locale is the POSIX locale (C locale),

:py:data:sys.stdin and :py:data:sys.stdout are now using the
surrogateescape error handler, instead of the strict error handler
(:issue:19977). Significantly Improved Library Modules:

--- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -615,6 +615,50 @@ class SysModuleTest(unittest.TestCase): expected = None self.check_fsencoding(fs_encoding, expected)

@unittest.skipIf(sys.platform == 'win32',

                'test specific to UNIX')[](#l2.8)

def test_c_locale_surrogateescape(self):
```
   # Force the POSIX locale[](#l2.10)
```
```
   env = os.environ.copy()[](#l2.11)
```
```
   env["LC_ALL"] = "C"[](#l2.12)
```
```
   code = '\n'.join(([](#l2.13)
```
```
       'import codecs, sys',[](#l2.14)
```
```
       'def dump(name):',[](#l2.15)
```

       '    std = getattr(sys, name)',[](#l2.16)

       '    encoding = codecs.lookup(std.encoding).name',[](#l2.17)

       '    print("%s: %s:%s" % (name, encoding, std.errors))',[](#l2.18)

```
       'dump("stdin")',[](#l2.19)
```
```
       'dump("stdout")',[](#l2.20)
```
```
       'dump("stderr")',[](#l2.21)
```
```
   ))[](#l2.22)
```

   p = subprocess.Popen([sys.executable, "-I", "-c", code],[](#l2.23)

                         stdout=subprocess.PIPE, env=env)[](#l2.24)

```
   out = p.communicate()[0][](#l2.25)
```
```
   self.assertEqual(out,[](#l2.26)
```

                    b'stdin: ascii:surrogateescape\n'[](#l2.27)

                    b'stdout: ascii:surrogateescape\n'[](#l2.28)

                    b'stderr: ascii:backslashreplace\n')[](#l2.29)

   # replace the default error handler[](#l2.31)

   env['PYTHONIOENCODING'] = ':strict'[](#l2.32)

   p = subprocess.Popen([sys.executable, "-c", code],[](#l2.33)

                         stdout=subprocess.PIPE, env=env)[](#l2.34)

```
   out = p.communicate()[0][](#l2.35)
```
```
   self.assertEqual(out,[](#l2.36)
```

                    b'stdin: ascii:strict\n'[](#l2.37)

                    b'stdout: ascii:strict\n'[](#l2.38)

                    b'stderr: ascii:backslashreplace\n')[](#l2.39)

```
   # force the encoding[](#l2.41)
```

   env['PYTHONIOENCODING'] = 'iso8859-1'[](#l2.42)

   p = subprocess.Popen([sys.executable, "-c", code],[](#l2.43)

                         stdout=subprocess.PIPE, env=env)[](#l2.44)

```
   out = p.communicate()[0][](#l2.45)
```
```
   self.assertEqual(out,[](#l2.46)
```

                    b'stdin: iso8859-1:surrogateescape\n'[](#l2.47)

                    b'stdout: iso8859-1:surrogateescape\n'[](#l2.48)

                    b'stderr: iso8859-1:backslashreplace\n')[](#l2.49)

+ def test_implementation(self): # This test applies to all implementations equally.

--- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,10 @@ Core and Builtins Library ------- +- Issue #19977: When the LC_TYPE locale is the POSIX locale (C locale),

:py:data:sys.stdin and :py:data:sys.stdout are now using the
surrogateescape error handler, instead of the strict error handler. +

Issue #20574: Implement incremental decoder for cp65001 code (Windows code page 65001, Microsoft UTF-8).

--- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1156,6 +1156,15 @@ initstdio(void) encoding = _Py_StandardStreamEncoding; errors = _Py_StandardStreamErrors; if (!encoding || !errors) {

```
   if (!errors) {[](#l4.7)
```

       /* When the LC_CTYPE locale is the POSIX locale ("C locale"),[](#l4.8)

          stdin and stdout use the surrogateescape error handler by[](#l4.9)

          default, instead of the strict error handler. */[](#l4.10)

       char *loc = setlocale(LC_CTYPE, NULL);[](#l4.11)

       if (loc != NULL && strcmp(loc, "C") == 0)[](#l4.12)

           errors = "surrogateescape";[](#l4.13)

```
   }[](#l4.14)
```

+ pythonioencoding = Py_GETENV("PYTHONIOENCODING"); if (pythonioencoding) { char *err; @@ -1168,7 +1177,7 @@ initstdio(void) if (err) { *err = '\0'; err++;

           if (*err && !errors) {[](#l4.23)

           if (*err && !_Py_StandardStreamErrors) {[](#l4.24)
               errors = err;[](#l4.25)
           }[](#l4.26)
       }[](#l4.27)