cpython: 013a1cd673fc (original) (raw)

Mercurial > cpython

changeset 68499:013a1cd673fc 3.2

#11554: reactivate test_email_codecs, and make it pass. The fix is to charset.py, which was not doing the encoding to the correct output character set when doing a body_encode for either the shift-jis or euc-jp charsets. There's also a fix for handling a bytes input in encoders.py. Patch by Michael Henry, comment changes by me. [#11554]

R David Murray rdmurray@bitdance.com
date	Tue, 15 Mar 2011 12:20:02 -0400
parents	1579808ef25b
children	a6390ebff835 983ed0ba44d0 f3ae657067ef
files	Lib/email/charset.py Lib/email/encoders.py Lib/email/test/test_email.py Lib/email/test/test_email_codecs.py Lib/test/test_email.py Misc/ACKS Misc/NEWS
diffstat	7 files changed, 54 insertions(+), 22 deletions(-)[+] [-] Lib/email/charset.py 9 Lib/email/encoders.py 9 Lib/email/test/test_email.py 4 Lib/email/test/test_email_codecs.py 46 Lib/test/test_email.py 2 Misc/ACKS 1 Misc/NEWS 5

line wrap: on

line diff

--- a/Lib/email/charset.py +++ b/Lib/email/charset.py @@ -263,7 +263,7 @@ class Charset: Returns "quoted-printable" if self.body_encoding is QP. Returns "base64" if self.body_encoding is BASE64.

```
   Returns "7bit" otherwise.[](#l1.7)
```

   Returns conversion function otherwise.[](#l1.8)
   """[](#l1.9)
   assert self.body_encoding != SHORTEST[](#l1.10)
   if self.body_encoding == QP:[](#l1.11)

@@ -381,7 +381,10 @@ class Charset: """Body-encode a string by converting it first to bytes. The type of encoding (base64 or quoted-printable) will be based on

```
   self.body_encoding.[](#l1.16)
```

   self.body_encoding.  If body_encoding is None, we assume the[](#l1.17)

   output charset is a 7bit encoding, so re-encoding the decoded[](#l1.18)

   string using the ascii codec produces the correct string version[](#l1.19)

   of the content.[](#l1.20)
   """[](#l1.21)
   # 7bit/8bit encodings return the string unchanged (module conversions)[](#l1.22)
   if self.body_encoding is BASE64:[](#l1.23)

@@ -391,4 +394,6 @@ class Charset: elif self.body_encoding is QP: return email.quoprimime.body_encode(string) else:

       if isinstance(string, str):[](#l1.28)

           string = string.encode(self.output_charset).decode('ascii')[](#l1.29)
       return string[](#l1.30)

--- a/Lib/email/encoders.py +++ b/Lib/email/encoders.py @@ -54,10 +54,13 @@ def encode_7or8bit(msg): # There's no payload. For backwards compatibility we use 7bit msg['Content-Transfer-Encoding'] = '7bit' return

We play a trick to make this go fast. If encoding to ASCII succeeds, we
know the data must be 7bit, otherwise treat it as 8bit.

We play a trick to make this go fast. If encoding/decode to ASCII
succeeds, we know the data must be 7bit, otherwise treat it as 8bit.
try:

```
   orig.encode('ascii')[](#l2.12)
```

```
   if isinstance(orig, str):[](#l2.13)
```
```
       orig.encode('ascii')[](#l2.14)
```
```
   else:[](#l2.15)
```
```
       orig.decode('ascii')[](#l2.16)
```
except UnicodeError: # iso-2022-* is non-ASCII but still 7-bit charset = msg.get_charset()

--- a/Lib/email/test/test_email.py +++ b/Lib/email/test/test_email.py @@ -3365,9 +3365,9 @@ class TestCharset(unittest.TestCase): # built-in encodings where the header encoding is QP but the body # encoding is not. from email import charset as CharsetModule

   CharsetModule.add_charset('fake', CharsetModule.QP, None)[](#l3.7)

   CharsetModule.add_charset('fake', CharsetModule.QP, None, 'utf-8')[](#l3.8)
   c = Charset('fake')[](#l3.9)

   eq('hello w\xf6rld', c.body_encode('hello w\xf6rld'))[](#l3.10)

   eq('hello world', c.body_encode('hello world'))[](#l3.11)

def test_unicode_charset_name(self): charset = Charset('us-ascii')

--- a/Lib/email/test/test_email_codecs.py +++ b/Lib/email/test/test_email_codecs.py @@ -13,7 +13,7 @@ from email.message import Message

We're compatible with Python 2.3, but it doesn't have the built-in Asian

codecs, so we have to skip all these tests.

try:

str('foo', 'euc-jp')

str(b'foo', 'euc-jp')

except LookupError: raise unittest.SkipTest @@ -22,11 +22,14 @@ except LookupError: class TestEmailAsianCodecs(TestEmailBase): def test_japanese_codecs(self): eq = self.ndiffAssertEqual

```
   j = Charset("euc-jp")[](#l4.16)
```
```
   g = Charset("iso-8859-1")[](#l4.17)
```

```
   jcode = "euc-jp"[](#l4.18)
```
```
   gcode = "iso-8859-1"[](#l4.19)
```
```
   j = Charset(jcode)[](#l4.20)
```

   g = Charset(gcode)[](#l4.21)
   h = Header("Hello World!")[](#l4.22)

   jhello = '\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc\xa5\xeb\xa5\xc9\xa1\xaa'[](#l4.23)

   ghello = 'Gr\xfc\xdf Gott!'[](#l4.24)

   jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc'[](#l4.25)

                b'\xa5\xeb\xa5\xc9\xa1\xaa', jcode)[](#l4.26)

   ghello = str(b'Gr\xfc\xdf Gott!', gcode)[](#l4.27)
   h.append(jhello, j)[](#l4.28)
   h.append(ghello, g)[](#l4.29)
   # BAW: This used to -- and maybe should -- fold the two iso-8859-1[](#l4.30)

@@ -36,13 +39,17 @@ class TestEmailAsianCodecs(TestEmailBase # encoded word. eq(h.encode(), """[](#l4.33) Hello World! =?iso-2022-jp?b?GyRCJU8lbSE8JW8hPCVrJUkhKhsoQg==?=

=?iso-8859-1?q?Gr=FC=DF?= =?iso-8859-1?q?_Gott!?=""")

=?iso-8859-1?q?Gr=FC=DF_Gott!?=""") eq(decode_header(h.encode()),

      [('Hello World!', None),[](#l4.38)

       ('\x1b$B%O%m!<%o!<%k%I!*\x1b(B', 'iso-2022-jp'),[](#l4.39)

       ('Gr\xfc\xdf Gott!', 'iso-8859-1')])[](#l4.40)

   int = 'test-ja \xa4\xd8\xc5\xea\xb9\xc6\xa4\xb5\xa4\xec\xa4\xbf\xa5\xe1\xa1\xbc\xa5\xeb\xa4\xcf\xbb\xca\xb2\xf1\xbc\xd4\xa4\xce\xbe\xb5\xc7\xa7\xa4\xf2\xc2\xd4\xa4\xc3\xa4\xc6\xa4\xa4\xa4\xde\xa4\xb9'[](#l4.41)

   h = Header(int, j, header_name="Subject")[](#l4.42)

      [(b'Hello World!', None),[](#l4.43)

       (b'\x1b$B%O%m!<%o!<%k%I!*\x1b(B', 'iso-2022-jp'),[](#l4.44)

       (b'Gr\xfc\xdf Gott!', gcode)])[](#l4.45)

   subject_bytes = (b'test-ja \xa4\xd8\xc5\xea\xb9\xc6\xa4\xb5'[](#l4.46)

       b'\xa4\xec\xa4\xbf\xa5\xe1\xa1\xbc\xa5\xeb\xa4\xcf\xbb\xca\xb2'[](#l4.47)

       b'\xf1\xbc\xd4\xa4\xce\xbe\xb5\xc7\xa7\xa4\xf2\xc2\xd4\xa4\xc3'[](#l4.48)

       b'\xa4\xc6\xa4\xa4\xa4\xde\xa4\xb9')[](#l4.49)

   subject = str(subject_bytes, jcode)[](#l4.50)

   h = Header(subject, j, header_name="Subject")[](#l4.51)
   # test a very long header[](#l4.52)
   enc = h.encode()[](#l4.53)
   # TK: splitting point may differ by codec design and/or Header encoding[](#l4.54)

@@ -50,15 +57,24 @@ Hello World! =?iso-2022-jp?b?GyRCJU8lbSE =?iso-2022-jp?b?dGVzdC1qYSAbJEIkWEVqOUYkNSRsJD8lYSE8JWskTztKGyhC?= =?iso-2022-jp?b?GyRCMnE8VCROPjVHJyRyQlQkQyRGJCQkXiQ5GyhC?=""") # TK: full decode comparison

   eq(h.__unicode__().encode('euc-jp'), int)[](#l4.59)

   eq(str(h).encode(jcode), subject_bytes)[](#l4.60)

def test_payload_encoding_utf8(self):

   jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc'[](#l4.63)

                b'\xa5\xeb\xa5\xc9\xa1\xaa', 'euc-jp')[](#l4.64)

```
   msg = Message()[](#l4.65)
```

   msg.set_payload(jhello, 'utf-8')[](#l4.66)

   ustr = msg.get_payload(decode=True).decode(msg.get_content_charset())[](#l4.67)

   self.assertEqual(jhello, ustr)[](#l4.68)

def test_payload_encoding(self):

   jhello = '\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc\xa5\xeb\xa5\xc9\xa1\xaa'[](#l4.71)
   jcode  = 'euc-jp'[](#l4.72)

   jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc'[](#l4.73)

                b'\xa5\xeb\xa5\xc9\xa1\xaa', jcode)[](#l4.74)
   msg = Message()[](#l4.75)
   msg.set_payload(jhello, jcode)[](#l4.76)

   ustr = str(msg.get_payload(), msg.get_content_charset())[](#l4.77)

   self.assertEqual(jhello, ustr.encode(jcode))[](#l4.78)

   ustr = msg.get_payload(decode=True).decode(msg.get_content_charset())[](#l4.79)

   self.assertEqual(jhello, ustr)[](#l4.80)

--- a/Lib/test/test_email.py +++ b/Lib/test/test_email.py @@ -3,10 +3,12 @@

The specific tests now live in Lib/email/test

from email.test.test_email import suite +from email.test.test_email_codecs import suite as codecs_suite from test import support def test_main(): support.run_unittest(suite())

support.run_unittest(codecs_suite())

if name == 'main': test_main()

--- a/Misc/ACKS +++ b/Misc/ACKS @@ -371,6 +371,7 @@ Kevan Heydon Jason Hildebrand Richie Hindle Konrad Hinsen +Michael Henry David Hobley Tim Hochberg Joerg-Cyril Hoehle

--- a/Misc/NEWS +++ b/Misc/NEWS @@ -34,6 +34,9 @@ Core and Builtins Library ------- +- Issue #11554: Fixed support for Japanese codecs; previously the body output

encoding was not done if euc-jp or shift-jis was specified as the charset. +

Issue #11500: Fixed a bug in the os x proxy bypass code for fully qualified IP addresses in the proxy exception list. @@ -99,6 +102,8 @@ Tools/Demos Tests ----- +- Issue #11554: Reactivated test_email_codecs. +
Issue #11490: test_subprocess:test_leaking_fds_on_error no longer gives a false positive if the last directory in the path is inaccessible.

cpython: 013a1cd673fc (original) (raw)

Mercurial > cpython

changeset 68499:013a1cd673fc 3.2

We play a trick to make this go fast. If encoding to ASCII succeeds, we

know the data must be 7bit, otherwise treat it as 8bit.

We play a trick to make this go fast. If encoding/decode to ASCII

succeeds, we know the data must be 7bit, otherwise treat it as 8bit.

We're compatible with Python 2.3, but it doesn't have the built-in Asian

codecs, so we have to skip all these tests.

The specific tests now live in Lib/email/test