cpython: d90f25e1a705 (original) (raw)

--- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -27,6 +27,10 @@ byte strings, but only using the Base64 ASCII-only Unicode strings are now accepted by the decoding functions of the modern interface. +.. versionchanged:: 3.4

• Any :term:bytes-like object\ s are now accepted by all
• encoding and decoding functions in this module. +

The modern interface provides: .. function:: b64encode(s, altchars=None)

--- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1208,36 +1208,41 @@ mappings. .. tabularcolumns:: |l|L|L| -+----------------------+---------------------------+------------------------------+ -| Codec | Purpose | Encoder/decoder | -+======================+===========================+==============================+ -| base64_codec [#b64]_ | Convert operand to MIME | :meth:base64.b64encode, | -| | base64 (the result always | :meth:base64.b64decode | -| | includes a trailing | | -| | '\n') | | -+----------------------+---------------------------+------------------------------+ -| bz2_codec | Compress the operand | :meth:bz2.compress, | -| | using bz2 | :meth:bz2.decompress | -+----------------------+---------------------------+------------------------------+ -| hex_codec | Convert operand to | :meth:base64.b16encode, | -| | hexadecimal | :meth:base64.b16decode | -| | representation, with two | | -| | digits per byte | | -+----------------------+---------------------------+------------------------------+ -| quopri_codec | Convert operand to MIME | :meth:quopri.encodestring, | -| | quoted printable | :meth:quopri.decodestring | -+----------------------+---------------------------+------------------------------+ -| uu_codec | Convert the operand using | :meth:uu.encode, | -| | uuencode | :meth:uu.decode | -+----------------------+---------------------------+------------------------------+ -| zlib_codec | Compress the operand | :meth:zlib.compress, | -| | using gzip | :meth:zlib.decompress | -+----------------------+---------------------------+------------------------------+ ++----------------------+------------------------------+------------------------------+ +| Codec | Purpose | Encoder / decoder | ++======================+==============================+==============================+ +| base64_codec [#b64]_ | Convert operand to MIME | :meth:base64.b64encode / | +| | base64 (the result always | :meth:base64.b64decode | +| | includes a trailing | | +| | '\n') | | +| | | | +| | .. versionchanged:: 3.4 | | +| | accepts any | | +| | :term:bytes-like object | | +| | as input for encoding and | | +| | decoding | | ++----------------------+------------------------------+------------------------------+ +| bz2_codec | Compress the operand | :meth:bz2.compress / | +| | using bz2 | :meth:bz2.decompress | ++----------------------+------------------------------+------------------------------+ +| hex_codec | Convert operand to | :meth:base64.b16encode / | +| | hexadecimal | :meth:base64.b16decode | +| | representation, with two | | +| | digits per byte | | ++----------------------+------------------------------+------------------------------+ +| quopri_codec | Convert operand to MIME | :meth:quopri.encodestring /| +| | quoted printable | :meth:quopri.decodestring | ++----------------------+------------------------------+------------------------------+ +| uu_codec | Convert the operand using | :meth:uu.encode / | +| | uuencode | :meth:uu.decode | ++----------------------+------------------------------+------------------------------+ +| zlib_codec | Compress the operand | :meth:zlib.compress / | +| | using gzip | :meth:zlib.decompress | ++----------------------+------------------------------+------------------------------+ -.. [#b64] Rather than accepting any :term:bytes-like object,

• 'base64_codec' accepts only :class:bytes and :class:bytearray for
• encoding and only :class:bytes, :class:bytearray, and ASCII-only
• instances of :class:str for decoding +.. [#b64] In addition to :term:bytes-like objects <bytes-like object>,

• 'base64_codec' also accepts ASCII-only instances of :class:str for
• decoding

The following codecs provide :class:str to :class:str mappings.

--- a/Lib/base64.py +++ b/Lib/base64.py @@ -35,11 +35,13 @@ def _bytes_from_decode_data(s): return s.encode('ascii') except UnicodeEncodeError: raise ValueError('string argument should contain only ASCII characters')

• elif isinstance(s, bytes_types):

• if isinstance(s, bytes_types): return s

• else:

•    raise TypeError("argument should be bytes or ASCII string, not %s" % s.__class__.__name__)[](#l3.11)

-

• try:

•    return memoryview(s).tobytes()[](#l3.14)

• except TypeError:

•    raise TypeError("argument should be a bytes-like object or ASCII "[](#l3.16)

•                    "string, not %r" % s.__class__.__name__) from None[](#l3.17)

Base64 encoding/decoding uses binascii

@@ -54,14 +56,9 @@ def b64encode(s, altchars=None): The encoded byte string is returned. """

• if not isinstance(s, bytes_types):

•    raise TypeError("expected bytes, not %s" % s.__class__.__name__)[](#l3.26)

  Strip off the trailing newline

  encoded = binascii.b2a_base64(s)[:-1] if altchars is not None:

•    if not isinstance(altchars, bytes_types):[](#l3.30)

•        raise TypeError("expected bytes, not %s"[](#l3.31)

•                        % altchars.__class__.__name__)[](#l3.32)
     assert len(altchars) == 2, repr(altchars)[](#l3.33)
     return encoded.translate(bytes.maketrans(b'+/', altchars))[](#l3.34)

  return encoded @@ -149,7 +146,7 @@ def b32encode(s): s is the byte string to encode. The encoded byte string is returned. """ if not isinstance(s, bytes_types):

•    raise TypeError("expected bytes, not %s" % s.__class__.__name__)[](#l3.40)

•    s = memoryview(s).tobytes()[](#l3.41)

  leftover = len(s) % 5

  Pad the last quantum with zero bits if necessary

  if leftover: @@ -250,8 +247,6 @@ def b16encode(s): s is the byte string to encode. The encoded byte string is returned. """

• if not isinstance(s, bytes_types):

•    raise TypeError("expected bytes, not %s" % s.__class__.__name__)[](#l3.50)

  return binascii.hexlify(s).upper() @@ -306,12 +301,26 @@ def decode(input, output): s = binascii.a2b_base64(line) output.write(s)

+def _input_type_check(s):

• try:

•    m = memoryview(s)[](#l3.60)

• except TypeError as err:

•    msg = "expected bytes-like object, not %s" % s.__class__.__name__[](#l3.62)

•    raise TypeError(msg) from err[](#l3.63)

• if m.format not in ('c', 'b', 'B'):

•    msg = ("expected single byte elements, not %r from %s" %[](#l3.65)

•                                      (m.format, s.__class__.__name__))[](#l3.66)

•    raise TypeError(msg)[](#l3.67)

• if m.ndim != 1:

•    msg = ("expected 1-D data, not %d-D data from %s" %[](#l3.69)

•                                      (m.ndim, s.__class__.__name__))[](#l3.70)

•    raise TypeError(msg)[](#l3.71)

+ def encodebytes(s): """Encode a bytestring into a bytestring containing multiple lines of base-64 data."""

• if not isinstance(s, bytes_types):

•    raise TypeError("expected bytes, not %s" % s.__class__.__name__)[](#l3.78)

• _input_type_check(s) pieces = [] for i in range(0, len(s), MAXBINSIZE): chunk = s[i : i + MAXBINSIZE]

@@ -328,8 +337,7 @@ def encodestring(s): def decodebytes(s): """Decode a bytestring of base-64 data into a bytestring."""

• if not isinstance(s, bytes_types):

•    raise TypeError("expected bytes, not %s" % s.__class__.__name__)[](#l3.88)

• _input_type_check(s) return binascii.a2b_base64(s) def decodestring(s):

--- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -5,10 +5,21 @@ import binascii import os import sys import subprocess - +import struct +from array import array class LegacyBase64TestCase(unittest.TestCase): +

• Legacy API is not as permissive as the modern API

• def check_type_errors(self, f):

•    self.assertRaises(TypeError, f, "")[](#l4.16)

•    self.assertRaises(TypeError, f, [])[](#l4.17)

•    multidimensional = memoryview(b"1234").cast('B', (2, 2))[](#l4.18)

•    self.assertRaises(TypeError, f, multidimensional)[](#l4.19)

•    int_data = memoryview(b"1234").cast('I')[](#l4.20)

•    self.assertRaises(TypeError, f, int_data)[](#l4.21)

+ def test_encodebytes(self): eq = self.assertEqual eq(base64.encodebytes(b"www.python.org"), b"d3d3LnB5dGhvbi5vcmc=\n") @@ -24,7 +35,9 @@ class LegacyBase64TestCase(unittest.Test b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==\n") # Non-bytes eq(base64.encodebytes(bytearray(b'abc')), b'YWJj\n')

•    self.assertRaises(TypeError, base64.encodebytes, "")[](#l4.30)

•    eq(base64.encodebytes(memoryview(b'abc')), b'YWJj\n')[](#l4.31)

•    eq(base64.encodebytes(array('B', b'abc')), b'YWJj\n')[](#l4.32)

•    self.check_type_errors(base64.encodebytes)[](#l4.33)

def test_decodebytes(self): eq = self.assertEqual @@ -41,7 +54,9 @@ class LegacyBase64TestCase(unittest.Test eq(base64.decodebytes(b''), b'') # Non-bytes eq(base64.decodebytes(bytearray(b'YWJj\n')), b'abc')

•    self.assertRaises(TypeError, base64.decodebytes, "")[](#l4.41)

•    eq(base64.decodebytes(memoryview(b'YWJj\n')), b'abc')[](#l4.42)

•    eq(base64.decodebytes(array('B', b'YWJj\n')), b'abc')[](#l4.43)

•    self.check_type_errors(base64.decodebytes)[](#l4.44)

def test_encode(self): eq = self.assertEqual @@ -73,6 +88,38 @@ class LegacyBase64TestCase(unittest.Test class BaseXYTestCase(unittest.TestCase): +

• Modern API completely ignores exported dimension and format data and

• treats any buffer as a stream of bytes

• def check_encode_type_errors(self, f):

•    self.assertRaises(TypeError, f, "")[](#l4.56)

•    self.assertRaises(TypeError, f, [])[](#l4.57)

+

• def check_decode_type_errors(self, f):

•    self.assertRaises(TypeError, f, [])[](#l4.60)

+

• def check_other_types(self, f, bytes_data, expected):

•    eq = self.assertEqual[](#l4.63)

•    eq(f(bytearray(bytes_data)), expected)[](#l4.64)

•    eq(f(memoryview(bytes_data)), expected)[](#l4.65)

•    eq(f(array('B', bytes_data)), expected)[](#l4.66)

•    self.check_nonbyte_element_format(base64.b64encode, bytes_data)[](#l4.67)

•    self.check_multidimensional(base64.b64encode, bytes_data)[](#l4.68)

+

• def check_multidimensional(self, f, data):

•    padding = b"\x00" if len(data) % 2 else b""[](#l4.71)

•    bytes_data = data + padding # Make sure cast works[](#l4.72)

•    shape = (len(bytes_data) // 2, 2)[](#l4.73)

•    multidimensional = memoryview(bytes_data).cast('B', shape)[](#l4.74)

•    self.assertEqual(f(multidimensional), f(bytes_data))[](#l4.75)

+

• def check_nonbyte_element_format(self, f, data):

•    padding = b"\x00" * ((4 - len(data)) % 4)[](#l4.78)

•    bytes_data = data + padding # Make sure cast works[](#l4.79)

•    int_data = memoryview(bytes_data).cast('I')[](#l4.80)

•    self.assertEqual(f(int_data), f(bytes_data))[](#l4.81)

+ + def test_b64encode(self): eq = self.assertEqual # Test default alphabet @@ -90,13 +137,16 @@ class BaseXYTestCase(unittest.TestCase): b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==") # Test with arbitrary alternative characters eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=b'$'), b'01ab$cd')

•    # Non-bytes[](#l4.91)

•    eq(base64.b64encode(bytearray(b'abcd')), b'YWJjZA==')[](#l4.92)
     eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=bytearray(b'*$')),[](#l4.93)
        b'01a*b$cd')[](#l4.94)

•    # Check if passing a str object raises an error[](#l4.95)

•    self.assertRaises(TypeError, base64.b64encode, "")[](#l4.96)

•    self.assertRaises(TypeError, base64.b64encode, b"", altchars="")[](#l4.97)

•    eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=memoryview(b'*$')),[](#l4.98)

•       b'01a*b$cd')[](#l4.99)

•    eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=array('B', b'*$')),[](#l4.100)

•       b'01a*b$cd')[](#l4.101)

•    # Non-bytes[](#l4.102)

•    self.check_other_types(base64.b64encode, b'abcd', b'YWJjZA==')[](#l4.103)

•    self.check_encode_type_errors(base64.b64encode)[](#l4.104)

•    self.assertRaises(TypeError, base64.b64encode, b"", altchars="*$")[](#l4.105)
     # Test standard alphabet[](#l4.106)
     eq(base64.standard_b64encode(b"www.python.org"), b"d3d3LnB5dGhvbi5vcmc=")[](#l4.107)
     eq(base64.standard_b64encode(b"a"), b"YQ==")[](#l4.108)

@@ -110,15 +160,15 @@ class BaseXYTestCase(unittest.TestCase): b"RUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0NT" b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==") # Non-bytes

•    eq(base64.standard_b64encode(bytearray(b'abcd')), b'YWJjZA==')[](#l4.113)

•    # Check if passing a str object raises an error[](#l4.114)

•    self.assertRaises(TypeError, base64.standard_b64encode, "")[](#l4.115)

•    self.check_other_types(base64.standard_b64encode,[](#l4.116)

•                           b'abcd', b'YWJjZA==')[](#l4.117)

•    self.check_encode_type_errors(base64.standard_b64encode)[](#l4.118)
     # Test with 'URL safe' alternative characters[](#l4.119)
     eq(base64.urlsafe_b64encode(b'\xd3V\xbeo\xf7\x1d'), b'01a-b_cd')[](#l4.120)
     # Non-bytes[](#l4.121)

•    eq(base64.urlsafe_b64encode(bytearray(b'\xd3V\xbeo\xf7\x1d')), b'01a-b_cd')[](#l4.122)

•    # Check if passing a str object raises an error[](#l4.123)

•    self.assertRaises(TypeError, base64.urlsafe_b64encode, "")[](#l4.124)

•    self.check_other_types(base64.urlsafe_b64encode,[](#l4.125)

•                           b'\xd3V\xbeo\xf7\x1d', b'01a-b_cd')[](#l4.126)

•    self.check_encode_type_errors(base64.urlsafe_b64encode)[](#l4.127)

def test_b64decode(self): eq = self.assertEqual @@ -141,7 +191,8 @@ class BaseXYTestCase(unittest.TestCase): eq(base64.b64decode(data), res) eq(base64.b64decode(data.decode('ascii')), res) # Non-bytes

•    eq(base64.b64decode(bytearray(b"YWJj")), b"abc")[](#l4.135)

•    self.check_other_types(base64.b64decode, b"YWJj", b"abc")[](#l4.136)

•    self.check_decode_type_errors(base64.b64decode)[](#l4.137)

# Test with arbitrary alternative characters tests_altchars = {(b'01ab$cd', b'$'): b'\xd3V\xbeo\xf7\x1d', @@ -160,7 +211,8 @@ class BaseXYTestCase(unittest.TestCase): eq(base64.standard_b64decode(data), res) eq(base64.standard_b64decode(data.decode('ascii')), res) # Non-bytes

•    eq(base64.standard_b64decode(bytearray(b"YWJj")), b"abc")[](#l4.145)

•    self.check_other_types(base64.standard_b64decode, b"YWJj", b"abc")[](#l4.146)

•    self.check_decode_type_errors(base64.standard_b64decode)[](#l4.147)

# Test with 'URL safe' alternative characters tests_urlsafe = {b'01a-b_cd': b'\xd3V\xbeo\xf7\x1d', @@ -170,7 +222,9 @@ class BaseXYTestCase(unittest.TestCase): eq(base64.urlsafe_b64decode(data), res) eq(base64.urlsafe_b64decode(data.decode('ascii')), res) # Non-bytes

•    eq(base64.urlsafe_b64decode(bytearray(b'01a-b_cd')), b'\xd3V\xbeo\xf7\x1d')[](#l4.155)

•    self.check_other_types(base64.urlsafe_b64decode, b'01a-b_cd',[](#l4.156)

•                           b'\xd3V\xbeo\xf7\x1d')[](#l4.157)

•    self.check_decode_type_errors(base64.urlsafe_b64decode)[](#l4.158)

def test_b64decode_padding_error(self): self.assertRaises(binascii.Error, base64.b64decode, b'abc') @@ -205,8 +259,8 @@ class BaseXYTestCase(unittest.TestCase): eq(base64.b32encode(b'abcd'), b'MFRGGZA=') eq(base64.b32encode(b'abcde'), b'MFRGGZDF') # Non-bytes

•    eq(base64.b32encode(bytearray(b'abcd')), b'MFRGGZA=')[](#l4.166)

•    self.assertRaises(TypeError, base64.b32encode, "")[](#l4.167)

•    self.check_other_types(base64.b32encode, b'abcd', b'MFRGGZA=')[](#l4.168)

•    self.check_encode_type_errors(base64.b32encode)[](#l4.169)

def test_b32decode(self): eq = self.assertEqual @@ -222,7 +276,8 @@ class BaseXYTestCase(unittest.TestCase): eq(base64.b32decode(data), res) eq(base64.b32decode(data.decode('ascii')), res) # Non-bytes

•    eq(base64.b32decode(bytearray(b'MFRGG===')), b'abc')[](#l4.177)

•    self.check_other_types(base64.b32decode, b'MFRGG===', b"abc")[](#l4.178)

•    self.check_decode_type_errors(base64.b32decode)[](#l4.179)

def test_b32decode_casefold(self): eq = self.assertEqual @@ -277,8 +332,9 @@ class BaseXYTestCase(unittest.TestCase): eq(base64.b16encode(b'\x01\x02\xab\xcd\xef'), b'0102ABCDEF') eq(base64.b16encode(b'\x00'), b'00') # Non-bytes

•    eq(base64.b16encode(bytearray(b'\x01\x02\xab\xcd\xef')), b'0102ABCDEF')[](#l4.187)

•    self.assertRaises(TypeError, base64.b16encode, "")[](#l4.188)

•    self.check_other_types(base64.b16encode, b'\x01\x02\xab\xcd\xef',[](#l4.189)

•                           b'0102ABCDEF')[](#l4.190)

•    self.check_encode_type_errors(base64.b16encode)[](#l4.191)

def test_b16decode(self): eq = self.assertEqual @@ -293,7 +349,15 @@ class BaseXYTestCase(unittest.TestCase): eq(base64.b16decode(b'0102abcdef', True), b'\x01\x02\xab\xcd\xef') eq(base64.b16decode('0102abcdef', True), b'\x01\x02\xab\xcd\xef') # Non-bytes

•    eq(base64.b16decode(bytearray(b"0102ABCDEF")), b'\x01\x02\xab\xcd\xef')[](#l4.199)

•    self.check_other_types(base64.b16decode, b"0102ABCDEF",[](#l4.200)

•                           b'\x01\x02\xab\xcd\xef')[](#l4.201)

•    self.check_decode_type_errors(base64.b16decode)[](#l4.202)

•    eq(base64.b16decode(bytearray(b"0102abcdef"), True),[](#l4.203)

•       b'\x01\x02\xab\xcd\xef')[](#l4.204)

•    eq(base64.b16decode(memoryview(b"0102abcdef"), True),[](#l4.205)

•       b'\x01\x02\xab\xcd\xef')[](#l4.206)

•    eq(base64.b16decode(array('B', b"0102abcdef"), True),[](#l4.207)

•       b'\x01\x02\xab\xcd\xef')[](#l4.208)

def test_decode_nonascii_str(self): decode_funcs = (base64.b64decode,

--- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -2285,6 +2285,24 @@ class TransformCodecTest(unittest.TestCa sout = reader.readline() self.assertEqual(sout, b"\x80")

• def test_buffer_api_usage(self):

•    # We check all the transform codecs accept memoryview input[](#l5.8)

•    # for encoding and decoding[](#l5.9)

•    # and also that they roundtrip correctly[](#l5.10)

•    original = b"12345\x80"[](#l5.11)

•    for encoding in bytes_transform_encodings:[](#l5.12)

•        data = original[](#l5.13)

•        view = memoryview(data)[](#l5.14)

•        data = codecs.encode(data, encoding)[](#l5.15)

•        view_encoded = codecs.encode(view, encoding)[](#l5.16)

•        self.assertEqual(view_encoded, data)[](#l5.17)

•        view = memoryview(data)[](#l5.18)

•        data = codecs.decode(data, encoding)[](#l5.19)

•        self.assertEqual(data, original)[](#l5.20)

•        view_decoded = codecs.decode(view, encoding)[](#l5.21)

•        self.assertEqual(view_decoded, data)[](#l5.22)

+ + @unittest.skipUnless(sys.platform == 'win32', 'code pages are specific to Windows')

--- a/Misc/NEWS +++ b/Misc/NEWS @@ -20,6 +20,10 @@ Core and Builtins Library ------- +- Issue #17839: base64.decodebytes and base64.encodebytes now accept any

• object that exports a 1 dimensional array of bytes (this means the same
• is now also true for base64_codec) +

• Issue #19132: The pprint module now supports compact mode.
• Issue #19137: The pprint module now correctly formats instances of set and