(original) (raw)

# HG changeset patch # Parent 9fcfdb53e8af859b86bd8dfd84368c015f88f659 Issue #22233: Change email header parser to only accept CR, LF, and CRLF Also adds a slightly faster route when pushing a partial line, which compensates for the slower behaviour of StringIO compared with splitlines(). diff -r 9fcfdb53e8af Lib/email/feedparser.py --- a/Lib/email/feedparser.py Thu Nov 26 23:52:34 2015 +0200 +++ b/Lib/email/feedparser.py Sat Nov 28 01:30:17 2015 +0000 @@ -27,6 +27,7 @@ from email import message from email._policybase import compat32 from collections import deque +from io import StringIO NLCRE = re.compile('\r\n|\r|\n') NLCRE_bol = re.compile('(\r\n|\r|\n)') @@ -51,8 +52,8 @@ simple abstraction -- it parses until EOF closes the current message. """ def __init__(self): - # Chunks of the last partial line pushed into this object. - self._partial = [] + # Text stream of the last partial line pushed into this object. + self._partial = StringIO(newline='') # A deque of full, pushed lines self._lines = deque() # The stack of false-EOF checking predicates. @@ -68,8 +69,10 @@ def close(self): # Don't forget any trailing partial line. - self.pushlines(''.join(self._partial).splitlines(True)) - self._partial = [] + self._partial.seek(0) + self.pushlines(self._partial.readlines()) + self._partial.seek(0) + self._partial.truncate() self._closed = True def readline(self): @@ -97,26 +100,24 @@ def push(self, data): """Push some new data into this object.""" - # Crack into lines, but preserve the linesep characters on the end of each - parts = data.splitlines(True) - - if not parts or not parts[0].endswith(('\n', '\r')): + self._partial.write(data) + if '\n' not in data and '\r' not in data: # No new complete lines, so just accumulate partials - self._partial += parts + # This avoids quadratic copying when building a very long line return - if self._partial: - # If there are previous leftovers, complete them now - self._partial.append(parts[0]) - parts[0:1] = ''.join(self._partial).splitlines(True) - del self._partial[:] + # Crack into lines, but preserve the linesep characters on the end of each + self._partial.seek(0) + parts = self._partial.readlines() + self._partial.seek(0) + self._partial.truncate() # If the last element of the list does not end in a newline, then treat # it as a partial line. We only check for '\n' here because a line # ending with '\r' might be a line that was split in the middle of a # '\r\n' sequence (see bugs 1555570 and 1721862). if not parts[-1].endswith('\n'): - self._partial = [parts.pop()] + self._partial.write(parts.pop()) self.pushlines(parts) def pushlines(self, lines): diff -r 9fcfdb53e8af Lib/test/test_email/test_email.py --- a/Lib/test/test_email/test_email.py Thu Nov 26 23:52:34 2015 +0200 +++ b/Lib/test/test_email/test_email.py Sat Nov 28 01:30:17 2015 +0000 @@ -3444,10 +3444,12 @@ self.assertEqual(m.keys(), ['a', 'b']) m = self.parse(['a:\r', '\nb:\n']) self.assertEqual(m.keys(), ['a', 'b']) + + # Only CR and LF should break header fields m = self.parse(['a:\x85b:\u2028c:\n']) - self.assertEqual(m.items(), [('a', '\x85'), ('b', '\u2028'), ('c', '')]) + self.assertEqual(m.items(), [('a', '\x85b:\u2028c:')]) m = self.parse(['a:\r', 'b:\x85', 'c:\n']) - self.assertEqual(m.items(), [('a', ''), ('b', '\x85'), ('c', '')]) + self.assertEqual(m.items(), [('a', ''), ('b', '\x85c:')]) def test_long_lines(self): # Expected peak memory use on 32-bit platform: 6*N*M bytes. diff -r 9fcfdb53e8af Lib/test/test_email/test_parser.py --- a/Lib/test/test_email/test_parser.py Thu Nov 26 23:52:34 2015 +0200 +++ b/Lib/test/test_email/test_parser.py Sat Nov 28 01:30:17 2015 +0000 @@ -32,5 +32,33 @@ # XXX add tests for other functions that take Message arg. +class TestNewlines(unittest.TestCase): + def test_non_crlf(self): + msg = email.message_from_string( + "Next-Line: not\x85broken\r\n" + "Null: not\x00broken\r\n" + "Vertical-Tab: not\vbroken\r\n" + "Form-Feed: not\fbroken\r\n" + "File-Separator: not\x1Cbroken\r\n" + "Group-Separator: not\x1Dbroken\r\n" + "Record-Separator: not\x1Ebroken\r\n" + "Line-Separator: not\u2028broken\r\n" + "Paragraph-Separator: not\u2029broken\r\n" + "\r\n" + ) + self.assertSequenceEqual(msg.items(), ( + ("Next-Line", "not\x85broken"), + ("Null", "not\x00broken"), + ("Vertical-Tab", "not\vbroken"), + ("Form-Feed", "not\fbroken"), + ("File-Separator", "not\x1Cbroken"), + ("Group-Separator", "not\x1Dbroken"), + ("Record-Separator", "not\x1Ebroken"), + ("Line-Separator", "not\u2028broken"), + ("Paragraph-Separator", "not\u2029broken"), + )) + self.assertEqual(msg.get_payload(), "") + + if __name__ == '__main__': unittest.main() diff -r 9fcfdb53e8af Lib/test/test_httplib.py --- a/Lib/test/test_httplib.py Thu Nov 26 23:52:34 2015 +0200 +++ b/Lib/test/test_httplib.py Sat Nov 28 01:30:17 2015 +0000 @@ -283,6 +283,36 @@ self.assertEqual(resp.getheader('First'), 'val') self.assertEqual(resp.getheader('Second'), 'val') + def test_parse_all_octets(self): + # Ensure no valid header field octet breaks the parser + body = ( + b'HTTP/1.1 200 OK\r\n' + b"!#$%&'*+-.^_`|~: value\r\n" # Special token characters + b'VCHAR: ' + bytes(range(0x21, 0x7E + 1)) + b'\r\n' + b'obs-text: ' + bytes(range(0x80, 0xFF + 1)) + b'\r\n' + b'obs-fold: text\r\n' + b' folded with space\r\n' + b'\tfolded with tab\r\n' + b'Content-Length: 0\r\n' + b'\r\n' + ) + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.getheader('Content-Length'), '0') + self.assertEqual(resp.msg['Content-Length'], '0') + self.assertEqual(resp.getheader("!#$%&'*+-.^_`|~"), 'value') + self.assertEqual(resp.msg["!#$%&'*+-.^_`|~"], 'value') + vchar = ''.join(map(chr, range(0x21, 0x7E + 1))) + self.assertEqual(resp.getheader('VCHAR'), vchar) + self.assertEqual(resp.msg['VCHAR'], vchar) + self.assertIsNotNone(resp.getheader('obs-text')) + self.assertIn('obs-text', resp.msg) + for folded in (resp.getheader('obs-fold'), resp.msg['obs-fold']): + self.assertTrue(folded.startswith('text')) + self.assertIn(' folded with space', folded) + self.assertTrue(folded.endswith('folded with tab')) + def test_invalid_headers(self): conn = client.HTTPConnection('example.com') conn.sock = FakeSocket('') diff -r 9fcfdb53e8af Misc/NEWS --- a/Misc/NEWS Thu Nov 26 23:52:34 2015 +0200 +++ b/Misc/NEWS Sat Nov 28 01:30:17 2015 +0000 @@ -102,6 +102,11 @@ Library ------- +- Issue #22233: Change the email package's parser to only accept CR, LF, and + CRLF sequences as breaking header fields, rather than anything accepted by + str.splitlines(). This allows HTTP headers to be parsed when they include + the byte 0x85. + - Issue #10131: Fixed deep copying of minidom documents. Based on patch by Marian Ganisin.