[issue4428] make io.BufferedWriter observe max_buffer_size limits - Code Review (original) (raw)

OLD

NEW

1 """The io module provides the Python interfaces to stream handling. The

2 builtin open function is defined in this module.

4 At the top of the I/O hierarchy is the abstract base class IOBase. It

5 defines the basic interface to a stream. Note, however, that there is no

6 seperation between reading and writing to streams; implementations are

7 allowed to throw an IOError if they do not support a given operation.

9 Extending IOBase is RawIOBase which deals simply with the reading and

10 writing of raw bytes to a stream. FileIO subclasses RawIOBase to provide

(...skipping 1029 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...

1040 raise TypeError("can't write str to binary stream")

1041 with self._write_lock:

1042 # XXX we can implement some more tricks to try and avoid

1043 # partial writes

1044 if len(self._write_buf) > self.buffer_size:

1045 # We're full, so let's pre-flush the buffer

1046 try:

1047 self._flush_unlocked()

1048 except BlockingIOError as e:

1049 # We can't accept anything else.

1050 # XXX Why not just let the exception pass through?

1050 # Reraise this with 0 in the written field as none of the

1051 # data passed to this call has been written.

1051 raise BlockingIOError(e.errno, e.strerror, 0)

1052 raise BlockingIOError(e.errno, e.strerror, 0)

1052 before = len(self._write_buf)

1053 before = len(self._write_buf)

1053 self._write_buf.extend(b)

1054 bytes_to_consume = self.max_buffer_size - before

1054 written = len(self._write_buf) - before

1055 # b is an iterable of ints, it won't always support len().

1056 if hasattr(b, '__len__') and len(b) > bytes_to_consume:

1057 try:

1058 chunk = memoryview(b)[:bytes_to_consume]

1059 except TypeError:

1060 # No buffer API? Make intermediate slice copies instead.

1061 chunk = b[:bytes_to_consume]

1062 # Loop over the data, flushing it to the underlying raw IO

1063 # stream in self.max_buffer_size chunks.

1064 written = 0

1065 self._write_buf.extend(chunk)

1066 while chunk and len(self._write_buf) > self.buffer_size:

1067 try:

1068 self._flush_unlocked()

1069 except BlockingIOError as e:

1070 written += e.characters_written

1071 raise BlockingIOError(e.errno, e.strerror, written)

1072 written += len(chunk)

1073 assert not self._write_buf, "_write_buf should be empty"

1074 if isinstance(chunk, memoryview):

1075 chunk = memoryview(b)[written:

1076 written + self.max_buffer_size]

1077 else:

1078 chunk = b[written:written + self.max_buffer_size]

1079 self._write_buf.extend(chunk)

1080 else:

1081 # This could go beyond self.max_buffer_size as we don't know

1082 # the length of b. The alternative of iterating over it one

1083 # byte at a time in python would be slow.

1084 self._write_buf.extend(b)

1085 written = len(self._write_buf) - before

1055 if len(self._write_buf) > self.buffer_size:

1086 if len(self._write_buf) > self.buffer_size:

1056 try:

1087 try:

1057 self._flush_unlocked()

1088 self._flush_unlocked()

1058 except BlockingIOError as e:

1089 except BlockingIOError as e:

1059 if len(self._write_buf) > self.max_buffer_size:

1090 if len(self._write_buf) > self.max_buffer_size:

1060 # We've hit max_buffer_size. We have to accept a

1091 # We've hit max_buffer_size. We have to accept a

1061 # partial write and cut back our buffer.

1092 # partial write and cut back our buffer.

1062 overage = len(self._write_buf) - self.max_buffer_size

1093 overage = len(self._write_buf) - self.max_buffer_size

1063 self._write_buf = self._write_buf[:self.max_buffer_size]

1094 self._write_buf = self._write_buf[:self.max_buffer_size]

1064 raise BlockingIOError(e.errno, e.strerror, overage)

1095 raise BlockingIOError(e.errno, e.strerror, overage)

(...skipping 1066 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...

2131 ("\r", "\n"),

2162 ("\r", "\n"),

2132 "\r\n",

2163 "\r\n",

2133 ("\n", "\r\n"),

2164 ("\n", "\r\n"),

2134 ("\r", "\r\n"),

2165 ("\r", "\r\n"),

2135 ("\r", "\n", "\r\n")

2166 ("\r", "\n", "\r\n")

2136 )[self._seennl]

2167 )[self._seennl]

2137

2168

2138

2169

2139 except ImportError:

2170 except ImportError:

2140 StringIO = _StringIO

2171 StringIO = _StringIO

OLD

NEW