cpython@a5ebc20 (original) (raw)

5 files changed

lines changed

Original file line number	Diff line number	Diff line change
@@ -29,6 +29,19 @@ PyAPI_FUNC(char*) _Py_EncodeLocaleEx(
29	29
30	30	PyAPI_FUNC(PyObject *) _Py_device_encoding(int);
31	31
	32	+#if defined(MS_WINDOWS) \|
	33	+/* On Windows, the count parameter of read() is an int (bpo-9015, bpo-9611).
	34	+ On macOS 10.13, read() and write() with more than INT_MAX bytes
	35	+ fail with EINVAL (bpo-24658). */
	36	+# define _PY_READ_MAX INT_MAX
	37	+# define _PY_WRITE_MAX INT_MAX
	38	+#else
	39	+/* write() should truncate the input to PY_SSIZE_T_MAX bytes,
	40	+ but it's safer to do it ourself to have a portable behaviour */
	41	+# define _PY_READ_MAX PY_SSIZE_T_MAX
	42	+# define _PY_WRITE_MAX PY_SSIZE_T_MAX
	43	+#endif
	44	+
32	45	#ifdef MS_WINDOWS
33	46	struct _Py_stat_struct {
34	47	unsigned long st_dev;

Original file line number	Diff line number	Diff line change
@@ -5,12 +5,12 @@
5	5	import stat
6	6	import sys
7	7	import unittest
8		-from test.support import TESTFN, requires, unlink
	8	+from test.support import TESTFN, requires, unlink, bigmemtest
9	9	import io # C implementation of io
10	10	import _pyio as pyio # Python implementation of io
11	11
12		-# size of file to create (>2GB; 2GB == 2147483648 bytes)
13		-size = 2500000000
	12	+# size of file to create (>2 GiB; 2 GiB == 2,147,483,648 bytes)
	13	+size = 2_500_000_000
14	14
15	15	class LargeFileTest:
16	16	"""Test that each file function works as expected for large
@@ -45,6 +45,15 @@ def tearDownClass(cls):
45	45	raise cls.failureException('File was not truncated by opening '
46	46	'with mode "wb"')
47	47
	48	+# _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes,
	49	+# so memuse=2 is needed
	50	+@bigmemtest(size=size, memuse=2, dry_run=False)
	51	+def test_large_read(self, _size):
	52	+# bpo-24658: Test that a read greater than 2GB does not fail.
	53	+with self.open(TESTFN, "rb") as f:
	54	+self.assertEqual(len(f.read()), size + 1)
	55	+self.assertEqual(f.tell(), size + 1)
	56	+
48	57	def test_osstat(self):
49	58	self.assertEqual(os.stat(TESTFN)[stat.ST_SIZE], size+1)
50	59

Original file line number	Diff line number	Diff line change
@@ -0,0 +1 @@
	1	+On macOS, fix reading from and writing into a file with a size larger than 2 GiB.

Original file line number	Diff line number	Diff line change
@@ -799,11 +799,9 @@ _io_FileIO_read_impl(fileio *self, Py_ssize_t size)
799	799	if (size < 0)
800	800	return _io_FileIO_readall_impl(self);
801	801
802		-#ifdef MS_WINDOWS
803		-/* On Windows, the count parameter of read() is an int */
804		-if (size > INT_MAX)
805		-size = INT_MAX;
806		-#endif
	802	+if (size > _PY_READ_MAX) {
	803	+size = _PY_READ_MAX;
	804	+ }
807	805
808	806	bytes = PyBytes_FromStringAndSize(NULL, size);
809	807	if (bytes == NULL)

Original file line number	Diff line number	Diff line change
@@ -1263,18 +1263,9 @@ _Py_read(int fd, void *buf, size_t count)
1263	1263	* handler raised an exception. */
1264	1264	assert(!PyErr_Occurred());
1265	1265
1266		-#ifdef MS_WINDOWS
1267		-if (count > INT_MAX) {
1268		-/* On Windows, the count parameter of read() is an int */
1269		-count = INT_MAX;
1270		- }
1271		-#else
1272		-if (count > PY_SSIZE_T_MAX) {
1273		-/* if count is greater than PY_SSIZE_T_MAX,
1274		- * read() result is undefined */
1275		-count = PY_SSIZE_T_MAX;
	1266	+if (count > _PY_READ_MAX) {
	1267	+count = _PY_READ_MAX;
1276	1268	}
1277		-#endif
1278	1269
1279	1270	_Py_BEGIN_SUPPRESS_IPH
1280	1271	do {
@@ -1325,15 +1316,10 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held)
1325	1316	depending on heap usage). */
1326	1317	count = 32767;
1327	1318	}
1328		-else if (count > INT_MAX)
1329		-count = INT_MAX;
1330		-#else
1331		-if (count > PY_SSIZE_T_MAX) {
1332		-/* write() should truncate count to PY_SSIZE_T_MAX, but it's safer
1333		- * to do it ourself to have a portable behaviour. */
1334		-count = PY_SSIZE_T_MAX;
1335		- }
1336	1319	#endif
	1320	+if (count > _PY_WRITE_MAX) {
	1321	+count = _PY_WRITE_MAX;
	1322	+ }
1337	1323
1338	1324	if (gil_held) {
1339	1325	do {