nibabel.load crashes for very large files on python 3, not python 2 · Issue #362 · nipy/nibabel (original) (raw)

I just noticed the following error: loading a very large file (such as a resting state image from the HCP dataset, e.g. with dimensions (91, 109, 91, 1200)) works fine with nibabel and python 2.7. The same command on python 3.5 fails with the following error:

---------------------------------------------------------------------------
OverflowError                             Traceback (most recent call last)
<ipython-input-5-6d7286f1eb52> in <module>()
----> 1 in_vol = in_img.get_data()

/usr/local/lib/python3.5/dist-packages/nibabel/spatialimages.py in get_data(self, caching)
    580         if self._data_cache is not None:
    581             return self._data_cache
--> 582         data = np.asanyarray(self._dataobj)
    583         if caching == 'fill':
    584             self._data_cache = data

/usr/local/lib/python3.5/dist-packages/numpy/core/numeric.py in asanyarray(a, dtype, order)
    512 
    513     """
--> 514     return array(a, dtype, copy=False, order=order, subok=True)
    515 
    516 def ascontiguousarray(a, dtype=None):

/usr/local/lib/python3.5/dist-packages/nibabel/arrayproxy.py in __array__(self)
    142     def __array__(self):
    143         # Read array and scale
--> 144         raw_data = self.get_unscaled()
    145         return apply_read_scaling(raw_data, self._slope, self._inter)
    146 

/usr/local/lib/python3.5/dist-packages/nibabel/arrayproxy.py in get_unscaled(self)
    137                                        offset=self._offset,
    138                                        order=self.order,
--> 139                                        mmap=self._mmap)
    140         return raw_data
    141 

/usr/local/lib/python3.5/dist-packages/nibabel/volumeutils.py in array_from_file(shape, in_dtype, infile, offset, order, mmap)
    512     if hasattr(infile, 'readinto'):
    513         data_bytes = bytearray(n_bytes)
--> 514         n_read = infile.readinto(data_bytes)
    515         needs_copy = False
    516     else:

/usr/lib/python3.5/gzip.py in read(self, size)
    272             import errno
    273             raise OSError(errno.EBADF, "read() on write-only GzipFile object")
--> 274         return self._buffer.read(size)
    275 
    276     def read1(self, size=-1):

/usr/lib/python3.5/_compression.py in readinto(self, b)
     66     def readinto(self, b):
     67         with memoryview(b) as view, view.cast("B") as byte_view:
---> 68             data = self.read(len(byte_view))
     69             byte_view[:len(data)] = data
     70         return len(data)

/usr/lib/python3.5/gzip.py in read(self, size)
    467             buf = self._fp.read(io.DEFAULT_BUFFER_SIZE)
    468 
--> 469             uncompress = self._decompressor.decompress(buf, size)
    470             if self._decompressor.unconsumed_tail != b"":
    471                 self._fp.prepend(self._decompressor.unconsumed_tail)

OverflowError: Python int too large for C unsigned int

Happy to provide further info, not sure what is going on.