BUG: ArrowInvalid dividing large int64[pyarrow] · Issue #56645 · pandas-dev/pandas (original) (raw)
This appears to be a 2.2 regression. I'm trying to illustrate using the wrong ns/ms for datetime conversion but the above code is failing with this error:
---------------------------------------------------------------------------
ArrowInvalid Traceback (most recent call last)
Cell In[199], line 1
----> 1 pd.Series([1425801600000000000,
2 1425803400000000000,
3 1425805200000000000,
4 1425807000000000000,
5 1425808800000000000,
6 1425801600000000000,
7 1425803400000000000,
8 1425805200000000000,
9 1425807000000000000,
10 1446359400000000000,
11 1446361200000000000,
12 1446363000000000000,
13 1446364800000000000,
14 1446366600000000000,
15 1446364800000000000,
16 1446366600000000000,
17 1446368400000000000,
18 1446370200000000000,
19 1446372000000000000], dtype='int64[pyarrow]') / 1_000_000
File ~/.envs/pd22rc/lib/python3.11/site-packages/pandas/core/ops/common.py:76, in _unpack_zerodim_and_defer.<locals>.new_method(self, other)
72 return NotImplemented
74 other = item_from_zerodim(other)
---> 76 return method(self, other)
File ~/.envs/pd22rc/lib/python3.11/site-packages/pandas/core/arraylike.py:210, in OpsMixin.__truediv__(self, other)
208 @unpack_zerodim_and_defer("__truediv__")
209 def __truediv__(self, other):
--> 210 return self._arith_method(other, operator.truediv)
File ~/.envs/pd22rc/lib/python3.11/site-packages/pandas/core/series.py:5995, in Series._arith_method(self, other, op)
5993 def _arith_method(self, other, op):
5994 self, other = self._align_for_op(other)
-> 5995 return base.IndexOpsMixin._arith_method(self, other, op)
File ~/.envs/pd22rc/lib/python3.11/site-packages/pandas/core/base.py:1382, in IndexOpsMixin._arith_method(self, other, op)
1379 rvalues = np.arange(rvalues.start, rvalues.stop, rvalues.step)
1381 with np.errstate(all="ignore"):
-> 1382 result = ops.arithmetic_op(lvalues, rvalues, op)
1384 return self._construct_result(result, name=res_name)
File ~/.envs/pd22rc/lib/python3.11/site-packages/pandas/core/ops/array_ops.py:273, in arithmetic_op(left, right, op)
260 # NB: We assume that extract_array and ensure_wrapped_if_datetimelike
261 # have already been called on `left` and `right`,
262 # and `maybe_prepare_scalar_for_op` has already been called on `right`
263 # We need to special-case datetime64/timedelta64 dtypes (e.g. because numpy
264 # casts integer dtypes to timedelta64 when operating with timedelta64 - GH#22390)
266 if (
267 should_extension_dispatch(left, right)
268 or isinstance(right, (Timedelta, BaseOffset, Timestamp))
(...)
271 # Timedelta/Timestamp and other custom scalars are included in the check
272 # because numexpr will fail on it, see GH#31457
--> 273 res_values = op(left, right)
274 else:
275 # TODO we should handle EAs consistently and move this check before the if/else
276 # (https://github.com/pandas-dev/pandas/issues/41165)
277 # error: Argument 2 to "_bool_arith_check" has incompatible type
278 # "Union[ExtensionArray, ndarray[Any, Any]]"; expected "ndarray[Any, Any]"
279 _bool_arith_check(op, left, right) # type: ignore[arg-type]
File ~/.envs/pd22rc/lib/python3.11/site-packages/pandas/core/ops/common.py:76, in _unpack_zerodim_and_defer.<locals>.new_method(self, other)
72 return NotImplemented
74 other = item_from_zerodim(other)
---> 76 return method(self, other)
File ~/.envs/pd22rc/lib/python3.11/site-packages/pandas/core/arraylike.py:210, in OpsMixin.__truediv__(self, other)
208 @unpack_zerodim_and_defer("__truediv__")
209 def __truediv__(self, other):
--> 210 return self._arith_method(other, operator.truediv)
File ~/.envs/pd22rc/lib/python3.11/site-packages/pandas/core/arrays/arrow/array.py:757, in ArrowExtensionArray._arith_method(self, other, op)
756 def _arith_method(self, other, op):
--> 757 return self._evaluate_op_method(other, op, ARROW_ARITHMETIC_FUNCS)
File ~/.envs/pd22rc/lib/python3.11/site-packages/pandas/core/arrays/arrow/array.py:745, in ArrowExtensionArray._evaluate_op_method(self, other, op, arrow_funcs)
742 if pc_func is NotImplemented:
743 raise NotImplementedError(f"{op.__name__} not implemented.")
--> 745 result = pc_func(self._pa_array, other)
746 return type(self)(result)
File ~/.envs/pd22rc/lib/python3.11/site-packages/pandas/core/arrays/arrow/array.py:138, in <lambda>(x, y)
128 result = result.cast(left.type)
129 return result
131 ARROW_ARITHMETIC_FUNCS = {
132 "add": pc.add_checked,
133 "radd": lambda x, y: pc.add_checked(y, x),
134 "sub": pc.subtract_checked,
135 "rsub": lambda x, y: pc.subtract_checked(y, x),
136 "mul": pc.multiply_checked,
137 "rmul": lambda x, y: pc.multiply_checked(y, x),
--> 138 "truediv": lambda x, y: pc.divide(cast_for_truediv(x, y), y),
139 "rtruediv": lambda x, y: pc.divide(y, cast_for_truediv(x, y)),
140 "floordiv": lambda x, y: floordiv_compat(x, y),
141 "rfloordiv": lambda x, y: floordiv_compat(y, x),
142 "mod": NotImplemented,
143 "rmod": NotImplemented,
144 "divmod": NotImplemented,
145 "rdivmod": NotImplemented,
146 "pow": pc.power_checked,
147 "rpow": lambda x, y: pc.power_checked(y, x),
148 }
150 if TYPE_CHECKING:
151 from collections.abc import Sequence
File ~/.envs/pd22rc/lib/python3.11/site-packages/pandas/core/arrays/arrow/array.py:116, in cast_for_truediv(arrow_array, pa_object)
108 def cast_for_truediv(
109 arrow_array: pa.ChunkedArray, pa_object: pa.Array | pa.Scalar
110 ) -> pa.ChunkedArray:
111 # Ensure int / int -> float mirroring Python/Numpy behavior
112 # as pc.divide_checked(int, int) -> int
113 if pa.types.is_integer(arrow_array.type) and pa.types.is_integer(
114 pa_object.type
115 ):
--> 116 return arrow_array.cast(pa.float64())
117 return arrow_array
File ~/.envs/pd22rc/lib/python3.11/site-packages/pyarrow/table.pxi:565, in pyarrow.lib.ChunkedArray.cast()
File ~/.envs/pd22rc/lib/python3.11/site-packages/pyarrow/compute.py:404, in cast(arr, target_type, safe, options, memory_pool)
402 else:
403 options = CastOptions.safe(target_type)
--> 404 return call_function("cast", [arr], options, memory_pool)
File ~/.envs/pd22rc/lib/python3.11/site-packages/pyarrow/_compute.pyx:590, in pyarrow._compute.call_function()
File ~/.envs/pd22rc/lib/python3.11/site-packages/pyarrow/_compute.pyx:385, in pyarrow._compute.Function.call()
File ~/.envs/pd22rc/lib/python3.11/site-packages/pyarrow/error.pxi:154, in pyarrow.lib.pyarrow_internal_check_status()
File ~/.envs/pd22rc/lib/python3.11/site-packages/pyarrow/error.pxi:91, in pyarrow.lib.check_status()
ArrowInvalid: Integer value 1425801600000000000 not in range: -9007199254740992 to 9007199254740992