BUG: Pandas 2.2: Index.difference regression with categoricals · Issue #57318 · pandas-dev/pandas (original) (raw)

Pandas version checks

Reproducible Example

In [1]: import pandas as pd

In [2]: pd.version Out[2]: '2.2.0'

In [3]: i = pd.Index(['a', 'b', 'c', None], dtype='category')

In [4]: i Out[4]: CategoricalIndex(['a', 'b', 'c', nan], categories=['a', 'b', 'c'], ordered=False, dtype='category')

In [5]: i.difference(['1']) Out[5]: CategoricalIndex(['a', 'b', 'c', nan], categories=['a', 'b', 'c'], ordered=False, dtype='category')

In [6]: i.difference(['1', None])

KeyError Traceback (most recent call last) File /nvme/0/pgali/envs/cudfdev/lib/python3.10/site-packages/pandas/core/indexes/base.py:3802, in Index.get_loc(self, key) 3801 try: -> 3802 return self._engine.get_loc(casted_key) 3803 except KeyError as err:

File index.pyx:153, in pandas._libs.index.IndexEngine.get_loc()

File index.pyx:182, in pandas._libs.index.IndexEngine.get_loc()

File pandas/_libs/hashtable_class_helper.pxi:7081, in pandas._libs.hashtable.PyObjectHashTable.get_item()

File pandas/_libs/hashtable_class_helper.pxi:7089, in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: nan

The above exception was the direct cause of the following exception:

KeyError Traceback (most recent call last) Cell In[6], line 1 ----> 1 i.difference(['1', None])

File /nvme/0/pgali/envs/cudfdev/lib/python3.10/site-packages/pandas/core/indexes/base.py:3661, in Index.difference(self, other, sort) 3658 return result.sort_values() 3659 return result -> 3661 result = self._difference(other, sort=sort) 3662 return self._wrap_difference_result(other, result)

File /nvme/0/pgali/envs/cudfdev/lib/python3.10/site-packages/pandas/core/indexes/base.py:3667, in Index._difference(self, other, sort) 3664 def _difference(self, other, sort): 3665 # overridden by RangeIndex 3666 other = other.unique() -> 3667 the_diff = self[other.get_indexer_for(self) == -1] 3668 the_diff = the_diff if self.is_unique else the_diff.unique() 3669 the_diff = _maybe_try_sort(the_diff, sort)

File /nvme/0/pgali/envs/cudfdev/lib/python3.10/site-packages/pandas/core/indexes/base.py:6181, in Index.get_indexer_for(self, target) 6163 """ 6164 Guaranteed return of an indexer even when non-unique. 6165 (...) 6178 array([0, 2]) 6179 """ 6180 if self._index_as_unique: -> 6181 return self.get_indexer(target) 6182 indexer, _ = self.get_indexer_non_unique(target) 6183 return indexer

File /nvme/0/pgali/envs/cudfdev/lib/python3.10/site-packages/pandas/core/indexes/base.py:3923, in Index.get_indexer(self, target, method, limit, tolerance) 3917 indexer = algos.take_nd(categories_indexer, target.codes, fill_value=-1) 3919 if (not self._is_multi and self.hasnans) and target.hasnans: 3920 # Exclude MultiIndex because hasnans raises NotImplementedError 3921 # we should only get here if we are unique, so loc is an integer 3922 # GH#41934 -> 3923 loc = self.get_loc(np.nan) 3924 mask = target.isna() 3925 indexer[mask] = loc

File /nvme/0/pgali/envs/cudfdev/lib/python3.10/site-packages/pandas/core/indexes/base.py:3809, in Index.get_loc(self, key) 3804 if isinstance(casted_key, slice) or ( 3805 isinstance(casted_key, abc.Iterable) 3806 and any(isinstance(x, slice) for x in casted_key) 3807 ): 3808 raise InvalidIndexError(key) -> 3809 raise KeyError(key) from err 3810 except TypeError: 3811 # If we have a listlike key, _check_indexing_error will raise 3812 # InvalidIndexError. Otherwise we fall through and re-raise 3813 # the TypeError. 3814 self._check_indexing_error(key)

KeyError: nan

Issue Description

There seems to be a KeyError in pandas-2.2, but in previous versions of pandas there was no error.

Expected Behavior

No error.

Installed Versions

INSTALLED VERSIONS

commit : fd3f571
python : 3.10.13.final.0
python-bits : 64
OS : Linux
OS-release : 5.15.0-91-generic
Version : #101-Ubuntu SMP Tue Nov 14 13:30:08 UTC 2023
machine : x86_64
processor : x86_64
byteorder : little
LC_ALL : None
LANG : en_US.UTF-8
LOCALE : en_US.UTF-8

pandas : 2.2.0
numpy : 1.26.4
pytz : 2024.1
dateutil : 2.8.2
setuptools : 69.0.3
pip : 24.0
Cython : 3.0.8
pytest : 7.4.4
hypothesis : 6.98.3
sphinx : 6.2.1
blosc : None
feather : None
xlsxwriter : None
lxml.etree : None
html5lib : None
pymysql : None
psycopg2 : None
jinja2 : 3.1.3
IPython : 8.21.0
pandas_datareader : None
adbc-driver-postgresql: None
adbc-driver-sqlite : None
bs4 : 4.12.3
bottleneck : None
dataframe-api-compat : None
fastparquet : None
fsspec : 2024.2.0
gcsfs : None
matplotlib : None
numba : 0.59.0
numexpr : None
odfpy : None
openpyxl : None
pandas_gbq : None
pyarrow : 14.0.1
pyreadstat : None
python-calamine : None
pyxlsb : None
s3fs : 2024.2.0
scipy : 1.12.0
sqlalchemy : 2.0.25
tables : None
tabulate : 0.9.0
xarray : None
xlrd : None
zstandard : None
tzdata : 2023.4
qtpy : None
pyqt5 : None