BUG: Series has no attribute "reshape" after adding a new category in df · Issue #35731 · pandas-dev/pandas (original) (raw)

Note: Please read this guide detailing how to provide the necessary information for us to reproduce your bug.

Code Sample, a copy-pastable example

Your code here

import pandas as pd from pandas.api.types import CategoricalDtype

create dataframe (note: every single column is a category)

df = pd.DataFrame( { "a": pd.Series([np.nan, 2.0, 3.0, 1.0]).astype("category"), "b": pd.Series(["A", "A", "B", "C"]).astype("category"), "c": pd.Series(["D", "E", "E", np.nan]).astype("category"), } )

get all categories from column "a"

cats =df["a"].cat.categories.tolist()

append new category

cats.append(0.0) df["a"] = df["a"].astype(CategoricalDtype(categories=cats, ordered=False))

fillna with that new category

df["a"].fillna(0, inplace=True)

run df.isnull()

df.isnull().sum()

Problem description

We are trying to add the new fillna as a new category in the dataframe, but it fails when we are trying to use df.isnull() In this case we are pretty much blocked from using df.isull().sum() functionality. Running the above snippet will get us the attribute error
AttributeError: 'Series' object has no attribute 'reshape'

Full stack trace is shown below.

AttributeError                            Traceback (most recent call last)
<ipython-input-29-cc81fd27034f> in <module>
----> 1 df.isnull()

~/.pyenv/versions/3.7.0/envs/fair_ml/lib/python3.7/site-packages/pandas/core/frame.py in isnull(self)
   4865     @doc(NDFrame.isna, klass=_shared_doc_kwargs["klass"])
   4866     def isnull(self) -> "DataFrame":
-> 4867         return self.isna()
   4868 
   4869     @doc(NDFrame.notna, klass=_shared_doc_kwargs["klass"])

~/.pyenv/versions/3.7.0/envs/fair_ml/lib/python3.7/site-packages/pandas/core/frame.py in isna(self)
   4860     @doc(NDFrame.isna, klass=_shared_doc_kwargs["klass"])
   4861     def isna(self) -> "DataFrame":
-> 4862         result = self._constructor(self._data.isna(func=isna))
   4863         return result.__finalize__(self, method="isna")
   4864 

~/.pyenv/versions/3.7.0/envs/fair_ml/lib/python3.7/site-packages/pandas/core/internals/managers.py in isna(self, func)
    500 
    501     def isna(self, func) -> "BlockManager":
--> 502         return self.apply("apply", func=func)
    503 
    504     def where(

~/.pyenv/versions/3.7.0/envs/fair_ml/lib/python3.7/site-packages/pandas/core/internals/managers.py in apply(self, f, align_keys, **kwargs)
    394                 applied = b.apply(f, **kwargs)
    395             else:
--> 396                 applied = getattr(b, f)(**kwargs)
    397             result_blocks = _extend_blocks(applied, result_blocks)
    398 

~/.pyenv/versions/3.7.0/envs/fair_ml/lib/python3.7/site-packages/pandas/core/internals/blocks.py in apply(self, func, **kwargs)
    346             result = func(self.values, **kwargs)
    347 
--> 348         return self._split_op_result(result)
    349 
    350     def _split_op_result(self, result) -> List["Block"]:

~/.pyenv/versions/3.7.0/envs/fair_ml/lib/python3.7/site-packages/pandas/core/internals/blocks.py in _split_op_result(self, result)
    361 
    362         if not isinstance(result, Block):
--> 363             result = self.make_block(result)
    364 
    365         return [result]

~/.pyenv/versions/3.7.0/envs/fair_ml/lib/python3.7/site-packages/pandas/core/internals/blocks.py in make_block(self, values, placement)
    250             placement = self.mgr_locs
    251         if self.is_extension:
--> 252             values = _block_shape(values, ndim=self.ndim)
    253 
    254         return make_block(values, placement=placement, ndim=self.ndim)

~/.pyenv/versions/3.7.0/envs/fair_ml/lib/python3.7/site-packages/pandas/core/internals/blocks.py in _block_shape(values, ndim)
   2745             # block.shape is incorrect for "2D" ExtensionArrays
   2746             # We can't, and don't need to, reshape.
-> 2747             values = values.reshape(tuple((1,) + shape))  # type: ignore
   2748     return values
   2749 

~/.pyenv/versions/3.7.0/envs/fair_ml/lib/python3.7/site-packages/pandas/core/generic.py in __getattr__(self, name)
   5128             if self._info_axis._can_hold_identifiers_and_holds_name(name):
   5129                 return self[name]
-> 5130             return object.__getattribute__(self, name)
   5131 
   5132     def __setattr__(self, name: str, value) -> None:

AttributeError: 'Series' object has no attribute 'reshape'

Expected Output

a 0
b 0
c 1

Output of pd.show_versions()

INSTALLED VERSIONS

commit : d9fff27
python : 3.7.0.final.0
python-bits : 64
OS : Darwin
OS-release : 19.6.0
Version : Darwin Kernel Version 19.6.0: Sun Jul 5 00:43:10 PDT 2020; root:xnu-6153.141.1~9/RELEASE_X86_64
machine : x86_64
processor : i386
byteorder : little
LC_ALL : None
LANG : en_US.UTF-8
LOCALE : en_US.UTF-8

pandas : 1.1.0
numpy : 1.19.0
pytz : 2020.1
dateutil : 2.8.1
pip : 18.1
setuptools : 47.3.1
Cython : None
pytest : 5.4.3
hypothesis : None
sphinx : None
blosc : None
feather : None
xlsxwriter : None
lxml.etree : None
html5lib : None
pymysql : None
psycopg2 : None
jinja2 : 2.11.2
IPython : 7.17.0
pandas_datareader: None
bs4 : None
bottleneck : None
fsspec : None
fastparquet : None
gcsfs : None
matplotlib : 3.3.1
numexpr : None
odfpy : None
openpyxl : None
pandas_gbq : None
pyarrow : None
pytables : None
pyxlsb : None
s3fs : None
scipy : 1.5.0
sqlalchemy : 1.3.13
tables : None
tabulate : 0.8.7
xarray : None
xlrd : None
xlwt : None
numba : None