BUG: "groupby" failed for dataframe with object-type index · Issue #40014 · pandas-dev/pandas (original) (raw)

Code Sample, a copy-pastable example

I have a dataframe where the index is object type, and I found the error by accident. The following code is a simplified reproduction of the error.

df = pd.DataFrame( { "c0": ["x", "x", "x"], "c1": ["x", "x", "y"], "p": [0,1,2] } ) ​ df.index = df.index.astype("O") df.groupby(['c0','c1']).p.agg(lambda x:all(x>0))

ValueError Traceback (most recent call last) in 5 6 df.index = df.index.astype("O") ----> 7 df.groupby(['c0','c1']).p.agg(lambda x:all(x>0))

/opt/conda/lib/python3.8/site-packages/pandas/core/groupby/generic.py in aggregate(self, func, engine, engine_kwargs, *args, **kwargs) 254 255 if self.grouper.nkeys > 1: --> 256 return self._python_agg_general(func, *args, **kwargs) 257 258 try:

/opt/conda/lib/python3.8/site-packages/pandas/core/groupby/groupby.py in _python_agg_general(self, func, *args, **kwargs) 1171 try: 1172 # if this function is invalid for this dtype, we will ignore it. -> 1173 result, counts = self.grouper.agg_series(obj, f) 1174 except TypeError: 1175 continue

/opt/conda/lib/python3.8/site-packages/pandas/core/groupby/ops.py in agg_series(self, obj, func) 704 705 try: --> 706 return self._aggregate_series_fast(obj, func) 707 except ValueError as err: 708 if "Must produce aggregated value" in str(err):

/opt/conda/lib/python3.8/site-packages/pandas/core/groupby/ops.py in _aggregate_series_fast(self, obj, func) 730 group_index = algorithms.take_nd(group_index, indexer, allow_fill=False) 731 grouper = libreduction.SeriesGrouper(obj, func, group_index, ngroups, dummy) --> 732 result, counts = grouper.get_result() 733 return result, counts 734

pandas/_libs/reduction.pyx in pandas._libs.reduction.SeriesGrouper.get_result()

pandas/_libs/reduction.pyx in pandas._libs.reduction._BaseGrouper._apply_to_group()

/opt/conda/lib/python3.8/site-packages/pandas/core/groupby/groupby.py in (x) 1158 def _python_agg_general(self, func, *args, **kwargs): 1159 func = self._is_builtin_func(func) -> 1160 f = lambda x: func(x, *args, **kwargs) 1161 1162 # iterate through "columns" ex exclusions to populate output dict

in (x) 5 6 df.index = df.index.astype("O") ----> 7 df.groupby(['c0','c1']).p.agg(lambda x:all(x>0))

/opt/conda/lib/python3.8/site-packages/pandas/core/ops/common.py in new_method(self, other) 63 other = item_from_zerodim(other) 64 ---> 65 return method(self, other) 66 67 return new_method

/opt/conda/lib/python3.8/site-packages/pandas/core/arraylike.py in gt(self, other) 43 @unpack_zerodim_and_defer("gt") 44 def gt(self, other): ---> 45 return self._cmp_method(other, operator.gt) 46 47 @unpack_zerodim_and_defer("ge")

/opt/conda/lib/python3.8/site-packages/pandas/core/series.py in _cmp_method(self, other, op) 4975 res_values = ops.comparison_op(lvalues, rvalues, op) 4976 -> 4977 return self._construct_result(res_values, name=res_name) 4978 4979 def _logical_method(self, other, op):

/opt/conda/lib/python3.8/site-packages/pandas/core/series.py in _construct_result(self, result, name) 2759 # We do not pass dtype to ensure that the Series constructor 2760 # does inference in the case where result has object-dtype. -> 2761 out = self._constructor(result, index=self.index) 2762 out = out.finalize(self) 2763

/opt/conda/lib/python3.8/site-packages/pandas/core/series.py in init(self, data, index, dtype, name, copy, fastpath) 346 try: 347 if len(index) != len(data): --> 348 raise ValueError( 349 f"Length of passed values is {len(data)}, " 350 f"index implies {len(index)}."

ValueError: Length of passed values is 1, index implies 2.

Problem description

If the index is not an object type, you will not get this error, so I guess there is something wrong with object type index.

Expected Output

c0 c1 x x False y True Name: p, dtype: bool

Output of pd.show_versions()

INSTALLED VERSIONS

commit : 7d32926
python : 3.8.5.final.0
python-bits : 64
OS : Linux
OS-release : 4.19.121-linuxkit
Version : #1 SMP Tue Dec 1 17:50:32 UTC 2020
machine : x86_64
processor : x86_64
byteorder : little
LC_ALL : en_US.UTF-8
LANG : en_US.UTF-8
LOCALE : en_US.UTF-8

pandas : 1.2.2
numpy : 1.19.1
pytz : 2020.1
dateutil : 2.8.1
pip : 20.2.3
setuptools : 49.6.0.post20200917
Cython : 0.29.21
pytest : 6.2.2
hypothesis : None
sphinx : None
blosc : None
feather : None
xlsxwriter : 1.3.7
lxml.etree : None
html5lib : None
pymysql : None
psycopg2 : None
jinja2 : 2.11.2
IPython : 7.18.1
pandas_datareader: None
bs4 : 4.9.2
bottleneck : 1.3.2
fsspec : 0.8.3
fastparquet : None
gcsfs : None
matplotlib : 3.3.2
numexpr : 2.7.1
odfpy : None
openpyxl : 3.0.6
pandas_gbq : None
pyarrow : 1.0.1
pyxlsb : None
s3fs : 0.2.2
scipy : 1.5.2
sqlalchemy : 1.3.19
tables : 3.6.1
tabulate : 0.8.7
xarray : 0.16.1
xlrd : 1.2.0
xlwt : None
numba : 0.51.2