handle nan values in DataFrame.update when overwrite=False · Issue #15593 · pandas-dev/pandas (original) (raw)

Code Sample

from pandas import DataFrame, date_range df1 = DataFrame({'A': [1,None,3], 'B': date_range('2000', periods=3)}) df2 = DataFrame({'A': [None, 2, 3]}) df1.update(df2, overwrite=False) df1

Problem description

I got TypeError: invalid type promotion error when updating a DF with a datetime column. The 2nd DF doesn't have this column. The error message is in the details (although bad formatted).

IMHO, the culpit is in the DataFrame.update. The block checking mask.all should be outside the if block and applies to the case overwrite=False as well.

            if overwrite:
                mask = isnull(that)

                # don't overwrite columns unecessarily
                if mask.all():
                    continue
            else:
                mask = notnull(this)

--------------------------------------------------------------------------- TypeError Traceback (most recent call last) in () 1 df1 = DataFrame({'A': [1,None,3], 'B': date_range('2000', periods=3)}) 2 df2 = DataFrame({'A': [None, 2, 3]}) ----> 3 df1.update(df2, overwrite=False) 4 df1 5

C:\Users\pcluo\Anaconda3\lib\site-packages\pandas\core\frame.py in update(self, other, join, overwrite, filter_func, raise_conflict)
3845
3846 self[col] = expressions.where(mask, this, that,
-> 3847 raise_on_error=True)
3848
3849 # ----------------------------------------------------------------------

C:\Users\pcluo\Anaconda3\lib\site-packages\pandas\computation\expressions.py in where(cond, a, b, raise_on_error, use_numexpr)
228
229 if use_numexpr:
--> 230 return _where(cond, a, b, raise_on_error=raise_on_error)
231 return _where_standard(cond, a, b, raise_on_error=raise_on_error)
232

C:\Users\pcluo\Anaconda3\lib\site-packages\pandas\computation\expressions.py in _where_numexpr(cond, a, b, raise_on_error)
151
152 if result is None:
--> 153 result = _where_standard(cond, a, b, raise_on_error)
154
155 return result

C:\Users\pcluo\Anaconda3\lib\site-packages\pandas\computation\expressions.py in _where_standard(cond, a, b, raise_on_error)
126 def _where_standard(cond, a, b, raise_on_error=True):
127 return np.where(_values_from_object(cond), _values_from_object(a),
--> 128 _values_from_object(b))
129
130

TypeError: invalid type promotion