PERF: optimize DataFrame.sparse.from_spmatrix performance by rth · Pull Request #32825 · pandas-dev/pandas (original) (raw)

With #32856 merged, there is a new failure for non unique column names. In particular, TestFrameAccessor::test_from_spmatrix_columns[columns2] calling,

pd.DataFrame.sparse.from_spmatrix(mat, columns=['a', 'a'])

fails with,

pandas/core/arrays/sparse/accessor.py:251: in from_spmatrix return DataFrame._from_arrays( pandas/core/frame.py:1919: in _from_arrays mgr = arrays_to_mgr( pandas/core/internals/construction.py:77: in arrays_to_mgr return create_block_manager_from_arrays(arrays, arr_names, axes) pandas/core/internals/managers.py:1689: in create_block_manager_from_arrays blocks = form_blocks(arrays, names, axes)

arrays = [[0, 0.7151164908889105, 0, 0, 0.7790091888007832, 0.8726851819885113, 0.8634012065028459, 0, 0, 0.6582977581693065] F...13, 0, 0, 0.6245750826925724, 0, 0, 0.6870139075931206] Fill: 0 IntIndex Indices: array([1, 2, 3, 6, 9], dtype=int32) ] names = ['a', 'a'], axes = [['a', 'a'], RangeIndex(start=0, stop=10, step=1)]

def form_blocks(arrays, names, axes):
    # put "leftover" items in float bucket, where else?
    # generalize?
    items_dict = defaultdict(list)
    extra_locs = []

    names_idx = ensure_index(names)
    if names_idx.equals(axes[0]):
        names_indexer = np.arange(len(names_idx))
    else:

      assert names_idx.intersection(axes[0]).is_unique

E AssertionError

any suggestions @jorisvandenbossche ?