modin Axis apply that gives partitions mismatching metadata along complementary axis leaves dataframe in invalid state.

System information

OS Platform and Distribution (e.g., Linux Ubuntu 16.04):
MacOSX x86
Modin version (modin.__version__):
0.15.2
Python version:
3.8.13
Code we can use to reproduce: I am trying to return a dataframe after doing a pandas apply. But I am getting a long error. This works perfectly on regular pandas. This is what I am trying todo in a Jupyter notebook:

def f(row):
    for daw in row[1]:
        col = f"descriptor_{daw['descriptor']}"
        row[col] = daw['weight']
    for metagenre in row[2]:
        col = f"metagenre_{metagenre}"
        row[col] = 1
    for genre in row[3]:
        col = f"genre_{genre}"
        row[col] = 1
    return row

df = training_set_data.apply(f, axis=1)
df

And I am getting the following error:

AssertionError Traceback (most recent call last) ~/.pyenv/versions/3.8.12/envs/delivery-notebooks/lib/python3.8/site-packages/IPython/core/formatters.py in call(self, obj) 700 type_pprinters=self.type_printers, 701 deferred_pprinters=self.deferred_printers) --> 702 printer.pretty(obj) 703 printer.flush() 704 return stream.getvalue()

~/.pyenv/versions/3.8.12/envs/delivery-notebooks/lib/python3.8/site-packages/IPython/lib/pretty.py in pretty(self, obj) 392 if cls is not object
393 and callable(cls.dict.get('repr')): --> 394 return _repr_pprint(obj, self, cycle) 395 396 return _default_pprint(obj, self, cycle)

~/.pyenv/versions/3.8.12/envs/delivery-notebooks/lib/python3.8/site-packages/IPython/lib/pretty.py in repr_pprint(obj, p, cycle) 698 """A pprint that just redirects to the normal repr function.""" 699 # Find newlines and replace them with p.break() --> 700 output = repr(obj) 701 lines = output.splitlines() 702 with p.group():

~/.pyenv/versions/3.8.12/envs/delivery-notebooks/lib/python3.8/site-packages/modin/logging/logger_metaclass.py in log_wrap(*args, **kwargs) 66 logger.info(f"END::PANDAS-API::{class_name}.{method_name}") 67 return result ---> 68 return method(*args, **kwargs) 69 70 return log_wrap

~/.pyenv/versions/3.8.12/envs/delivery-notebooks/lib/python3.8/site-packages/modin/pandas/dataframe.py in repr(self) 213 214 num_cols += len(self.columns) - i --> 215 result = repr(self._build_repr_df(num_rows, num_cols)) 216 if len(self.index) > num_rows or len(self.columns) > num_cols: 217 # The split here is so that we don't repr pandas row lengths.

~/.pyenv/versions/3.8.12/envs/delivery-notebooks/lib/python3.8/site-packages/modin/logging/logger_metaclass.py in log_wrap(*args, **kwargs) 66 logger.info(f"END::PANDAS-API::{class_name}.{method_name}") 67 return result ---> 68 return method(*args, **kwargs) 69 70 return log_wrap

~/.pyenv/versions/3.8.12/envs/delivery-notebooks/lib/python3.8/site-packages/modin/pandas/base.py in _build_repr_df(self, num_rows, num_cols) 201 else: 202 indexer = row_indexer --> 203 return self.iloc[indexer]._query_compiler.to_pandas() 204 205 def _update_inplace(self, new_query_compiler):

~/.pyenv/versions/3.8.12/envs/delivery-notebooks/lib/python3.8/site-packages/modin/logging/logger_metaclass.py in log_wrap(*args, **kwargs) 66 logger.info(f"END::PANDAS-API::{class_name}.{method_name}") 67 return result ---> 68 return method(*args, **kwargs) 69 70 return log_wrap

~/.pyenv/versions/3.8.12/envs/delivery-notebooks/lib/python3.8/site-packages/modin/core/storage_formats/pandas/query_compiler.py in to_pandas(self) 257 258 def to_pandas(self): --> 259 return self._modin_frame.to_pandas() 260 261 @classmethod

~/.pyenv/versions/3.8.12/envs/delivery-notebooks/lib/python3.8/site-packages/modin/logging/logger_metaclass.py in log_wrap(*args, **kwargs) 66 logger.info(f"END::PANDAS-API::{class_name}.{method_name}") 67 return result ---> 68 return method(*args, **kwargs) 69 70 return log_wrap

~/.pyenv/versions/3.8.12/envs/delivery-notebooks/lib/python3.8/site-packages/modin/core/dataframe/pandas/dataframe/dataframe.py in run_f_on_minimally_updated_metadata(self, *args, **kwargs) 113 elif apply_axis == "rows": 114 obj._propagate_index_objs(axis=0) --> 115 result = f(self, *args, **kwargs) 116 if apply_axis is None and not transpose: 117 result._deferred_index = self._deferred_index

~/.pyenv/versions/3.8.12/envs/delivery-notebooks/lib/python3.8/site-packages/modin/core/dataframe/pandas/dataframe/dataframe.py in to_pandas(self) 2833 pandas.DataFrame 2834 """ -> 2835 df = self._partition_mgr_cls.to_pandas(self._partitions) 2836 if df.empty: 2837 df = pandas.DataFrame(columns=self.columns, index=self.index)

~/.pyenv/versions/3.8.12/envs/delivery-notebooks/lib/python3.8/site-packages/modin/core/dataframe/pandas/partitioning/partition_manager.py in to_pandas(cls, partitions) 658 return pandas.DataFrame() 659 else: --> 660 return concatenate(df_rows) 661 662 @classmethod

~/.pyenv/versions/3.8.12/envs/delivery-notebooks/lib/python3.8/site-packages/modin/core/dataframe/pandas/utils.py in concatenate(dfs) 36 """ 37 for df in dfs: ---> 38 assert df.columns.equals(dfs[0].columns) 39 for i in range(len(dfs[0].columns)): 40 if dfs[0].dtypes.iloc[i].name != "category":

AssertionError:

AssertionError Traceback (most recent call last) ~/.pyenv/versions/3.8.12/envs/delivery-notebooks/lib/python3.8/site-packages/IPython/core/formatters.py in call(self, obj) 343 method = get_real_method(obj, self.print_method) 344 if method is not None: --> 345 return method() 346 return None 347 else:

~/.pyenv/versions/3.8.12/envs/delivery-notebooks/lib/python3.8/site-packages/modin/logging/logger_metaclass.py in log_wrap(*args, **kwargs) 66 logger.info(f"END::PANDAS-API::{class_name}.{method_name}") 67 return result ---> 68 return method(*args, **kwargs) 69 70 return log_wrap

~/.pyenv/versions/3.8.12/envs/delivery-notebooks/lib/python3.8/site-packages/modin/pandas/dataframe.py in repr_html(self) 235 # We use pandas repr_html to get a string of the HTML representation 236 # of the dataframe. --> 237 result = self._build_repr_df(num_rows, num_cols).repr_html() 238 if len(self.index) > num_rows or len(self.columns) > num_cols: 239 # We split so that we insert our correct dataframe dimensions.