pytorch
Unable to add DataPipe function name list_files_by_ais_custom as it is already taken
🐛 Describe the bug
import torchdata.datapipes.iter.transform.bucketbatcher as b print(b.file)
Exception Traceback (most recent call last) Cell In[1], line 1 ----> 1 import torchdata.datapipes.iter.transform.bucketbatcher as b 2 print(b.file)
File ~/anaconda3/envs/regformer/lib/python3.8/site-packages/torchdata/init.py:9 1 # Copyright (c) Meta Platforms, Inc. and affiliates. 2 # All rights reserved. 3 # 4 # This source code is licensed under the BSD-style license found in the 5 # LICENSE file in the root directory of this source tree. 7 from torchdata import _extension # noqa: F401 ----> 9 from . import datapipes 11 janitor = datapipes.utils.janitor 13 try:
File ~/anaconda3/envs/regformer/lib/python3.8/site-packages/torchdata/datapipes/init.py:9 1 # Copyright (c) Meta Platforms, Inc. and affiliates. 2 # All rights reserved. 3 # 4 # This source code is licensed under the BSD-style license found in the 5 # LICENSE file in the root directory of this source tree. 7 from torch.utils.data import DataChunk, functional_datapipe ----> 9 from . import iter, map, utils 11 all = ["DataChunk", "functional_datapipe", "iter", "map", "utils"]
File ~/anaconda3/envs/regformer/lib/python3.8/site-packages/torchdata/datapipes/iter/init.py:32 10 from torch.utils.data import IterDataPipe 11 from torch.utils.data.datapipes.iter import ( 12 Batcher, 13 Collator, (...) 30 Zipper, 31 ) ---> 32 from torchdata.datapipes.iter.load.aisio import ( 33 AISFileListerIterDataPipe as AISFileLister, 34 AISFileLoaderIterDataPipe as AISFileLoader, 35 ) 37 ############################################################################### 38 # TorchData 39 ############################################################################### 40 from torchdata.datapipes.iter.load.fsspec import ( 41 FSSpecFileListerIterDataPipe as FSSpecFileLister, 42 FSSpecFileOpenerIterDataPipe as FSSpecFileOpener, 43 FSSpecSaverIterDataPipe as FSSpecSaver, 44 )
File ~/anaconda3/envs/regformer/lib/python3.8/site-packages/torchdata/datapipes/iter/load/aisio.py:102
97 raise TypeError(f"{type(self).name} instance doesn't have valid length")
98 return self.length
101 @functional_datapipe("list_files_by_ais_custom")
--> 102 class AISFileLoaderIterDataPipe(IterDataPipe[Tuple[str, StreamWrapper]]):
103 """
104 Iterable DataPipe that loads files from AIStore with the given URLs (functional name: load_files_by_ais).
105 Iterates all files in BytesIO format and returns a tuple (url, BytesIO).
(...)
127 ... pass
128 """
130 def init(self, source_datapipe: IterDataPipe[str], url: str, length: int = -1) -> None:
File ~/anaconda3/envs/regformer/lib/python3.8/site-packages/torch/utils/data/datapipes/_decorator.py:36, in functional_datapipe.call(self, cls)
32 if not isinstance(cls, non_deterministic) and
33 not (hasattr(cls, 'self') and
34 isinstance(cls.self, non_deterministic)):
35 raise TypeError('functional_datapipe can only decorate IterDataPipe')
---> 36 IterDataPipe.register_datapipe_as_function(self.name, cls, enable_df_api_tracing=self.enable_df_api_tracing)
37 elif issubclass(cls, MapDataPipe):
38 MapDataPipe.register_datapipe_as_function(self.name, cls)
File ~/anaconda3/envs/regformer/lib/python3.8/site-packages/torch/utils/data/datapipes/datapipe.py:135, in IterDataPipe.register_datapipe_as_function(cls, function_name, cls_to_register, enable_df_api_tracing) 132 @classmethod 133 def register_datapipe_as_function(cls, function_name, cls_to_register, enable_df_api_tracing=False): 134 if function_name in cls.functions: --> 135 raise Exception(f"Unable to add DataPipe function name {function_name} as it is already taken") 137 def class_function(cls, enable_df_api_tracing, source_dp, *args, **kwargs): 138 result_pipe = cls(source_dp, *args, **kwargs)
Exception: Unable to add DataPipe function name list_files_by_ais_custom as it is already taken
Versions
(regformer) cai045@cailabgpu:~$ pip list | grep torch torch 2.3.1+cu118 torchaudio 2.3.1+cu118 torchdata 0.7.1 torchtext 0.18.0 torchvision 0.18.1+cu118
Datapipes were deleted from torchdata 0.9 onwards. It will be hard for us to resolve this issue
