securefederatedai
[Workflow API] ModuleNotFound error observed while running Workflow API tutorials with Python 3.12
Description ModuleNotFound error observed while running Workflow API tutorial in virtual environment created with Python 3.12
To Reproduce:
- Create a virtual environment with Python 3.12
- Run the Workflow Interface Tutorial:
openfl-tutorials/experimental/workflow/101_MNIST.ipynb - The tutorial fails with errors
Expected behavior The tutorial should run successfully without issues
Logs and Snapshot:
---------------------------------------------------------------------------
ModuleNotFoundError Traceback (most recent call last)
Cell In[5], line 3
1 from copy import deepcopy
----> 3 from openfl.experimental.workflow.interface import FLSpec, Aggregator, Collaborator
4 from openfl.experimental.workflow.runtime import LocalRuntime
5 from openfl.experimental.workflow.placement import aggregator, collaborator
File ~/miniforge-pypy3/envs/env_openfl_latest_3.12/lib/python3.12/site-packages/openfl/experimental/workflow/interface/__init__.py:7
1 # Copyright 2020-2024 Intel Corporation
2 # SPDX-License-Identifier: Apache-2.0
5 """openfl.experimental.workflow.interface package."""
----> 7 from openfl.experimental.workflow.interface.fl_spec import FLSpec
8 from openfl.experimental.workflow.interface.participants import Aggregator, Collaborator
File ~/miniforge-pypy3/envs/env_openfl_latest_3.12/lib/python3.12/site-packages/openfl/experimental/workflow/interface/fl_spec.py:16
13 if TYPE_CHECKING:
14 from openfl.experimental.workflow.runtime import FederatedRuntime, LocalRuntime, Runtime
---> 16 from openfl.experimental.workflow.utilities import (
17 MetaflowInterface,
18 SerializationError,
19 aggregator_to_collaborator,
20 checkpoint,
21 collaborator_to_aggregator,
22 filter_attributes,
23 generate_artifacts,
24 should_transfer,
25 )
28 class FLSpec:
29 """FLSpec Class
30
31 A class representing a Federated Learning Specification. It manages clones,
(...) 39 _runtime (RuntimeType): The runtime of the flow.
40 """
File ~/miniforge-pypy3/envs/env_openfl_latest_3.12/lib/python3.12/site-packages/openfl/experimental/workflow/utilities/__init__.py:12
5 """openfl.experimental.workflow.utilities package."""
7 from openfl.experimental.workflow.utilities.exceptions import (
8 ResourcesAllocationError,
9 ResourcesNotAvailableError,
10 SerializationError,
11 )
---> 12 from openfl.experimental.workflow.utilities.metaflow_utils import MetaflowInterface
13 from openfl.experimental.workflow.utilities.resources import get_number_of_gpus
14 from openfl.experimental.workflow.utilities.runtime_utils import (
15 check_resource_allocation,
16 checkpoint,
(...) 19 parse_attrs,
20 )
File ~/miniforge-pypy3/envs/env_openfl_latest_3.12/lib/python3.12/site-packages/openfl/experimental/workflow/utilities/metaflow_utils.py:23
21 import ray
22 from dill.source import getsource # nosec
---> 23 from metaflow.datastore import DATASTORES, FlowDataStore
24 from metaflow.datastore.exceptions import DataException, UnpicklableArtifactException
25 from metaflow.datastore.task_datastore import TaskDataStore, only_if_not_done, require_mode
File ~/miniforge-pypy3/envs/env_openfl_latest_3.12/lib/python3.12/site-packages/metaflow/__init__.py:114
111 from .datatools import S3
113 # includefile
--> 114 from .includefile import IncludeFile
116 # Decorators
117 from .decorators import step, _import_plugin_decorators
File ~/miniforge-pypy3/envs/env_openfl_latest_3.12/lib/python3.12/site-packages/metaflow/includefile.py:50
48 # From here on out, this is the IncludeFile implementation.
49 from .datatools import Local, S3
---> 50 from metaflow.plugins.azure.includefile_support import Azure
51 from metaflow.plugins.gcp.includefile_support import GS
53 DATACLIENTS = {
54 "local": Local,
55 "s3": S3,
56 "azure": Azure,
57 "gs": GS,
58 }
File ~/miniforge-pypy3/envs/env_openfl_latest_3.12/lib/python3.12/site-packages/metaflow/plugins/__init__.py:115
113 from .retry_decorator import RetryDecorator
114 from .resources_decorator import ResourcesDecorator
--> 115 from .aws.batch.batch_decorator import BatchDecorator
116 from .kubernetes.kubernetes_decorator import KubernetesDecorator
117 from .argo.argo_workflows_decorator import ArgoWorkflowsInternalDecorator
File ~/miniforge-pypy3/envs/env_openfl_latest_3.12/lib/python3.12/site-packages/metaflow/plugins/aws/batch/batch_decorator.py:8
5 import time
7 from metaflow import util
----> 8 from metaflow import R, current
10 from metaflow.decorators import StepDecorator
11 from metaflow.plugins import ResourcesDecorator
File ~/miniforge-pypy3/envs/env_openfl_latest_3.12/lib/python3.12/site-packages/metaflow/R.py:2
1 import os
----> 2 import imp
3 from tempfile import NamedTemporaryFile
5 from .util import to_bytes
ModuleNotFoundError: No module named 'imp'
Desktop (please complete the following information):
- OS: Ubuntu (WSL)
- OS Version: 22.04.3 LTS
- Python: 3.12
Additional Information:
- Issue is specific to Python 3.12 (no issues observed with Python 3.10 and 3.11)
- This issue is caused since imp module has been removed in Python 3.12 (Ref: https://docs.python.org/3/whatsnew/3.12.html#imp)
Additional Information After upgrading metaflow to latest 2.15.6 following issues are observed
Screenshot
Traceback
ImportError Traceback (most recent call last) Cell In[3], line 3 1 from copy import deepcopy ----> 3 from openfl.experimental.workflow.interface import FLSpec, Aggregator, Collaborator 4 from openfl.experimental.workflow.runtime import LocalRuntime 5 from openfl.experimental.workflow.placement import aggregator, collaborator
File ~/miniforge-pypy3/envs/env_openfl_latest_3.12/lib/python3.12/site-packages/openfl/experimental/workflow/interface/init.py:7 1 # Copyright 2020-2024 Intel Corporation 2 # SPDX-License-Identifier: Apache-2.0 5 """openfl.experimental.workflow.interface package.""" ----> 7 from openfl.experimental.workflow.interface.fl_spec import FLSpec 8 from openfl.experimental.workflow.interface.participants import Aggregator, Collaborator
File ~/miniforge-pypy3/envs/env_openfl_latest_3.12/lib/python3.12/site-packages/openfl/experimental/workflow/interface/fl_spec.py:16 13 if TYPE_CHECKING: 14 from openfl.experimental.workflow.runtime import FederatedRuntime, LocalRuntime, Runtime ---> 16 from openfl.experimental.workflow.utilities import ( 17 MetaflowInterface, 18 SerializationError, 19 aggregator_to_collaborator, 20 checkpoint, 21 collaborator_to_aggregator, 22 filter_attributes, 23 generate_artifacts, 24 should_transfer, 25 ) 28 class FLSpec: 29 """FLSpec Class 30 31 A class representing a Federated Learning Specification. It manages clones, (...) 39 _runtime (RuntimeType): The runtime of the flow. 40 """
File ~/miniforge-pypy3/envs/env_openfl_latest_3.12/lib/python3.12/site-packages/openfl/experimental/workflow/utilities/init.py:12 5 """openfl.experimental.workflow.utilities package.""" 7 from openfl.experimental.workflow.utilities.exceptions import ( 8 ResourcesAllocationError, 9 ResourcesNotAvailableError, 10 SerializationError, 11 ) ---> 12 from openfl.experimental.workflow.utilities.metaflow_utils import MetaflowInterface 13 from openfl.experimental.workflow.utilities.resources import get_number_of_gpus 14 from openfl.experimental.workflow.utilities.runtime_utils import ( 15 check_resource_allocation, 16 checkpoint, (...) 19 parse_attrs, 20 )
File ~/miniforge-pypy3/envs/env_openfl_latest_3.12/lib/python3.12/site-packages/openfl/experimental/workflow/utilities/metaflow_utils.py:23 21 import ray 22 from dill.source import getsource # nosec ---> 23 from metaflow.datastore import DATASTORES, FlowDataStore 24 from metaflow.datastore.exceptions import DataException, UnpicklableArtifactException 25 from metaflow.datastore.task_datastore import TaskDataStore, only_if_not_done, require_mode
ImportError: cannot import name 'DATASTORES' from 'metaflow.datastore' (/home/scngupta/miniforge-pypy3/envs/env_openfl_latest_3.12/lib/python3.12/site-packages/metaflow/datastore/init.py)
I can corroborate this. I am also getting the following error when using OpenFL:
ImportError: cannot import name 'DATASTORES' from 'metaflow.datastore'
metaflow 2.7 seems to be working it is unclear which version of metaflow OpenFL needs. Why is it not in the requirements ?
