ray-project
dreamerv3 failed at config.build()
What happened + What you expected to happen
code: `from ray.rllib.algorithms.dreamerv3 import DreamerV3Config
config = ( DreamerV3Config() .environment("CartPole-v1") .training( model_size="XS", training_ratio=1, model={ "batch_size_B": 1, "batch_length_T": 1, "horizon_H": 1, "gamma": 0.997, "model_size": "XS", }, ) )
algo = config.build() `
error:
AttributeError Traceback (most recent call last) Cell In[9], line 20 1 from ray.rllib.algorithms.dreamerv3 import DreamerV3Config 3 config = ( 4 DreamerV3Config() 5 .environment("CartPole-v1") (...) 17 ) 18 ) ---> 20 algo = config.build()
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/algorithms/algorithm_config.py:1137, in AlgorithmConfig.build(self, env, logger_creator, use_copy) 1134 if isinstance(self.algo_class, str): 1135 algo_class = get_trainable_cls(self.algo_class) -> 1137 return algo_class( 1138 config=self if not use_copy else copy.deepcopy(self), 1139 logger_creator=self.logger_creator, 1140 )
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/algorithms/algorithm.py:516, in Algorithm.init(self, config, env, logger_creator, **kwargs)
498 # Initialize common evaluation_metrics to nan, before they become
499 # available. We want to make sure the metrics are always present
500 # (although their values may be nan), so that Tune does not complain
501 # when we use these as stopping criteria.
502 self.evaluation_metrics = {
503 # TODO: Don't dump sampler results into top-level.
504 "evaluation": {
(...)
513 },
514 }
--> 516 super().init(
517 config=config,
518 logger_creator=logger_creator,
519 **kwargs,
520 )
522 # Check, whether training_iteration is still a tune.Trainable property
523 # and has not been overridden by the user in the attempt to implement the
524 # algos logic (this should be done now inside training_step).
525 try:
File ~/raytest/.venv/lib/python3.10/site-packages/ray/tune/trainable/trainable.py:161, in Trainable.init(self, config, logger_creator, storage) 157 logger.debug(f"StorageContext on the TRAINABLE:\n{storage}") 159 self._open_logfiles(stdout_file, stderr_file) --> 161 self.setup(copy.deepcopy(self.config)) 162 setup_time = time.time() - self._start_time 163 if setup_time > SETUP_TIME_THRESHOLD:
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/algorithms/dreamerv3/dreamerv3.py:511, in DreamerV3.setup(self, config) 509 @override(Algorithm) 510 def setup(self, config: AlgorithmConfig): --> 511 super().setup(config) 513 # Share RLModule between EnvRunner and single (local) Learner instance. 514 # To avoid possibly expensive weight synching step. 515 if self.config.share_module_between_env_runner_and_learner:
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/algorithms/algorithm.py:766, in Algorithm.setup(self, config) 764 module_spec = self.config.get_marl_module_spec(policy_dict=policy_dict) 765 learner_group_config = self.config.get_learner_group_config(module_spec) --> 766 self.learner_group = learner_group_config.build() 768 # check if there are modules to load from the module_spec 769 rl_module_ckpt_dirs = {}
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/core/learner/learner_group_config.py:102, in LearnerGroupConfig.build(self) 87 framework_hps = FrameworkHyperparameters( 88 eager_tracing=self.eager_tracing, 89 torch_compile_cfg=self.torch_compile_cfg, 90 torch_compile=self.torch_compile, 91 what_to_compile=self.torch_compile_what_to_compile, 92 ) 94 learner_spec = LearnerSpec( 95 learner_class=self.learner_class, 96 module_spec=self.module_spec, (...) 99 framework_hyperparameters=framework_hps, 100 ) --> 102 return self.learner_group_class(learner_spec)
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/core/learner/learner_group.py:99, in LearnerGroup.init(self, learner_spec, max_queue_len) 97 if self._is_local: 98 self._learner = learner_class(**learner_spec.get_params_dict()) ---> 99 self._learner.build() 100 self._worker_manager = None 101 self._in_queue = []
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/core/learner/tf/tf_learner.py:406, in TfLearner.build(self) 403 self._strategy = self._make_distributed_strategy_if_necessary() 405 with self._strategy.scope(): --> 406 super().build() 408 if self._enable_tf_function: 409 self._possibly_traced_update = tf.function( 410 self._untraced_update, reduce_retracing=True 411 )
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/core/learner/learner.py:981, in Learner.build(self)
978 self._is_built = True
980 # Build the module to be trained by this learner.
--> 981 self._module = self._make_module()
983 # Configure, construct, and register all optimizers needed to train
984 # self.module.
985 self.configure_optimizers()
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/core/learner/learner.py:1563, in Learner._make_module(self) 1561 module = self._module_obj 1562 else: -> 1563 module = self._module_spec.build() 1564 # If not already, convert to MultiAgentRLModule. 1565 module = module.as_multi_agent()
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/core/rl_module/marl_module.py:491, in MultiAgentRLModuleSpec.build(self, module_id) 488 return self.module_specs[module_id].build() 490 module_config = self.get_marl_config() --> 491 module = self.marl_module_class(module_config) 492 return module
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/core/rl_module/rl_module.py:377, in RLModule.init_subclass.
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/core/rl_module/marl_module.py:70, in MultiAgentRLModule.init(self, config) 64 def init(self, config: Optional["MultiAgentRLModuleConfig"] = None) -> None: 65 """Initializes a MultiagentRLModule instance. 66 67 Args: 68 config: The MultiAgentRLModuleConfig to use. 69 """ ---> 70 super().init(config or MultiAgentRLModuleConfig())
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/core/rl_module/rl_module.py:369, in RLModule.init(self, config)
361 if hasattr(self, "_is_setup") and self._is_setup:
362 raise RuntimeError(
363 "RLModule.setup() called twice within your RLModule implementation "
364 f"{self}! Make sure you are using the proper inheritance order "
(...)
367 "your custom constructor."
368 )
--> 369 self.setup()
370 self._is_setup = True
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/core/rl_module/marl_module.py:77, in MultiAgentRLModule.setup(self) 75 self.__check_module_configs(self.config.modules) 76 for module_id, module_spec in self.config.modules.items(): ---> 77 self._rl_modules[module_id] = module_spec.build()
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/core/rl_module/rl_module.py:104, in SingleAgentRLModuleSpec.build(self) 101 raise ValueError("Model config is not set.") 103 module_config = self.get_rl_module_config() --> 104 module = self.module_class(module_config) 105 return module
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/core/rl_module/rl_module.py:377, in RLModule.init_subclass.
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/core/rl_module/rl_module.py:377, in RLModule.init_subclass.
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/core/rl_module/tf/tf_rl_module.py:18, in TfRLModule.init(self, *args, **kwargs) 16 def init(self, *args, **kwargs) -> None: 17 tf.keras.Model.init(self) ---> 18 RLModule.init(self, *args, **kwargs)
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/core/rl_module/rl_module.py:369, in RLModule.init(self, config)
361 if hasattr(self, "_is_setup") and self._is_setup:
362 raise RuntimeError(
363 "RLModule.setup() called twice within your RLModule implementation "
364 f"{self}! Make sure you are using the proper inheritance order "
(...)
367 "your custom constructor."
368 )
--> 369 self.setup()
370 self._is_setup = True
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/algorithms/dreamerv3/dreamerv3_rl_module.py:104, in DreamerV3RLModule.setup(self) 98 else: 99 test_actions = np.tile( 100 np.expand_dims(self.config.action_space.sample(), (0, 1)), 101 reps=(B, T, 1), 102 ) --> 104 self.dreamer_model( 105 None, 106 _convert_to_tf(test_obs, dtype=tf.float32), 107 _convert_to_tf(test_actions, dtype=tf.float32), 108 _convert_to_tf(np.ones((B, T)), dtype=tf.bool), 109 _convert_to_tf(np.zeros((B * T,)), dtype=tf.bool), 110 ) 112 # Initialize the critic EMA net: 113 self.critic.init_ema()
File ~/raytest/.venv/lib/python3.10/site-packages/keras/src/utils/traceback_utils.py:70, in filter_traceback.tf.debugging.disable_traceback_filtering()
---> 70 raise e.with_traceback(filtered_tb) from None
71 finally:
72 del filtered_tb
File ~/raytest/.venv/lib/python3.10/site-packages/ray/rllib/algorithms/dreamerv3/tf/models/dreamer_model.py:127, in DreamerModel.call(self, inputs, observations, actions, is_first, start_is_terminated_BxT) 117 """Main call method for building this model in order to generate its variables. 118 119 Note: This method should NOT be used by users directly. It's purpose is only to 120 perform all forward passes necessary to define all variables of the DreamerV3. 121 """ 123 # Forward passes through all models are enough to build all trainable and 124 # non-trainable variables: 125 126 # World model. --> 127 results = self.world_model.forward_train( 128 observations, 129 actions, 130 is_first, 131 ) 132 # Actor. 133 _, distr_params = self.actor( 134 h=results["h_states_BxT"], 135 z=results["z_posterior_states_BxT"], 136 )
File /tmp/autograph_generated_fileypvnyjan.py:59, in outer_factory.
File /tmp/autograph_generated_fileyn83i0ke.py:18, in outer_factory.
File /tmp/autograph_generated_file5fo1gidn.py:23, in outer_factory.
File /tmp/autograph_generated_filevw4pfl2f.py:39, in outer_factory.
AttributeError: Exception encountered when calling layer 'dreamer_model' (type DreamerModel).
in user code:
File "/home/lailing2000/raytest/.venv/lib/python3.10/site-packages/ray/rllib/algorithms/dreamerv3/tf/models/world_model.py", line 290, in forward_train *
initial_states = tree.map_structure(
File "/home/lailing2000/raytest/.venv/lib/python3.10/site-packages/ray/rllib/algorithms/dreamerv3/tf/models/world_model.py", line 192, in get_initial_state *
_, z_probs = self.dynamics_predictor(h)
File "/home/lailing2000/raytest/.venv/lib/python3.10/site-packages/keras/src/utils/traceback_utils.py", line 70, in error_handler **
raise e.with_traceback(filtered_tb) from None
File "/tmp/__autograph_generated_file5fo1gidn.py", line 23, in tf__call
retval_ = ag__.converted_call(ag__.ld(self).representation_layer, (ag__.ld(out),), None, fscope)
File "/tmp/__autograph_generated_filevw4pfl2f.py", line 39, in tf__call
distribution = ag__.converted_call(ag__.ld(tfp).distributions.Independent, (ag__.converted_call(ag__.ld(tfp).distributions.OneHotCategorical, (), dict(logits=ag__.ld(logits)), fscope),), dict(reinterpreted_batch_ndims=1), fscope)
AttributeError: Exception encountered when calling layer 'dynamics_predictor' (type DynamicsPredictor).
in user code:
File "/home/lailing2000/raytest/.venv/lib/python3.10/site-packages/ray/rllib/algorithms/dreamerv3/tf/models/components/dynamics_predictor.py", line 84, in call *
return self.representation_layer(out)
File "/home/lailing2000/raytest/.venv/lib/python3.10/site-packages/keras/src/utils/traceback_utils.py", line 70, in error_handler **
raise e.with_traceback(filtered_tb) from None
File "/tmp/__autograph_generated_filevw4pfl2f.py", line 39, in tf__call
distribution = ag__.converted_call(ag__.ld(tfp).distributions.Independent, (ag__.converted_call(ag__.ld(tfp).distributions.OneHotCategorical, (), dict(logits=ag__.ld(logits)), fscope),), dict(reinterpreted_batch_ndims=1), fscope)
AttributeError: Exception encountered when calling layer 'z32x32' (type RepresentationLayer).
in user code:
File "/home/lailing2000/raytest/.venv/lib/python3.10/site-packages/ray/rllib/algorithms/dreamerv3/tf/models/components/representation_layer.py", line 112, in call *
distribution = tfp.distributions.Independent(
AttributeError: 'NoneType' object has no attribute 'distributions'
Call arguments received by layer 'z32x32' (type RepresentationLayer):
• inputs=tf.Tensor(shape=(None, 256), dtype=float32)
Call arguments received by layer 'dynamics_predictor' (type DynamicsPredictor):
• h=tf.Tensor(shape=(1, 256), dtype=float32)
Call arguments received by layer 'dreamer_model' (type DreamerModel): • inputs=None • observations=tf.Tensor(shape=(1, 64, 4), dtype=float32) • actions=tf.Tensor(shape=(1, 64, 2), dtype=float32) • is_first=tf.Tensor(shape=(1, 64), dtype=bool) • start_is_terminated_BxT=tf.Tensor(shape=(64,), dtype=bool)
Versions / Dependencies
win11 wsl
python 3.10 gymnasium==0.28.1 ray==2.9.1 tensorflow==2.15.0.post1 torch==2.2.0
Reproduction script
from ray.rllib.algorithms.dreamerv3 import DreamerV3Config
config = ( DreamerV3Config() .environment("CartPole-v1") .training( model_size="XS", training_ratio=1, model={ "batch_size_B": 1, "batch_length_T": 1, "horizon_H": 1, "gamma": 0.997, "model_size": "XS", }, ) )
algo = config.build()
Issue Severity
High: It blocks me from completing my task.
I've had the same issue. I've had to manipulate the package versions. I'm still having some issues w.r.t gymnasium compatibility, but hope, maybe this piece of information will help you out. Here are my package versions (poetry).
-> python version is set to 3.9.18
[tool.poetry.dependencies] python = ">=3.9,<3.12" ray = {extras = ["rllib"], version = "2.9.0"} mypy = "^1.4.1" pyyaml = "^6.0.1" torch = "^2.0.1" gputil = "^1.4.0" gymnasium = "0.28.1" glfw = "^2.6.2" click = "^8.1.6" black = "^23.7.0" hydra-core = "^1.3.2" tensorflow-estimator = "2.15.0" tensorflow-io-gcs-filesystem = "0.34.0" tensorflow-macos = "2.15.0" tensorflow-probability = "0.23.0"
I am still having this issue
from ray.rllib.algorithms.dreamerv3.dreamerv3 import DreamerV3Config
config = (
DreamerV3Config()
.environment("CartPole-v1")
.training(
model_size="XS",
training_ratio=1024,
)
)
algo = config.build()
I did install tensorflow-probability
(rayrltf) hinson@hinson-tower:~/projects/rl$ /home/hinson/anaconda3/envs/rayrltf/bin/python /home/hinson/projects/rl/dreamerv3.py
/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/gymnasium/spaces/box.py:130: UserWarning: WARN: Box bound precision lowered by casting to float32
gym.logger.warn(f"Box bound precision lowered by casting to {self.dtype}")
/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/gymnasium/utils/passive_env_checker.py:164: UserWarning: WARN: The obs returned by the `reset()` method was expecting numpy array dtype to be float32, actual type: float64
logger.warn(
/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/gymnasium/utils/passive_env_checker.py:188: UserWarning: WARN: The obs returned by the `reset()` method is not within the observation space.
logger.warn(f"{pre} is not within the observation space.")
/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/ray/rllib/algorithms/algorithm.py:516: RayDeprecationWarning: This API is deprecated and may be removed in future Ray releases. You could suppress this warning by setting env variable PYTHONWARNINGS="ignore::DeprecationWarning"
`UnifiedLogger` will be removed in Ray 2.7.
return UnifiedLogger(config, logdir, loggers=None)
/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/ray/tune/logger/unified.py:53: RayDeprecationWarning: This API is deprecated and may be removed in future Ray releases. You could suppress this warning by setting env variable PYTHONWARNINGS="ignore::DeprecationWarning"
The `JsonLogger interface is deprecated in favor of the `ray.tune.json.JsonLoggerCallback` interface and will be removed in Ray 2.7.
self._loggers.append(cls(self.config, self.logdir, self.trial))
/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/ray/tune/logger/unified.py:53: RayDeprecationWarning: This API is deprecated and may be removed in future Ray releases. You could suppress this warning by setting env variable PYTHONWARNINGS="ignore::DeprecationWarning"
The `CSVLogger interface is deprecated in favor of the `ray.tune.csv.CSVLoggerCallback` interface and will be removed in Ray 2.7.
self._loggers.append(cls(self.config, self.logdir, self.trial))
/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/ray/tune/logger/unified.py:53: RayDeprecationWarning: This API is deprecated and may be removed in future Ray releases. You could suppress this warning by setting env variable PYTHONWARNINGS="ignore::DeprecationWarning"
The `TBXLogger interface is deprecated in favor of the `ray.tune.tensorboardx.TBXLoggerCallback` interface and will be removed in Ray 2.7.
self._loggers.append(cls(self.config, self.logdir, self.trial))
/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/keras/src/layers/layer.py:372: UserWarning: `build()` was called on layer 'dreamer_model', however the layer does not have a `build()` method implemented and it looks like it has unbuilt state. This will cause the layer to be marked as built, despite not being actually built, which may cause failures down the line. Make sure to implement a proper `build()` method.
warnings.warn(
Traceback (most recent call last):
File "/home/hinson/projects/rl/dreamerv3.py", line 24, in <module>
algo = config.build()
File "/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/ray/rllib/algorithms/algorithm_config.py", line 869, in build
return algo_class(
File "/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/ray/rllib/algorithms/algorithm.py", line 545, in __init__
super().__init__(
File "/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/ray/tune/trainable/trainable.py", line 158, in __init__
self.setup(copy.deepcopy(self.config))
File "/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/ray/rllib/algorithms/dreamerv3/dreamerv3.py", line 492, in setup
super().setup(config)
File "/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/ray/rllib/algorithms/algorithm.py", line 747, in setup
self.learner_group = self.config.build_learner_group(
File "/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/ray/rllib/algorithms/algorithm_config.py", line 1130, in build_learner_group
learner_group = LearnerGroup(config=self, module_spec=rl_module_spec)
File "/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/ray/rllib/core/learner/learner_group.py", line 123, in __init__
self._learner.build()
File "/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/ray/rllib/core/learner/tf/tf_learner.py", line 399, in build
super().build()
File "/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/ray/rllib/core/learner/learner.py", line 327, in build
self.configure_optimizers()
File "/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/ray/rllib/core/learner/learner.py", line 441, in configure_optimizers
self.configure_optimizers_for_module(module_id=module_id, config=config)
File "/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/ray/rllib/algorithms/dreamerv3/tf/dreamerv3_tf_learner.py", line 57, in configure_optimizers_for_module
self.register_optimizer(
File "/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/ray/rllib/core/learner/learner.py", line 373, in register_optimizer
self._check_registered_optimizer(optimizer, params)
File "/home/hinson/anaconda3/envs/rayrltf/lib/python3.10/site-packages/ray/rllib/core/learner/tf/tf_learner.py", line 306, in _check_registered_optimizer
raise ValueError(
ValueError: One of the parameters (<KerasVariable shape=(4, 256), dtype=float32, path=dreamer_model/vector_encoder/dense/kernel>) in the registered optimizer is not a tf.Variable!
Here is my env
Package Version
---------------------------- -----------
absl-py 2.1.0
aiosignal 1.3.1
astunparse 1.6.3
attrs 23.2.0
certifi 2024.6.2
charset-normalizer 3.3.2
click 8.1.7
cloudpickle 3.0.0
decorator 5.1.1
dm-tree 0.1.8
Farama-Notifications 0.0.4
filelock 3.15.4
flatbuffers 24.3.25
frozenlist 1.4.1
fsspec 2024.6.1
gast 0.6.0
google-pasta 0.2.0
grpcio 1.64.1
gymnasium 0.28.1
h5py 3.11.0
idna 3.7
imageio 2.34.2
jax-jumpy 1.0.0
jsonschema 4.22.0
jsonschema-specifications 2023.12.1
keras 3.4.1
lazy_loader 0.4
libclang 18.1.1
lz4 4.3.3
Markdown 3.6
markdown-it-py 3.0.0
MarkupSafe 2.1.5
mdurl 0.1.2
ml-dtypes 0.3.2
msgpack 1.0.8
namex 0.0.8
networkx 3.3
numpy 1.26.4
opt-einsum 3.3.0
optree 0.11.0
packaging 24.1
pandas 2.2.2
pillow 10.3.0
pip 24.0
protobuf 4.25.3
pyarrow 16.1.0
Pygments 2.18.0
python-dateutil 2.9.0.post0
pytz 2024.1
PyYAML 6.0.1
ray 2.31.0
referencing 0.35.1
requests 2.32.3
rich 13.7.1
rpds-py 0.18.1
scikit-image 0.24.0
scipy 1.14.0
setuptools 69.5.1
shellingham 1.5.4
six 1.16.0
tensorboard 2.16.2
tensorboard-data-server 0.7.2
tensorboardX 2.6.2.2
tensorflow 2.16.2
tensorflow-io-gcs-filesystem 0.37.0
tensorflow-probability 0.24.0
termcolor 2.4.0
tf_keras 2.16.0
tifffile 2024.6.18
typer 0.12.3
typing_extensions 4.12.2
tzdata 2024.1
urllib3 2.2.2
Werkzeug 3.0.3
wheel 0.43.0
wrapt 1.16.0
