mlops-with-vertex-ai
mlops-with-vertex-ai copied to clipboard
Published
20 hours ago •
GoogleCloudPlatform
GoogleCloudPlatform
`Concatenate` layer requires inputs with matching shapes
Problem
03-training-formalization.ipynb
ValueError: A `Concatenate` layer requires inputs with matching shapes except for the concatenation axis. Received: input_shape=[(None, 2), (None, 4), (7,), (None, 3), (None, 1), (None, 1), (6,), (None, 3), (None, 3), (None, 1), (None, 10)]
At:
_train_module_file = 'src/model_training/runner.py'
trainer = tfx.components.Trainer(
module_file=_train_module_file,
examples=transform.outputs['transformed_examples'],
schema=schema_importer.outputs['result'],
base_model=latest_model_resolver.outputs['latest_model'],
transform_graph=transform.outputs['transform_graph'],
hyperparameters=hyperparams_gen.outputs['hyperparameters'],
)
context.run(trainer, enable_cache=False)
Output:
running bdist_wheel
running build
running build_py
creating build
creating build/lib
copying trainer.py -> build/lib
copying runner.py -> build/lib
copying defaults.py -> build/lib
copying task.py -> build/lib
copying model.py -> build/lib
copying exporter.py -> build/lib
copying data.py -> build/lib
installing to /tmp/tmpaet8vft8
running install
running install_lib
copying build/lib/task.py -> /tmp/tmpaet8vft8
copying build/lib/model.py -> /tmp/tmpaet8vft8
copying build/lib/data.py -> /tmp/tmpaet8vft8
copying build/lib/runner.py -> /tmp/tmpaet8vft8
copying build/lib/defaults.py -> /tmp/tmpaet8vft8
copying build/lib/trainer.py -> /tmp/tmpaet8vft8
copying build/lib/exporter.py -> /tmp/tmpaet8vft8
running install_egg_info
running egg_info
creating tfx_user_code_Trainer.egg-info
writing tfx_user_code_Trainer.egg-info/PKG-INFO
writing dependency_links to tfx_user_code_Trainer.egg-info/dependency_links.txt
writing top-level names to tfx_user_code_Trainer.egg-info/top_level.txt
writing manifest file 'tfx_user_code_Trainer.egg-info/SOURCES.txt'
reading manifest file 'tfx_user_code_Trainer.egg-info/SOURCES.txt'
writing manifest file 'tfx_user_code_Trainer.egg-info/SOURCES.txt'
Copying tfx_user_code_Trainer.egg-info to /tmp/tmpaet8vft8/tfx_user_code_Trainer-0.0+5bd9ac13044cc46c88b21ccdcff2b74d85a1c15be76527e33008964fa7da7d12-py3.7.egg-info
running install_scripts
creating /tmp/tmpaet8vft8/tfx_user_code_Trainer-0.0+5bd9ac13044cc46c88b21ccdcff2b74d85a1c15be76527e33008964fa7da7d12.dist-info/WHEEL
creating '/tmp/tmpv9boslhs/tfx_user_code_Trainer-0.0+5bd9ac13044cc46c88b21ccdcff2b74d85a1c15be76527e33008964fa7da7d12-py3-none-any.whl' and adding '/tmp/tmpaet8vft8' to it
adding 'data.py'
adding 'defaults.py'
adding 'exporter.py'
adding 'model.py'
adding 'runner.py'
adding 'task.py'
adding 'trainer.py'
adding 'tfx_user_code_Trainer-0.0+5bd9ac13044cc46c88b21ccdcff2b74d85a1c15be76527e33008964fa7da7d12.dist-info/METADATA'
adding 'tfx_user_code_Trainer-0.0+5bd9ac13044cc46c88b21ccdcff2b74d85a1c15be76527e33008964fa7da7d12.dist-info/WHEEL'
adding 'tfx_user_code_Trainer-0.0+5bd9ac13044cc46c88b21ccdcff2b74d85a1c15be76527e33008964fa7da7d12.dist-info/top_level.txt'
adding 'tfx_user_code_Trainer-0.0+5bd9ac13044cc46c88b21ccdcff2b74d85a1c15be76527e33008964fa7da7d12.dist-info/RECORD'
removing /tmp/tmpaet8vft8
/opt/conda/lib/python3.7/site-packages/setuptools/command/install.py:37: SetuptoolsDeprecationWarning: setup.py install is deprecated. Use build and pip and other standards-based tools.
setuptools.SetuptoolsDeprecationWarning,
WARNING:absl:Examples artifact does not have payload_format custom property. Falling back to FORMAT_TF_EXAMPLE
WARNING:absl:Examples artifact does not have payload_format custom property. Falling back to FORMAT_TF_EXAMPLE
WARNING:absl:Examples artifact does not have payload_format custom property. Falling back to FORMAT_TF_EXAMPLE
WARNING: Ignoring invalid distribution -umpy (/opt/conda/lib/python3.7/site-packages)
Processing /tmp/tmp31cqmx79/tfx_user_code_Trainer-0.0+5bd9ac13044cc46c88b21ccdcff2b74d85a1c15be76527e33008964fa7da7d12-py3-none-any.whl
WARNING: Ignoring invalid distribution -umpy (/opt/conda/lib/python3.7/site-packages)
Installing collected packages: tfx-user-code-Trainer
Successfully installed tfx-user-code-Trainer-0.0+5bd9ac13044cc46c88b21ccdcff2b74d85a1c15be76527e33008964fa7da7d12
WARNING: Ignoring invalid distribution -umpy (/opt/conda/lib/python3.7/site-packages)
WARNING: Ignoring invalid distribution -umpy (/opt/conda/lib/python3.7/site-packages)
WARNING: Ignoring invalid distribution -umpy (/opt/conda/lib/python3.7/site-packages)
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
/tmp/ipykernel_10107/3433963696.py in <module>
10 )
11
---> 12 context.run(trainer, enable_cache=False)
~/.local/lib/python3.7/site-packages/tfx/orchestration/experimental/interactive/notebook_utils.py in run_if_ipython(*args, **kwargs)
29 # __IPYTHON__ variable is set by IPython, see
30 # https://ipython.org/ipython-doc/rel-0.10.2/html/interactive/reference.html#embedding-ipython.
---> 31 return fn(*args, **kwargs)
32 else:
33 logging.warning(
~/.local/lib/python3.7/site-packages/tfx/orchestration/experimental/interactive/interactive_context.py in run(self, component, enable_cache, beam_pipeline_args)
162 telemetry_utils.LABEL_TFX_RUNNER: runner_label,
163 }):
--> 164 execution_id = launcher.launch().execution_id
165
166 return execution_result.ExecutionResult(
~/.local/lib/python3.7/site-packages/tfx/orchestration/launcher/base_component_launcher.py in launch(self)
201 copy.deepcopy(execution_decision.input_dict),
202 execution_decision.output_dict,
--> 203 copy.deepcopy(execution_decision.exec_properties))
204
205 absl.logging.info('Running publisher for %s',
~/.local/lib/python3.7/site-packages/tfx/orchestration/launcher/in_process_component_launcher.py in _run_executor(self, execution_id, input_dict, output_dict, exec_properties)
72 # output_dict can still be changed, specifically properties.
73 executor.Do(
---> 74 copy.deepcopy(input_dict), output_dict, copy.deepcopy(exec_properties))
~/.local/lib/python3.7/site-packages/tfx/components/trainer/executor.py in Do(self, input_dict, output_dict, exec_properties)
176 # Train the model
177 absl.logging.info('Training model.')
--> 178 run_fn(fn_args)
179
180 # Note: If trained with multi-node distribution workers, it is the user
/tmp/tmp7ilh_lit/runner.py in run_fn(fn_args)
51 hyperparams=hyperparams,
52 log_dir=log_dir,
---> 53 base_model_dir=fn_args.base_model,
54 )
55
~/home/repositories/git/oonisim/python-programs/courses/mlops-with-vertex-ai/src/model_training/trainer.py in train(train_data_dir, eval_data_dir, tft_output_dir, hyperparams, log_dir, base_model_dir)
57 tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir)
58
---> 59 classifier = model.create_binary_classifier(tft_output, hyperparams)
60 if base_model_dir:
61 try:
~/home/repositories/git/oonisim/python-programs/courses/mlops-with-vertex-ai/src/model_training/model.py in create_binary_classifier(tft_output, hyperparams)
83 )
84
---> 85 return _create_binary_classifier(feature_vocab_sizes, hyperparams)
~/home/repositories/git/oonisim/python-programs/courses/mlops-with-vertex-ai/src/model_training/model.py in _create_binary_classifier(feature_vocab_sizes, hyperparams)
62 pass
63
---> 64 joined = keras.layers.Concatenate(name="combines_inputs")(layers)
65 feedforward_output = keras.Sequential(
66 [
/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py in error_handler(*args, **kwargs)
65 except Exception as e: # pylint: disable=broad-except
66 filtered_tb = _process_traceback_frames(e.__traceback__)
---> 67 raise e.with_traceback(filtered_tb) from None
68 finally:
69 del filtered_tb
/opt/conda/lib/python3.7/site-packages/keras/layers/merge.py in build(self, input_shape)
517 ranks = set(len(shape) for shape in shape_set)
518 if len(ranks) != 1:
--> 519 raise ValueError(err_msg)
520 # Get the only rank for the set.
521 (rank,) = ranks
ValueError: A `Concatenate` layer requires inputs with matching shapes except for the concatenation axis. Received: input_shape=[(None, 2), (None, 4), (7,), (None, 3), (None, 1), (None, 1), (6,), (None, 3), (None, 3), (None, 1), (None, 10)]
Environment
tfx 1.8.0
Soluition
import tfx.v1
Cause
keras.layers.experimental.preprocessing.CategoryEncoding generates the shape (N,), which cannot be concatenated with keras.layers.Embedding which has shape (None, M).
Need to use tf.keras.layers.CategoryEncoding which generates (None, N) shape.
Fix
keras - cannot concatenate Embedding and CategoryEncoding layers
src/model.py
def _create_binary_classifier(feature_vocab_sizes, hyperparams):
input_layers = create_model_inputs()
layers = []
for key in input_layers:
feature_name = features.original_name(key)
if feature_name in features.EMBEDDING_CATEGORICAL_FEATURES:
vocab_size = feature_vocab_sizes[feature_name]
embedding_size = features.EMBEDDING_CATEGORICAL_FEATURES[feature_name]
embedding_output = keras.layers.Embedding(
input_dim=vocab_size + 1,
output_dim=embedding_size,
name=f"{key}_embedding",
)(input_layers[key])
print(f"Shape of embed layer [{key}] has None [{embedding_output.shape}] output_dim is {embedding_size}")
layers.append(embedding_output)
elif feature_name in features.ONEHOT_CATEGORICAL_FEATURE_NAMES:
vocab_size = feature_vocab_sizes[feature_name]
#onehot_layer = keras.layers.experimental.preprocessing.CategoryEncoding(
# max_tokens=vocab_size,
# output_mode="binary",
# name=f"{key}_onehot",
#)(input_layers[key])
onehot_layer = tf.keras.layers.CategoryEncoding(
num_tokens=vocab_size,
output_mode="one_hot",
name=f"{key}_onehot",
)(input_layers[key])
#print(f"Shape of one hot layer [{key}] has None [{onehot_layer.shape}]")
layers.append(onehot_layer)
elif feature_name in features.NUMERICAL_FEATURE_NAMES:
numeric_layer = tf.expand_dims(input_layers[key], -1)
#print(f"Shape of numeric layer [{key}] has None [{numeric_layer.shape}]")
layers.append(numeric_layer)
else:
pass
joined = keras.layers.Concatenate(name="combines_inputs")(layers)
feedforward_output = keras.Sequential(
[
keras.layers.Dense(units, activation="relu")
for units in hyperparams["hidden_units"]
],
name="feedforward_network",
)(joined)
logits = keras.layers.Dense(units=1, name="logits")(feedforward_output)
model = keras.Model(inputs=input_layers, outputs=[logits])
return model
Result
Model generated.
classifier = model.create_binary_classifier(tft_output, hyperparams)
classifier.summary()
keras.utils.plot_model(
classifier,
show_shapes=True,
show_dtype=True,
to_file='model.png'
)
