piper
piper copied to clipboard
rhasspy
Validation split & num test examples seems broken
I run python -m piper_train --dataset-dir ../../../voice/training --accelerator 'gpu' --gpus 1 --batch-size 5 --validation-split 0.05 --num-test-examples 7 --max_epochs 3220 --checkpoint-epochs 10 --precision 32 --max-phoneme-ids 600 --quality high --resume_from_checkpoint ../../../checkpoint-files/checkpoint.ckpt and eventually get the following error:
Traceback (most recent call last):
File "<frozen runpy>", line 198, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "/home/wilbur/bin/pipertts/piper/src/python/piper_train/__main__.py", line 147, in <module>
main()
File "/home/wilbur/bin/pipertts/piper/src/python/piper_train/__main__.py", line 124, in main
trainer.fit(model)
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 608, in fit
call._call_and_handle_interrupt(
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/trainer/call.py", line 38, in _call_and_handle_interrupt
return trainer_fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 650, in _fit_impl
self._run(model, ckpt_path=self.ckpt_path)
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 1112, in _run
results = self._run_stage()
^^^^^^^^^^^^^^^^^
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 1191, in _run_stage
self._run_train()
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 1204, in _run_train
self._run_sanity_check()
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 1276, in _run_sanity_check
val_loop.run()
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
self.advance(*args, **kwargs)
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/loops/dataloader/evaluation_loop.py", line 152, in advance
dl_outputs = self.epoch_loop.run(self._data_fetcher, dl_max_batches, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
self.advance(*args, **kwargs)
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py", line 137, in advance
output = self._evaluation_step(**kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py", line 234, in _evaluation_step
output = self.trainer._call_strategy_hook(hook_name, *kwargs.values())
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 1494, in _call_strategy_hook
output = fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/strategies/strategy.py", line 390, in validation_step
return self.model.validation_step(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/wilbur/bin/pipertts/piper/src/python/piper_train/vits/lightning.py", line 302, in validation_step
self.logger.experiment.add_audio(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AttributeError: 'ExperimentWriter' object has no attribute 'add_audio'
This error does not occur if either --num-test-examples or --validation-split are zero.
Did that, but now I've got another error:
Traceback (most recent call last):
File "<frozen runpy>", line 198, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "/home/wilbur/bin/pipertts/piper/src/python/piper_train/__main__.py", line 147, in <module>
main()
File "/home/wilbur/bin/pipertts/piper/src/python/piper_train/__main__.py", line 124, in main
trainer.fit(model)
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 608, in fit
call._call_and_handle_interrupt(
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/trainer/call.py", line 38, in _call_and_handle_interrupt
return trainer_fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 650, in _fit_impl
self._run(model, ckpt_path=self.ckpt_path)
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 1112, in _run
results = self._run_stage()
^^^^^^^^^^^^^^^^^
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 1191, in _run_stage
self._run_train()
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 1204, in _run_train
self._run_sanity_check()
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 1276, in _run_sanity_check
val_loop.run()
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
self.advance(*args, **kwargs)
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/loops/dataloader/evaluation_loop.py", line 152, in advance
dl_outputs = self.epoch_loop.run(self._data_fetcher, dl_max_batches, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
self.advance(*args, **kwargs)
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py", line 137, in advance
output = self._evaluation_step(**kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py", line 234, in _evaluation_step
output = self.trainer._call_strategy_hook(hook_name, *kwargs.values())
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/trainer/trainer.py", line 1494, in _call_strategy_hook
output = fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/pytorch_lightning/strategies/strategy.py", line 390, in validation_step
return self.model.validation_step(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/wilbur/bin/pipertts/piper/src/python/piper_train/vits/lightning.py", line 302, in validation_step
self.logger.experiment.add_audio(
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/tensorboardX/writer.py", line 885, in add_audio
audio(tag, snd_tensor, sample_rate=sample_rate), global_step, walltime)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/wilbur/bin/pipertts/.venv/lib64/python3.11/site-packages/tensorboardX/summary.py", line 453, in audio
assert tensor.ndim == 2, 'Input tensor should be 2 dimensional.'
^^^^^^^^^^^^^^^^
AssertionError: Input tensor should be 2 dimensional.
Should I file this as a separate issue?