OneTrainer icon indicating copy to clipboard operation
OneTrainer copied to clipboard

Published 20 hours ago • verified publisher icon Nerogar

[Bug]: Prodigy lr stays at Initial D value and not dynamic

Open betterftr opened this issue 6 months ago • 13 comments

What happened?

trying to train cascade tenc only with prodigy, but no matter what I tried to change the lr stays at the initial D value of the optimizer setting and does not move and the model does not learn anything. 1 image

(sd3_attention_mask branch)

What did you expect would happen?

config:

{ "__version": 4, "training_method": "FINE_TUNE", "model_type": "STABLE_CASCADE_1", "debug_mode": false, "debug_dir": "C:\train\debug", "workspace_dir": "C:\train", "cache_dir": "C:\train", "tensorboard": true, "tensorboard_expose": false, "continue_last_backup": true, "include_train_config": "NONE", "base_model_name": "C:\train\cascade\original-stable-cascade-prior", "weight_dtype": "BFLOAT_16", "output_dtype": "BFLOAT_16", "output_model_format": "SAFETENSORS", "output_model_destination": "C:\train", "gradient_checkpointing": true, "force_circular_padding": false, "concept_file_name": "training_concepts/test.json", "concepts": [ { "__version": 1, "image": { "__version": 0, "enable_crop_jitter": false, "enable_random_flip": false, "enable_fixed_flip": false, "enable_random_rotate": false, "enable_fixed_rotate": false, "random_rotate_max_angle": 0.0, "enable_random_brightness": false, "enable_fixed_brightness": false, "random_brightness_max_strength": 0.0, "enable_random_contrast": false, "enable_fixed_contrast": false, "random_contrast_max_strength": 0.0, "enable_random_saturation": false, "enable_fixed_saturation": false, "random_saturation_max_strength": 0.0, "enable_random_hue": false, "enable_fixed_hue": false, "random_hue_max_strength": 0.0, "enable_resolution_override": false, "resolution_override": "512", "enable_random_circular_mask_shrink": false, "enable_random_mask_rotate_crop": false }, "text": { "__version": 0, "prompt_source": "sample", "prompt_path": "", "enable_tag_shuffling": true, "tag_delimiter": ",", "keep_tags_count": 0 }, "name": "test", "path": "F:/test", "seed": 484668417, "enabled": true, "include_subdirectories": false, "image_variations": 1, "text_variations": 70, "balancing": 1.0, "balancing_strategy": "REPEATS", "loss_weight": 1.0 } ], "aspect_ratio_bucketing": true, "latent_caching": true, "clear_cache_before_training": false, "learning_rate_scheduler": "CONSTANT", "custom_learning_rate_scheduler": null, "scheduler_params": [], "learning_rate": 1.0, "learning_rate_warmup_steps": 10, "learning_rate_cycles": 1, "epochs": 1000, "batch_size": 40, "gradient_accumulation_steps": 1, "ema": "OFF", "ema_decay": 0.999, "ema_update_step_interval": 5, "dataloader_threads": 2, "train_device": "cuda", "temp_device": "cpu", "train_dtype": "BFLOAT_16", "fallback_train_dtype": "BFLOAT_16", "enable_autocast_cache": false, "only_cache": false, "resolution": "512", "attention_mechanism": "SDP", "align_prop": false, "align_prop_probability": 0.1, "align_prop_loss": "AESTHETIC", "align_prop_weight": 0.01, "align_prop_steps": 20, "align_prop_truncate_steps": 0.5, "align_prop_cfg_scale": 7.0, "mse_strength": 0.7, "mae_strength": 0.3, "vb_loss_strength": 1.0, "loss_weight_fn": "DEBIASED_ESTIMATION", "loss_weight_strength": 1.0, "dropout_probability": 0.0, "loss_scaler": "NONE", "learning_rate_scaler": "NONE", "offset_noise_weight": 0.0, "perturbation_noise_weight": 0.0, "rescale_noise_scheduler_to_zero_terminal_snr": false, "force_v_prediction": false, "force_epsilon_prediction": false, "min_noising_strength": 0.0, "max_noising_strength": 1.0, "timestep_distribution": "HEAVY_TAIL", "noising_weight": 1.3, "noising_bias": 0.0, "unet": { "__version": 0, "model_name": "", "include": true, "train": true, "stop_training_after": 0, "stop_training_after_unit": "NEVER", "learning_rate": null, "weight_dtype": "NONE", "dropout_probability": 0.0, "train_embedding": true }, "prior": { "__version": 0, "model_name": "", "include": true, "train": false, "stop_training_after": 0, "stop_training_after_unit": "NEVER", "learning_rate": 5e-05, "weight_dtype": "NONE", "dropout_probability": 0.0, "train_embedding": true }, "text_encoder": { "__version": 0, "model_name": "", "include": true, "train": true, "stop_training_after": 0, "stop_training_after_unit": "NEVER", "learning_rate": null, "weight_dtype": "NONE", "dropout_probability": 0.0, "train_embedding": true }, "text_encoder_layer_skip": 0, "text_encoder_2": { "__version": 0, "model_name": "", "include": true, "train": true, "stop_training_after": 30, "stop_training_after_unit": "EPOCH", "learning_rate": null, "weight_dtype": "NONE", "dropout_probability": 0.0, "train_embedding": true }, "text_encoder_2_layer_skip": 0, "text_encoder_3": { "__version": 0, "model_name": "", "include": true, "train": true, "stop_training_after": 30, "stop_training_after_unit": "EPOCH", "learning_rate": null, "weight_dtype": "NONE", "dropout_probability": 0.0, "train_embedding": true }, "text_encoder_3_layer_skip": 0, "vae": { "__version": 0, "model_name": "", "include": true, "train": true, "stop_training_after": null, "stop_training_after_unit": "NEVER", "learning_rate": null, "weight_dtype": "FLOAT_32", "dropout_probability": 0.0, "train_embedding": true }, "effnet_encoder": { "__version": 0, "model_name": "C:/train/cascade/effnet_encoder.safetensors", "include": true, "train": true, "stop_training_after": null, "stop_training_after_unit": "NEVER", "learning_rate": null, "weight_dtype": "NONE", "dropout_probability": 0.0, "train_embedding": true }, "decoder": { "__version": 0, "model_name": "stabilityai/stable-cascade", "include": true, "train": true, "stop_training_after": null, "stop_training_after_unit": "NEVER", "learning_rate": null, "weight_dtype": "NONE", "dropout_probability": 0.0, "train_embedding": true }, "decoder_text_encoder": { "__version": 0, "model_name": "", "include": true, "train": true, "stop_training_after": null, "stop_training_after_unit": "NEVER", "learning_rate": null, "weight_dtype": "NONE", "dropout_probability": 0.0, "train_embedding": true }, "decoder_vqgan": { "__version": 0, "model_name": "", "include": true, "train": true, "stop_training_after": null, "stop_training_after_unit": "NEVER", "learning_rate": null, "weight_dtype": "NONE", "dropout_probability": 0.0, "train_embedding": true }, "masked_training": false, "unmasked_probability": 0.1, "unmasked_weight": 0.1, "normalize_masked_area_loss": false, "embedding_learning_rate": null, "preserve_embedding_norm": false, "embedding": { "__version": 0, "uuid": "1154881e-7ea5-437a-9220-86b64dcd2509", "model_name": "", "placeholder": "", "train": true, "stop_training_after": null, "stop_training_after_unit": "NEVER", "token_count": 1, "initial_embedding_text": "*" }, "additional_embeddings": [], "embedding_weight_dtype": "FLOAT_32", "lora_model_name": "", "lora_rank": 16, "lora_alpha": 1.0, "lora_weight_dtype": "FLOAT_32", "bundle_additional_embeddings": true, "optimizer": { "__version": 0, "optimizer": "PRODIGY", "adam_w_mode": false, "alpha": null, "amsgrad": false, "beta1": 0.9, "beta2": 0.999, "beta3": null, "bias_correction": false, "block_wise": false, "capturable": false, "centered": false, "clip_threshold": null, "d0": 1e-07, "d_coef": 2.0, "dampening": null, "decay_rate": null, "decouple": true, "differentiable": false, "eps": 1e-08, "eps2": null, "foreach": false, "fsdp_in_use": false, "fused": false, "fused_back_pass": false, "growth_rate": "inf", "initial_accumulator_value": null, "is_paged": false, "log_every": null, "lr_decay": null, "max_unorm": null, "maximize": false, "min_8bit_size": null, "momentum": null, "nesterov": false, "no_prox": false, "optim_bits": null, "percentile_clipping": null, "r": null, "relative_step": false, "safeguard_warmup": true, "scale_parameter": false, "stochastic_rounding": true, "use_bias_correction": true, "use_triton": false, "warmup_init": false, "weight_decay": 0.05, "weight_lr_power": null }

Update: changed back to default settings and lr still seems not moving, samples are the same as first one, config: { "__version": 4, "training_method": "FINE_TUNE", "model_type": "STABLE_CASCADE_1", "debug_mode": false, "debug_dir": "debug", "workspace_dir": "C:\train", "cache_dir": "C:\train", "tensorboard": true, "tensorboard_expose": false, "continue_last_backup": false, "include_train_config": "NONE", "base_model_name": "C:\train\cascade\original-stable-cascade-prior", "weight_dtype": "BFLOAT_16", "output_dtype": "BFLOAT_16", "output_model_format": "SAFETENSORS", "output_model_destination": "C:\train", "gradient_checkpointing": true, "force_circular_padding": false, "concept_file_name": "training_concepts/test.json", "concepts": [ { "__version": 1, "image": { "__version": 0, "enable_crop_jitter": false, "enable_random_flip": false, "enable_fixed_flip": false, "enable_random_rotate": false, "enable_fixed_rotate": false, "random_rotate_max_angle": 0.0, "enable_random_brightness": false, "enable_fixed_brightness": false, "random_brightness_max_strength": 0.0, "enable_random_contrast": false, "enable_fixed_contrast": false, "random_contrast_max_strength": 0.0, "enable_random_saturation": false, "enable_fixed_saturation": false, "random_saturation_max_strength": 0.0, "enable_random_hue": false, "enable_fixed_hue": false, "random_hue_max_strength": 0.0, "enable_resolution_override": false, "resolution_override": "512", "enable_random_circular_mask_shrink": false, "enable_random_mask_rotate_crop": false }, "text": { "__version": 0, "prompt_source": "sample", "prompt_path": "", "enable_tag_shuffling": true, "tag_delimiter": ",", "keep_tags_count": 0 }, "name": "test", "path": "F:/test", "seed": 484668417, "enabled": true, "include_subdirectories": false, "image_variations": 1, "text_variations": 70, "balancing": 1.0, "balancing_strategy": "REPEATS", "loss_weight": 1.0 } ], "aspect_ratio_bucketing": true, "latent_caching": true, "clear_cache_before_training": false, "learning_rate_scheduler": "CONSTANT", "custom_learning_rate_scheduler": null, "scheduler_params": [], "learning_rate": 1.0, "learning_rate_warmup_steps": 10, "learning_rate_cycles": 1, "epochs": 1000, "batch_size": 40, "gradient_accumulation_steps": 1, "ema": "OFF", "ema_decay": 0.999, "ema_update_step_interval": 5, "dataloader_threads": 2, "train_device": "cuda", "temp_device": "cpu", "train_dtype": "BFLOAT_16", "fallback_train_dtype": "BFLOAT_16", "enable_autocast_cache": false, "only_cache": false, "resolution": "512", "attention_mechanism": "SDP", "align_prop": false, "align_prop_probability": 0.1, "align_prop_loss": "AESTHETIC", "align_prop_weight": 0.01, "align_prop_steps": 20, "align_prop_truncate_steps": 0.5, "align_prop_cfg_scale": 7.0, "mse_strength": 1.0, "mae_strength": 0.0, "vb_loss_strength": 1.0, "loss_weight_fn": "CONSTANT", "loss_weight_strength": 1.0, "dropout_probability": 0.0, "loss_scaler": "NONE", "learning_rate_scaler": "NONE", "offset_noise_weight": 0.0, "perturbation_noise_weight": 0.0, "rescale_noise_scheduler_to_zero_terminal_snr": false, "force_v_prediction": false, "force_epsilon_prediction": false, "min_noising_strength": 0.0, "max_noising_strength": 1.0, "timestep_distribution": "UNIFORM", "noising_weight": 0.0, "noising_bias": 0.0, "unet": { "__version": 0, "model_name": "", "include": true, "train": true, "stop_training_after": 0, "stop_training_after_unit": "NEVER", "learning_rate": null, "weight_dtype": "NONE", "dropout_probability": 0.0, "train_embedding": true }, "prior": { "__version": 0, "model_name": "", "include": true, "train": false, "stop_training_after": 0, "stop_training_after_unit": "NEVER", "learning_rate": null, "weight_dtype": "NONE", "dropout_probability": 0.0, "train_embedding": true }, "text_encoder": { "__version": 0, "model_name": "", "include": true, "train": true, "stop_training_after": 0, "stop_training_after_unit": "NEVER", "learning_rate": null, "weight_dtype": "NONE", "dropout_probability": 0.0, "train_embedding": true }, "text_encoder_layer_skip": 0, "text_encoder_2": { "__version": 0, "model_name": "", "include": true, "train": true, "stop_training_after": 30, "stop_training_after_unit": "EPOCH", "learning_rate": null, "weight_dtype": "NONE", "dropout_probability": 0.0, "train_embedding": true }, "text_encoder_2_layer_skip": 0, "text_encoder_3": { "__version": 0, "model_name": "", "include": true, "train": true, "stop_training_after": 30, "stop_training_after_unit": "EPOCH", "learning_rate": null, "weight_dtype": "NONE", "dropout_probability": 0.0, "train_embedding": true }, "text_encoder_3_layer_skip": 0, "vae": { "__version": 0, "model_name": "", "include": true, "train": true, "stop_training_after": null, "stop_training_after_unit": "NEVER", "learning_rate": null, "weight_dtype": "FLOAT_32", "dropout_probability": 0.0, "train_embedding": true }, "effnet_encoder": { "__version": 0, "model_name": "C:/train/cascade/effnet_encoder.safetensors", "include": true, "train": true, "stop_training_after": null, "stop_training_after_unit": "NEVER", "learning_rate": null, "weight_dtype": "NONE", "dropout_probability": 0.0, "train_embedding": true }, "decoder": { "__version": 0, "model_name": "stabilityai/stable-cascade", "include": true, "train": true, "stop_training_after": null, "stop_training_after_unit": "NEVER", "learning_rate": null, "weight_dtype": "NONE", "dropout_probability": 0.0, "train_embedding": true }, "decoder_text_encoder": { "__version": 0, "model_name": "", "include": true, "train": true, "stop_training_after": null, "stop_training_after_unit": "NEVER", "learning_rate": null, "weight_dtype": "NONE", "dropout_probability": 0.0, "train_embedding": true }, "decoder_vqgan": { "__version": 0, "model_name": "", "include": true, "train": true, "stop_training_after": null, "stop_training_after_unit": "NEVER", "learning_rate": null, "weight_dtype": "NONE", "dropout_probability": 0.0, "train_embedding": true }, "masked_training": false, "unmasked_probability": 0.1, "unmasked_weight": 0.1, "normalize_masked_area_loss": false, "embedding_learning_rate": null, "preserve_embedding_norm": false, "embedding": { "__version": 0, "uuid": "b61ef5e9-100f-422c-af62-3bfafc763091", "model_name": "", "placeholder": "", "train": true, "stop_training_after": null, "stop_training_after_unit": "NEVER", "token_count": 1, "initial_embedding_text": "*" }, "additional_embeddings": [], "embedding_weight_dtype": "FLOAT_32", "lora_model_name": "", "lora_rank": 16, "lora_alpha": 1.0, "lora_weight_dtype": "FLOAT_32", "bundle_additional_embeddings": true, "optimizer": { "__version": 0, "optimizer": "PRODIGY", "adam_w_mode": false, "alpha": null, "amsgrad": false, "beta1": 0.9, "beta2": 0.999, "beta3": null, "bias_correction": false, "block_wise": false, "capturable": false, "centered": false, "clip_threshold": null, "d0": 1e-07, "d_coef": 2.0, "dampening": null, "decay_rate": null, "decouple": true, "differentiable": false, "eps": 1e-08, "eps2": null, "foreach": false, "fsdp_in_use": false, "fused": false, "fused_back_pass": false, "growth_rate": "inf", "initial_accumulator_value": null, "is_paged": false, "log_every": null, "lr_decay": null, "max_unorm": null, "maximize": false, "min_8bit_size": null, "momentum": null, "nesterov": false, "no_prox": false, "optim_bits": null, "percentile_clipping": null, "r": null, "relative_step": false, "safeguard_warmup": true, "scale_parameter": false, "stochastic_rounding": true, "use_bias_correction": true, "use_triton": false, "warmup_init": false, "weight_decay": 0.05, "weight_lr_power": null

Relevant log output

No response

Output of pip freeze

(venv) C:\OneTrainer\venv>pip freeze absl-py==2.1.0 accelerate==0.30.1 aiohttp==3.9.5 aiosignal==1.3.1 antlr4-python3-runtime==4.9.3 async-timeout==4.0.3 attrs==23.2.0 bitsandbytes==0.43.1 certifi==2024.7.4 charset-normalizer==3.3.2 cloudpickle==3.0.0 colorama==0.4.6 coloredlogs==15.0.1 contourpy==1.2.1 customtkinter==5.2.2 cycler==0.12.1 dadaptation==3.2 darkdetect==0.8.0 -e git+https://github.com/huggingface/diffusers.git@dd4b731e68f88f58dfabfb68f28e00ede2bb90ae#egg=diffusers filelock==3.15.4 flatbuffers==24.3.25 fonttools==4.53.1 frozenlist==1.4.1 fsspec==2024.6.1 ftfy==6.2.0 grpcio==1.65.1 huggingface-hub==0.23.3 humanfriendly==10.0 idna==3.7 importlib_metadata==8.0.0 intel-openmp==2021.4.0 invisible-watermark==0.2.0 Jinja2==3.1.4 kiwisolver==1.4.5 lightning-utilities==0.11.5 lion-pytorch==0.1.4 Markdown==3.6 markdown-it-py==3.0.0 MarkupSafe==2.1.5 matplotlib==3.9.1 mdurl==0.1.2 -e git+https://github.com/Nerogar/mgds.git@5ea389a62408cff79a8f3a11b2f25c185c7c1c2e#egg=mgds mkl==2021.4.0 mpmath==1.3.0 multidict==6.0.5 networkx==3.3 numpy==1.26.4 omegaconf==2.3.0 onnxruntime-gpu==1.18.1 open-clip-torch==2.24.0 opencv-python==4.9.0.80 packaging==24.1 pillow==10.3.0 platformdirs==4.2.2 pooch==1.8.1 prodigyopt==1.0 protobuf==4.25.3 psutil==6.0.0 Pygments==2.18.0 pynvml==11.5.0 pyparsing==3.1.2 pyreadline3==3.4.1 python-dateutil==2.9.0.post0 pytorch-lightning==2.2.5 PyWavelets==1.6.0 PyYAML==6.0.1 regex==2024.5.15 requests==2.32.3 rich==13.7.1 safetensors==0.4.3 scalene==1.5.41 schedulefree==1.2.5 sentencepiece==0.2.0 six==1.16.0 sympy==1.13.1 tbb==2021.13.0 tensorboard==2.17.0 tensorboard-data-server==0.7.2 timm==1.0.7 tokenizers==0.19.1 torch==2.3.1+cu118 torchmetrics==1.4.0.post0 torchvision==0.18.1+cu118 tqdm==4.66.4 transformers==4.42.3 typing_extensions==4.12.2 urllib3==2.2.2 wcwidth==0.2.13 Werkzeug==3.0.3 xformers==0.0.27+cu118 yarl==1.9.4 zipp==3.19.2

betterftr avatar Jul 29 '24 16:07 betterftr