Training command
bucket_no_upscale = true
bucket_reso_steps = 64
cache_latents = true
caption_extension = ".txt"
clip_skip = 1
dynamo_backend = "no"
enable_bucket = true
epoch = 10
gradient_accumulation_steps = 1
huber_c = 0.1
huber_schedule = "snr"
learning_rate = 0.0001
learning_rate_te1 = 1e-5
learning_rate_te2 = 1e-5
logging_dir = "C:/kohya_ss/outputs/NerdGirlHaruki\log"
loss_type = "l2"
lr_scheduler = "adafactor"
lr_scheduler_args = []
lr_scheduler_num_cycles = 1
lr_scheduler_power = 1
max_bucket_reso = 2048
max_data_loader_n_workers = 0
max_timestep = 1000
max_token_length = 75
max_train_epochs = 10
max_train_steps = 3500
min_bucket_reso = 512
mixed_precision = "fp16"
multires_noise_discount = 0.3
noise_offset_type = "Original"
optimizer_type = "Adafactor"
output_dir = "C:/kohya_ss/outputs/NerdGirlHaruki\model"
output_name = "NerdGirlHiruki_V0.2"
pretrained_model_name_or_path = "C:/kohya_ss/models/pony-sdxl.safetensors"
prior_loss_weight = 1
resolution = "1024,1024"
sample_every_n_epochs = 1
sample_prompts = "C:/kohya_ss/outputs/NerdGirlHaruki\model\prompt.txt"
sample_sampler = "euler_a"
save_every_n_epochs = 1
save_model_as = "safetensors"
save_precision = "fp16"
train_batch_size = 2
train_data_dir = "C:/kohya_ss/outputs/NerdGirlHaruki\img"
xformers = true
After the first Epoch starts, I get the following Error (This config was working yesterday):
Traceback (most recent call last):
File "C:\kohya_ss\sd-scripts\sdxl_train.py", line 948, in
train(args)
File "C:\kohya_ss\sd-scripts\sdxl_train.py", line 703, in train
noise_pred = unet(noisy_latents, timesteps, text_embedding, vector_embedding)
File "C:\kohya_ss\venv\lib\site-packages\torch\nn\modules\module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "C:\kohya_ss\venv\lib\site-packages\torch\nn\modules\module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "C:\kohya_ss\venv\lib\site-packages\accelerate\utils\operations.py", line 680, in forward
return model_forward(*args, **kwargs)
File "C:\kohya_ss\venv\lib\site-packages\accelerate\utils\operations.py", line 668, in call
return convert_to_fp32(self.model_forward(*args, **kwargs))
File "C:\kohya_ss\venv\lib\site-packages\torch\amp\autocast_mode.py", line 16, in decorate_autocast
return func(*args, **kwargs)
File "C:\kohya_ss\sd-scripts\library\sdxl_original_unet.py", line 1111, in forward
h = call_module(module, h, emb, context)
File "C:\kohya_ss\sd-scripts\library\sdxl_original_unet.py", line 1095, in call_module
x = layer(x, context)
File "C:\kohya_ss\venv\lib\site-packages\torch\nn\modules\module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "C:\kohya_ss\venv\lib\site-packages\torch\nn\modules\module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "C:\kohya_ss\sd-scripts\library\sdxl_original_unet.py", line 750, in forward
hidden_states = block(hidden_states, context=encoder_hidden_states, timestep=timestep)
File "C:\kohya_ss\venv\lib\site-packages\torch\nn\modules\module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "C:\kohya_ss\venv\lib\site-packages\torch\nn\modules\module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "C:\kohya_ss\sd-scripts\library\sdxl_original_unet.py", line 673, in forward
output = self.forward_body(hidden_states, context, timestep)
File "C:\kohya_ss\sd-scripts\library\sdxl_original_unet.py", line 655, in forward_body
hidden_states = self.ff(self.norm3(hidden_states)) + hidden_states
File "C:\kohya_ss\venv\lib\site-packages\torch\nn\modules\module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "C:\kohya_ss\venv\lib\site-packages\torch\nn\modules\module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "C:\kohya_ss\sd-scripts\library\sdxl_original_unet.py", line 599, in forward
hidden_states = module(hidden_states)
File "C:\kohya_ss\venv\lib\site-packages\torch\nn\modules\module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "C:\kohya_ss\venv\lib\site-packages\torch\nn\modules\module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "C:\kohya_ss\sd-scripts\library\sdxl_original_unet.py", line 577, in forward
hidden_states, gate = self.proj(hidden_states).chunk(2, dim=-1)
File "C:\kohya_ss\venv\lib\site-packages\torch\nn\modules\module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "C:\kohya_ss\venv\lib\site-packages\torch\nn\modules\module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "C:\kohya_ss\venv\lib\site-packages\torch\nn\modules\linear.py", line 114, in forward
return F.linear(input, self.weight, self.bias)
torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 80.00 MiB. GPU 0 has a total capacty of 15.99 GiB of which 0 bytes is free. Of the allocated memory 29.41 GiB is allocated by PyTorch, and 786.23 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation. See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
steps: 0%| | 0/3720 [00:24<?, ?it/s]
Traceback (most recent call last):
File "C:\Users\Nick\AppData\Local\Programs\Python\Python310\lib\runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "C:\Users\Nick\AppData\Local\Programs\Python\Python310\lib\runpy.py", line 86, in run_code
exec(code, run_globals)
File "C:\kohya_ss\venv\Scripts\accelerate.EXE_main.py", line 7, in
sys.exit(main())
File "C:\kohya_ss\venv\lib\site-packages\accelerate\commands\accelerate_cli.py", line 47, in main
args.func(args)
File "C:\kohya_ss\venv\lib\site-packages\accelerate\commands\launch.py", line 1017, in launch_command
simple_launcher(args)
File "C:\kohya_ss\venv\lib\site-packages\accelerate\commands\launch.py", line 637, in simple_launcher
raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)
subprocess.CalledProcessError: Command '['C:\kohya_ss\venv\Scripts\python.exe', 'C:/kohya_ss/sd-scripts/sdxl_train.py', '--config_file', 'C:/kohya_ss/outputs/NerdGirlHaruki\model/config_dreambooth-20250306-191604.toml']' returned non-zero exit status 1.
kohya-ss
