Dreambooth + 2.1 = NaN Loss

[anotherjesse]

When you train with SD2.1 (not SD2.1 base), the loss instantly NaN

Steps:   0%|          | 0/3000 [00:00<?, ?it/s]
Steps:   0%|          | 0/3000 [00:07<?, ?it/s, loss=nan, lr=1e-6]

[anotherjesse]

The branch is https://github.com/replicate/dreambooth/tree/sd21

[Pwntus]

With revision 76cd0b242e1248353a8604bcefd1d331d73743518604812ff7d3ca4db84c8da3

0%|          | 0/100 [00:00<?, ?it/s]
Steps:   0%|          | 0/100 [00:00<?, ?it/s]
Steps:   0%|          | 0/100 [00:09<?, ?it/s, loss=nan, lr=1e-6]
Steps:   1%|          | 1/100 [00:09<16:00,  9.70s/it, loss=nan, lr=1e-6]
Steps:   2%|▏         | 2/100 [00:09<06:47,  4.16s/it, loss=nan, lr=1e-6]
Steps:   3%|▎         | 3/100 [00:10<03:51,  2.39s/it, loss=nan, lr=1e-6]

https://replicate.com/p/xg5mpxkbkvf6lbf6jzwusq3tf4

{
  "pretrained_model_name_or_path": "stabilityai/stable-diffusion-2-1",
  "pretrained_vae_name_or_path": null,
  "revision": "main",
  "tokenizer_name": null,
  "instance_data_dir": "cog_instance_data",
  "class_data_dir": "cog_class_data",
  "instance_prompt": "a photo of a <1> man",
  "class_prompt": "a photo of a man",
  "save_sample_prompt": null,
  "save_sample_negative_prompt": null,
  "n_save_sample": 0,
  "save_guidance_scale": 7.5,
  "save_infer_steps": 50,
  "pad_tokens": false,
  "with_prior_preservation": true,
  "prior_loss_weight": 1.0,
  "num_class_images": 10,
  "seed": 1337,
  "resolution": 512,
  "center_crop": false,
  "train_text_encoder": true,
  "train_batch_size": 1,
  "sample_batch_size": 4,
  "num_train_epochs": 1,
  "max_train_steps": 100,
  "gradient_accumulation_steps": 1,
  "gradient_checkpointing": false,
  "learning_rate": 1e-06,
  "scale_lr": false,
  "lr_scheduler": "constant",
  "lr_warmup_steps": 0,
  "use_8bit_adam": false,
  "adam_beta1": 0.9,
  "adam_beta2": 0.999,
  "adam_weight_decay": 0.01,
  "adam_epsilon": 1e-08,
  "max_grad_norm": 1.0,
  "push_to_hub": false,
  "hub_token": null,
  "hub_model_id": null,
  "save_interval": 10000,
  "save_min_steps": 0,
  "mixed_precision": "fp16",
  "not_cache_latents": false,
  "local_rank": -1,
  "output_dir": "checkpoints",
  "concepts_list": null,
  "logging_dir": "logs",
  "log_interval": 10,
  "hflip": false
}

[anotherjesse]

Progress by @Pwntus in #34 !