Hi all, I need your help to understand and debug this error please

Bug description

I've just updated SLEAP (v1.3.3 to v1.4.1, via conda package) on my computer and since then, every time I restart the training of my model (I am using the GUI), I get an error that blocks everything. No other changes have been made to my data or training parameters.

Expected behaviour

start training

Actual behaviour

Doesn't run training "an error occured while training centered. Your command line terminal would have more information about the training" Error in terminal reported in section 'how to reproduce'

Your personal set up

Software versions: SLEAP: 1.4.1a2 TensorFlow: 2.7.0 Numpy: 1.21.6 Python: 3.7.12 OS: Windows-10-10.0.22621-SP0

For SLEAP update, I've used conda package from https://sleap.ai/installation.html#upgrading-and-uninstalling and instructions: conda env remove -n sleap conda create -y -n sleap -c conda-forge -c nvidia -c sleap/label/dev -c sleap -c anaconda sleap=1.4.1

How to reproduce

Go to 'run training'. Please see copy of parameters:

{ "_pipeline": "multi-animal top-down", "_ensure_channels": "", "controller_port": 9000, "publish_port": 9001, "outputs.run_name_prefix": "", "outputs.runs_folder": "C:/Windows/System32\models", "outputs.tags": "", "outputs.checkpointing.best_model": true, "outputs.checkpointing.latest_model": false, "outputs.checkpointing.final_model": false, "outputs.tensorboard.write_logs": false, "_save_viz": true, "_predict_frames": "nothing", "max_instances": null, "model.heads.centroid.sigma": 2.5, "model.heads.centered_instance.anchor_part": "spine1", "model.heads.centered_instance.sigma": 2.5, "model.heads.centroid.anchor_part": "spine1", "model.heads.multi_class_topdown.confmaps.anchor_part": "spine1", "data.instance_cropping.center_on_part": "spine1" } { "data": { "labels": { "training_labels": "C:/Windows/System32/topdownapproachvideomodif.v006.slp", "validation_labels": null, "validation_fraction": 0.35, "test_labels": null, "split_by_inds": false, "training_inds": [ 836, 108, 267, 271, 419, 565, 460, 829, 296, 825, 19, 803, 608, 520, 637, 444, 331, 140, 138, 429, 103, 348, 28, 761, 476, 636, 36, 450, 686, 109, 746, 291, 321, 553, 157, 674, 226, 779, 253, 342, 604, 264, 851, 237, 743, 676, 611, 796, 552, 230, 766, 813, 389, 324, 139, 574, 751, 110, 114, 250, 748, 727, 469, 277, 555, 581, 628, 173, 692, 220, 217, 256, 653, 191, 716, 307, 207, 489, 56, 46, 294, 804, 789, 745, 362, 765, 501, 300, 159, 379, 212, 791, 783, 314, 398, 11, 706, 135, 504, 293, 839, 219, 83, 675, 216, 286, 662, 633, 32, 547, 279, 0, 164, 695, 39, 532, 284, 347, 184, 335, 234, 364, 648, 710, 698, 464, 25, 411, 756, 561, 753, 521, 685, 425, 228, 272, 62, 614, 189, 437, 455, 280, 436, 454, 376, 560, 591, 567, 566, 426, 564, 276, 269, 152, 79, 113, 52, 696, 778, 245, 786, 31, 251, 742, 443, 479, 484, 53, 141, 69, 283, 718, 66, 23, 353, 447, 328, 223, 830, 725, 817, 641, 126, 149, 281, 399, 439, 788, 837, 623, 622, 317, 615, 193, 194, 63, 29, 735, 438, 95, 345, 729, 225, 244, 470, 295, 835, 415, 646, 472, 43, 526, 241, 814, 694, 288, 418, 150, 203, 325, 403, 583, 434, 652, 408, 599, 326, 246, 525, 634, 365, 800, 147, 274, 7, 312, 679, 412, 625, 332, 486, 490, 499, 467, 240, 315, 744, 717, 642, 44, 534, 620, 769, 239, 666, 738, 810, 612, 182, 587, 171, 382, 93, 318, 601, 201, 131, 232, 248, 395, 100, 784, 224, 243, 156, 154, 705, 359, 26, 167, 349, 38, 166, 278, 84, 589, 262, 665, 827, 863, 704, 586, 136, 363, 297, 92, 795, 420, 158, 377, 647, 733, 519, 740, 433, 860, 500, 127, 10, 13, 831, 491, 181, 856, 631, 402, 658, 338, 86, 838, 195, 807, 730, 577, 682, 496, 388, 261, 736, 233, 687, 372, 360, 571, 650, 754, 651, 118, 544, 308, 497, 229, 495, 320, 826, 96, 818, 517, 120, 400, 590, 198, 155, 313, 254, 683, 205, 762, 559, 302, 196, 134, 709, 671, 572, 381, 606, 820, 471, 855, 350, 593, 257, 824, 655, 417, 50, 481, 285, 550, 80, 485, 85, 708, 857, 598, 597, 542, 654, 596, 673, 531, 340, 463, 197, 273, 815, 680, 797, 629, 221, 688, 579, 351, 619, 823, 627, 677, 558, 72, 741, 793, 664, 163, 462, 260, 446, 453, 792, 806, 570, 160, 1, 702, 459, 554, 355, 693, 145, 265, 422, 584, 518, 266, 128, 119, 617, 390, 144, 543, 71, 739, 767, 311, 6, 172, 828, 714, 672, 657, 77, 643, 380, 768, 101, 616, 255, 27, 310, 613, 722, 516, 707, 798, 728, 35, 343, 305, 737, 299, 190, 457, 757, 582, 177, 850, 54, 132, 523, 776, 5, 222, 461, 323, 9, 721, 329, 603, 834, 396, 70, 67, 391, 91, 213, 755, 507, 211, 162, 4, 371, 133, 607, 122, 759, 624, 242, 370, 40, 808, 151, 375, 621, 723, 231, 556, 540, 684, 847, 15, 106, 752, 645, 361, 512, 82, 777, 168, 845, 701, 148, 505, 451, 787, 569, 384, 282, 413, 819, 20, 117, 822, 252, 594, 319, 515, 852, 249, 861, 18, 247, 770, 337, 292, 60, 394, 179, 539, 699, 98, 859, 200 ], "validation_inds": [ 423, 369, 774, 358, 458, 330, 494, 715, 393, 538, 153, 568, 48, 289, 732, 33, 790, 580, 442, 270, 61, 218, 502, 258, 528, 498, 750, 635, 374, 805, 392, 711, 414, 649, 697, 410, 333, 37, 794, 47, 104, 129, 511, 832, 764, 94, 510, 660, 430, 811, 690, 180, 275, 161, 75, 24, 404, 432, 763, 681, 16, 115, 378, 97, 487, 592, 407, 334, 482, 42, 199, 480, 843, 816, 713, 368, 514, 3, 111, 640, 2, 846, 541, 802, 209, 799, 386, 548, 595, 609, 65, 290, 64, 691, 492, 670, 427, 73, 493, 204, 656, 421, 55, 210, 483, 537, 445, 585, 30, 726, 187, 700, 448, 573, 405, 530, 780, 401, 304, 545, 663, 506, 639, 45, 524, 842, 668, 397, 105, 678, 175, 116, 549, 782, 259, 137, 841, 57, 781, 644, 208, 49, 431, 74, 227, 734, 215, 747, 121, 68, 235, 186, 833, 107, 773, 303, 238, 58, 125, 758, 309, 513, 771, 533, 22, 202, 146, 78, 503, 263, 801, 712, 344, 488, 661, 848, 508, 165, 602, 853, 383, 775, 409, 336, 206, 456, 659, 475, 529, 8, 527, 406, 563, 183, 170, 618, 416, 578, 440, 87, 812, 367, 772, 90, 449, 435, 785, 188, 287, 130, 626, 477, 719, 17, 356, 474, 354, 21, 174, 81, 849, 638, 551, 509, 142, 102, 749, 466, 821, 840, 76, 298, 465, 341, 34, 99, 536, 720, 123, 557, 192, 88, 59, 605, 306, 185, 575, 169, 473, 703, 522, 600, 51, 301, 327, 428, 178, 667, 176, 562, 858, 112, 339, 373, 357, 12, 576, 441, 387, 535, 316, 478, 669, 632, 14, 689, 731, 468, 346, 854, 610, 760, 424, 809, 41, 352, 236, 214, 724, 385, 588, 452, 268, 124, 630, 143, 89, 322, 844, 862, 366, 546 ], "test_inds": null, "search_path_hints": [ "", "", "", "", "", "", "", "" ], "skeletons": [] }, "preprocessing": { "ensure_rgb": false, "ensure_grayscale": false, "imagenet_mode": null, "input_scaling": 0.5, "pad_to_stride": 16, "resize_and_pad_to_target": true, "target_height": 768, "target_width": 1024 }, "instance_cropping": { "center_on_part": "spine1", "crop_size": 0, "crop_size_detection_padding": 16 } }, "model": { "backbone": { "leap": null, "unet": { "stem_stride": null, "max_stride": 16, "output_stride": 2, "filters": 16, "filters_rate": 2.0, "middle_block": true, "up_interpolate": true, "stacks": 1 }, "hourglass": null, "resnet": null, "pretrained_encoder": null }, "heads": { "single_instance": null, "centroid": { "anchor_part": "spine1", "sigma": 2.5, "output_stride": 2, "loss_weight": 1.0, "offset_refinement": false }, "centered_instance": null, "multi_instance": null, "multi_class_bottomup": null, "multi_class_topdown": null }, "base_checkpoint": null }, "optimization": { "preload_data": true, "augmentation_config": { "rotate": true, "rotation_min_angle": -180.0, "rotation_max_angle": 180.0, "translate": false, "translate_min": -5, "translate_max": 5, "scale": true, "scale_min": 0.5, "scale_max": 2.0, "uniform_noise": true, "uniform_noise_min_val": 0.0, "uniform_noise_max_val": 10.0, "gaussian_noise": true, "gaussian_noise_mean": 5.0, "gaussian_noise_stddev": 1.0, "contrast": true, "contrast_min_gamma": 0.5, "contrast_max_gamma": 2.0, "brightness": true, "brightness_min_val": -20.0, "brightness_max_val": 20.0, "random_crop": false, "random_crop_height": 256, "random_crop_width": 256, "random_flip": false, "flip_horizontal": false }, "online_shuffling": true, "shuffle_buffer_size": 128, "prefetch": true, "batch_size": 8, "batches_per_epoch": 200, "min_batches_per_epoch": 200, "val_batches_per_epoch": 18, "min_val_batches_per_epoch": 10, "epochs": 200, "optimizer": "adam", "initial_learning_rate": 0.0001, "learning_rate_schedule": { "reduce_on_plateau": true, "reduction_factor": 0.5, "plateau_min_delta": 1e-06, "plateau_patience": 5, "plateau_cooldown": 3, "min_learning_rate": 1e-08 }, "hard_keypoint_mining": { "online_mining": false, "hard_to_easy_ratio": 2.0, "min_hard_keypoints": 2, "max_hard_keypoints": null, "loss_scale": 5.0 }, "early_stopping": { "stop_training_on_plateau": true, "plateau_min_delta": 1e-08, "plateau_patience": 20 } }, "outputs": { "save_outputs": true, "run_name": null, "run_name_prefix": "", "run_name_suffix": null, "runs_folder": "C:/Windows/System32\models", "tags": [ "" ], "save_visualizations": true, "delete_viz_images": true, "zip_outputs": false, "log_to_csv": true, "checkpointing": { "initial_model": false, "best_model": true, "every_epoch": false, "latest_model": false, "final_model": false }, "tensorboard": { "write_logs": false, "loss_frequency": "epoch", "architecture_graph": false, "profile_graph": false, "visualizations": true }, "zmq": { "subscribe_to_controller": true, "controller_address": "tcp://127.0.0.1:9000", "controller_polling_timeout": 10, "publish_updates": true, "publish_address": "tcp://127.0.0.1:9001" } }, "name": "", "description": "", "sleap_version": "1.3.3", "filename": "C:/Windows/System32\models\250120_200358.centroid.n=864\training_config.json" } { "data": { "labels": { "training_labels": "C:/Windows/System32/topdownapproachvideomodif.v006.slp", "validation_labels": null, "validation_fraction": 0.35, "test_labels": null, "split_by_inds": false, "training_inds": [ 32, 430, 465, 99, 174, 45, 252, 401, 169, 450, 191, 200, 539, 532, 198, 74, 396, 166, 142, 263, 280, 7, 540, 120, 246, 492, 353, 326, 347, 315, 289, 69, 491, 560, 400, 80, 57, 167, 195, 508, 29, 368, 168, 346, 413, 499, 374, 402, 516, 5, 212, 304, 13, 448, 393, 349, 314, 115, 409, 517, 173, 476, 305, 535, 126, 154, 60, 367, 481, 84, 376, 328, 220, 119, 555, 211, 434, 435, 161, 544, 410, 193, 245, 541, 379, 332, 217, 175, 244, 148, 552, 184, 352, 218, 117, 106, 163, 265, 43, 66, 453, 49, 459, 251, 242, 129, 223, 197, 426, 398, 140, 546, 172, 519, 21, 399, 12, 523, 359, 135, 500, 25, 145, 502, 170, 185, 180, 458, 266, 188, 455, 423, 36, 162, 468, 417, 530, 512, 151, 533, 103, 464, 362, 443, 320, 271, 310, 343, 397, 318, 243, 156, 404, 474, 277, 147, 415, 323, 303, 336, 364, 494, 378, 282, 324, 357, 522, 296, 213, 365, 205, 411, 425, 92, 48, 138, 424, 518, 283, 86, 391, 360, 1, 116, 478, 83, 275, 164, 104, 548, 286, 471, 98, 385, 366, 128, 377, 456, 493, 134, 247, 259, 144, 429, 210, 449, 371, 439, 155, 510, 545, 338, 511, 190, 97, 182, 268, 79, 262, 177, 390, 549, 28, 421, 354, 526, 41, 75, 165, 344, 225, 301, 267, 319, 6, 427, 14, 8, 256, 356, 270, 127, 102, 461, 67, 231, 329, 137, 333, 447, 408, 108, 559, 437, 39, 309, 386, 467, 284, 72, 369, 171, 520, 419, 207, 2, 76, 477, 42, 316, 394, 64, 483, 537, 230, 470, 238, 90, 527, 228, 422, 528, 59, 317, 94, 109, 442, 279, 30, 260, 27, 73, 33, 22, 291, 416, 105, 216, 232, 70, 489, 221, 340, 78, 50, 531, 118, 311, 525, 53, 547, 63, 406, 46, 186, 58, 255, 249, 521, 47, 294, 9, 542, 554, 107, 157, 509, 176, 507, 181, 350, 479, 132, 431, 203, 558, 114, 278, 543, 387, 222, 321, 308, 529, 375, 538, 17, 463, 77, 65, 15, 31, 10, 131, 488, 469, 438, 100, 96, 363, 227, 312, 452, 553, 325 ], "validation_inds": [ 348, 130, 226, 550, 208, 298, 101, 123, 240, 202, 248, 204, 0, 372, 440, 250, 187, 3, 158, 24, 405, 146, 124, 159, 345, 482, 224, 139, 55, 487, 179, 4, 122, 160, 496, 91, 335, 234, 35, 460, 237, 214, 327, 451, 498, 342, 239, 56, 189, 473, 441, 418, 287, 480, 274, 143, 201, 215, 414, 229, 20, 16, 254, 551, 307, 515, 433, 281, 269, 355, 504, 253, 407, 194, 192, 276, 299, 490, 339, 330, 113, 432, 206, 428, 121, 112, 54, 52, 141, 38, 153, 125, 501, 62, 88, 258, 514, 196, 388, 395, 199, 40, 341, 26, 236, 497, 484, 87, 337, 306, 293, 111, 462, 290, 273, 322, 361, 11, 19, 373, 412, 233, 392, 82, 380, 136, 297, 257, 209, 436, 261, 300, 536, 557, 485, 381, 183, 444, 534, 23, 457, 34, 389, 513, 313, 506, 503, 71, 219, 81, 495, 133, 466, 556, 334, 454, 403, 351, 178, 382, 370, 95, 475, 524, 505, 486, 68, 44, 61, 420, 51, 37, 384, 272, 446, 331, 110, 241, 89, 18, 472, 292, 152, 235, 358, 85, 93, 445, 288, 264, 302, 285, 383, 295, 149, 150 ], "test_inds": null, "search_path_hints": [ "", "", "", "" ], "skeletons": [] }, "preprocessing": { "ensure_rgb": false, "ensure_grayscale": false, "imagenet_mode": null, "input_scaling": 1.0, "pad_to_stride": 1, "resize_and_pad_to_target": true, "target_height": 768, "target_width": 1024 }, "instance_cropping": { "center_on_part": "spine1", "crop_size": 128, "crop_size_detection_padding": 16 } }, "model": { "backbone": { "leap": null, "unet": { "stem_stride": null, "max_stride": 32, "output_stride": 4, "filters": 24, "filters_rate": 1.5, "middle_block": true, "up_interpolate": true, "stacks": 1 }, "hourglass": null, "resnet": null, "pretrained_encoder": null }, "heads": { "single_instance": null, "centroid": null, "centered_instance": { "anchor_part": "spine1", "part_names": [ "nose", "left_ear", "right_ear", "left_shoulder", "right_shoulder", "left_hand", "right_hand", "left_back", "right_back", "tail", "head", "left_paw", "right_paw", "spine1", "spine2", "spine3" ], "sigma": 2.5, "output_stride": 4, "loss_weight": 1.0, "offset_refinement": false }, "multi_instance": null, "multi_class_bottomup": null, "multi_class_topdown": null }, "base_checkpoint": null }, "optimization": { "preload_data": true, "augmentation_config": { "rotate": true, "rotation_min_angle": -180.0, "rotation_max_angle": 180.0, "translate": false, "translate_min": -5, "translate_max": 5, "scale": true, "scale_min": 0.6, "scale_max": 1.4, "uniform_noise": true, "uniform_noise_min_val": 0.0, "uniform_noise_max_val": 10.0, "gaussian_noise": true, "gaussian_noise_mean": 5.0, "gaussian_noise_stddev": 1.0, "contrast": true, "contrast_min_gamma": 0.5, "contrast_max_gamma": 2.0, "brightness": true, "brightness_min_val": -20.0, "brightness_max_val": 20.0, "random_crop": false, "random_crop_height": 256, "random_crop_width": 256, "random_flip": false, "flip_horizontal": false }, "online_shuffling": true, "shuffle_buffer_size": 128, "prefetch": true, "batch_size": 8, "batches_per_epoch": 311, "min_batches_per_epoch": 200, "val_batches_per_epoch": 34, "min_val_batches_per_epoch": 10, "epochs": 200, "optimizer": "adam", "initial_learning_rate": 0.0001, "learning_rate_schedule": { "reduce_on_plateau": true, "reduction_factor": 0.5, "plateau_min_delta": 1e-06, "plateau_patience": 5, "plateau_cooldown": 3, "min_learning_rate": 1e-08 }, "hard_keypoint_mining": { "online_mining": true, "hard_to_easy_ratio": 2.0, "min_hard_keypoints": 2, "max_hard_keypoints": null, "loss_scale": 5.0 }, "early_stopping": { "stop_training_on_plateau": true, "plateau_min_delta": 1e-08, "plateau_patience": 20 } }, "outputs": { "save_outputs": true, "run_name": null, "run_name_prefix": "", "run_name_suffix": null, "runs_folder": "C:/Windows/System32\models", "tags": [ "" ], "save_visualizations": true, "delete_viz_images": true, "zip_outputs": false, "log_to_csv": true, "checkpointing": { "initial_model": false, "best_model": true, "every_epoch": false, "latest_model": false, "final_model": false }, "tensorboard": { "write_logs": false, "loss_frequency": "epoch", "architecture_graph": false, "profile_graph": false, "visualizations": true }, "zmq": { "subscribe_to_controller": true, "controller_address": "tcp://127.0.0.1:9000", "controller_polling_timeout": 10, "publish_updates": true, "publish_address": "tcp://127.0.0.1:9001" } }, "name": "", "description": "", "sleap_version": "1.3.3", "filename": "C:/Windows/System32\models\250106_184019.centered_instance.n=561\training_config.json" }

Please see error Resetting monitor window. Polling: C:/Windows/System32\models\250121_134227.centroid.n=864\viz\validation.*.png Start training centroid... ['sleap-train', 'C:\Users\cviel\AppData\Local\Temp\tmpasrp01rk\250121_134227_training_job.json', 'C:/Windows/System32/topdownapproachvideomodif.v006.slp', '--zmq', '--controller_port', '9000', '--publish_port', '9001', '--save_viz'] INFO:sleap.nn.training:Versions: SLEAP: 1.4.1a2 TensorFlow: 2.7.0 Numpy: 1.21.6 Python: 3.7.12 OS: Windows-10-10.0.22621-SP0 INFO:sleap.nn.training:Training labels file: C:/Windows/System32/topdownapproachvideomodif.v006.slp INFO:sleap.nn.training:Training profile: C:\Users\cviel\AppData\Local\Temp\tmpasrp01rk\250121_134227_training_job.json INFO:sleap.nn.training: INFO:sleap.nn.training:Arguments: INFO:sleap.nn.training:{ "training_job_path": "C:\Users\cviel\AppData\Local\Temp\tmpasrp01rk\250121_134227_training_job.json", "labels_path": "C:/Windows/System32/topdownapproachvideomodif.v006.slp", "video_paths": [ "" ], "val_labels": null, "test_labels": null, "base_checkpoint": null, "tensorboard": false, "save_viz": true, "zmq": true, "publish_port": 9001, "controller_port": 9000, "run_name": "", "prefix": "", "suffix": "", "cpu": false, "first_gpu": false, "last_gpu": false, "gpu": "auto" } INFO:sleap.nn.training: INFO:sleap.nn.training:Training job: INFO:sleap.nn.training:{ "data": { "labels": { "training_labels": "C:/Windows/System32/topdownapproachvideomodif.v006.slp", "validation_labels": null, "validation_fraction": 0.35, "test_labels": null, "split_by_inds": false, "training_inds": [ 836, 108, 267, 271, 419, 565, 460, 829, 296, 825, 19, 803, 608, 520, 637, 444, 331, 140, 138, 429, 103, 348, 28, 761, 476, 636, 36, 450, 686, 109, 746, 291, 321, 553, 157, 674, 226, 779, 253, 342, 604, 264, 851, 237, 743, 676, 611, 796, 552, 230, 766, 813, 389, 324, 139, 574, 751, 110, 114, 250, 748, 727, 469, 277, 555, 581, 628, 173, 692, 220, 217, 256, 653, 191, 716, 307, 207, 489, 56, 46, 294, 804, 789, 745, 362, 765, 501, 300, 159, 379, 212, 791, 783, 314, 398, 11, 706, 135, 504, 293, 839, 219, 83, 675, 216, 286, 662, 633, 32, 547, 279, 0, 164, 695, 39, 532, 284, 347, 184, 335, 234, 364, 648, 710, 698, 464, 25, 411, 756, 561, 753, 521, 685, 425, 228, 272, 62, 614, 189, 437, 455, 280, 436, 454, 376, 560, 591, 567, 566, 426, 564, 276, 269, 152, 79, 113, 52, 696, 778, 245, 786, 31, 251, 742, 443, 479, 484, 53, 141, 69, 283, 718, 66, 23, 353, 447, 328, 223, 830, 725, 817, 641, 126, 149, 281, 399, 439, 788, 837, 623, 622, 317, 615, 193, 194, 63, 29, 735, 438, 95, 345, 729, 225, 244, 470, 295, 835, 415, 646, 472, 43, 526, 241, 814, 694, 288, 418, 150, 203, 325, 403, 583, 434, 652, 408, 599, 326, 246, 525, 634, 365, 800, 147, 274, 7, 312, 679, 412, 625, 332, 486, 490, 499, 467, 240, 315, 744, 717, 642, 44, 534, 620, 769, 239, 666, 738, 810, 612, 182, 587, 171, 382, 93, 318, 601, 201, 131, 232, 248, 395, 100, 784, 224, 243, 156, 154, 705, 359, 26, 167, 349, 38, 166, 278, 84, 589, 262, 665, 827, 863, 704, 586, 136, 363, 297, 92, 795, 420, 158, 377, 647, 733, 519, 740, 433, 860, 500, 127, 10, 13, 831, 491, 181, 856, 631, 402, 658, 338, 86, 838, 195, 807, 730, 577, 682, 496, 388, 261, 736, 233, 687, 372, 360, 571, 650, 754, 651, 118, 544, 308, 497, 229, 495, 320, 826, 96, 818, 517, 120, 400, 590, 198, 155, 313, 254, 683, 205, 762, 559, 302, 196, 134, 709, 671, 572, 381, 606, 820, 471, 855, 350, 593, 257, 824, 655, 417, 50, 481, 285, 550, 80, 485, 85, 708, 857, 598, 597, 542, 654, 596, 673, 531, 340, 463, 197, 273, 815, 680, 797, 629, 221, 688, 579, 351, 619, 823, 627, 677, 558, 72, 741, 793, 664, 163, 462, 260, 446, 453, 792, 806, 570, 160, 1, 702, 459, 554, 355, 693, 145, 265, 422, 584, 518, 266, 128, 119, 617, 390, 144, 543, 71, 739, 767, 311, 6, 172, 828, 714, 672, 657, 77, 643, 380, 768, 101, 616, 255, 27, 310, 613, 722, 516, 707, 798, 728, 35, 343, 305, 737, 299, 190, 457, 757, 582, 177, 850, 54, 132, 523, 776, 5, 222, 461, 323, 9, 721, 329, 603, 834, 396, 70, 67, 391, 91, 213, 755, 507, 211, 162, 4, 371, 133, 607, 122, 759, 624, 242, 370, 40, 808, 151, 375, 621, 723, 231, 556, 540, 684, 847, 15, 106, 752, 645, 361, 512, 82, 777, 168, 845, 701, 148, 505, 451, 787, 569, 384, 282, 413, 819, 20, 117, 822, 252, 594, 319, 515, 852, 249, 861, 18, 247, 770, 337, 292, 60, 394, 179, 539, 699, 98, 859, 200 ], "validation_inds": [ 423, 369, 774, 358, 458, 330, 494, 715, 393, 538, 153, 568, 48, 289, 732, 33, 790, 580, 442, 270, 61, 218, 502, 258, 528, 498, 750, 635, 374, 805, 392, 711, 414, 649, 697, 410, 333, 37, 794, 47, 104, 129, 511, 832, 764, 94, 510, 660, 430, 811, 690, 180, 275, 161, 75, 24, 404, 432, 763, 681, 16, 115, 378, 97, 487, 592, 407, 334, 482, 42, 199, 480, 843, 816, 713, 368, 514, 3, 111, 640, 2, 846, 541, 802, 209, 799, 386, 548, 595, 609, 65, 290, 64, 691, 492, 670, 427, 73, 493, 204, 656, 421, 55, 210, 483, 537, 445, 585, 30, 726, 187, 700, 448, 573, 405, 530, 780, 401, 304, 545, 663, 506, 639, 45, 524, 842, 668, 397, 105, 678, 175, 116, 549, 782, 259, 137, 841, 57, 781, 644, 208, 49, 431, 74, 227, 734, 215, 747, 121, 68, 235, 186, 833, 107, 773, 303, 238, 58, 125, 758, 309, 513, 771, 533, 22, 202, 146, 78, 503, 263, 801, 712, 344, 488, 661, 848, 508, 165, 602, 853, 383, 775, 409, 336, 206, 456, 659, 475, 529, 8, 527, 406, 563, 183, 170, 618, 416, 578, 440, 87, 812, 367, 772, 90, 449, 435, 785, 188, 287, 130, 626, 477, 719, 17, 356, 474, 354, 21, 174, 81, 849, 638, 551, 509, 142, 102, 749, 466, 821, 840, 76, 298, 465, 341, 34, 99, 536, 720, 123, 557, 192, 88, 59, 605, 306, 185, 575, 169, 473, 703, 522, 600, 51, 301, 327, 428, 178, 667, 176, 562, 858, 112, 339, 373, 357, 12, 576, 441, 387, 535, 316, 478, 669, 632, 14, 689, 731, 468, 346, 854, 610, 760, 424, 809, 41, 352, 236, 214, 724, 385, 588, 452, 268, 124, 630, 143, 89, 322, 844, 862, 366, 546 ], "test_inds": null, "search_path_hints": [ "", "", "", "", "", "", "", "" ], "skeletons": [] }, "preprocessing": { "ensure_rgb": false, "ensure_grayscale": false, "imagenet_mode": null, "input_scaling": 0.5, "pad_to_stride": 16, "resize_and_pad_to_target": true, "target_height": 768, "target_width": 1024 }, "instance_cropping": { "center_on_part": "spine1", "crop_size": 0, "crop_size_detection_padding": 16 } }, "model": { "backbone": { "leap": null, "unet": { "stem_stride": null, "max_stride": 16, "output_stride": 2, "filters": 16, "filters_rate": 2.0, "middle_block": true, "up_interpolate": true, "stacks": 1 }, "hourglass": null, "resnet": null, "pretrained_encoder": null }, "heads": { "single_instance": null, "centroid": { "anchor_part": "spine1", "sigma": 2.5, "output_stride": 2, "loss_weight": 1.0, "offset_refinement": false }, "centered_instance": null, "multi_instance": null, "multi_class_bottomup": null, "multi_class_topdown": null }, "base_checkpoint": null }, "optimization": { "preload_data": true, "augmentation_config": { "rotate": true, "rotation_min_angle": -180.0, "rotation_max_angle": 180.0, "translate": false, "translate_min": -5, "translate_max": 5, "scale": true, "scale_min": 0.5, "scale_max": 2.0, "uniform_noise": true, "uniform_noise_min_val": 0.0, "uniform_noise_max_val": 10.0, "gaussian_noise": true, "gaussian_noise_mean": 5.0, "gaussian_noise_stddev": 1.0, "contrast": true, "contrast_min_gamma": 0.5, "contrast_max_gamma": 2.0, "brightness": true, "brightness_min_val": -20.0, "brightness_max_val": 20.0, "random_crop": false, "random_crop_height": 256, "random_crop_width": 256, "random_flip": false, "flip_horizontal": false }, "online_shuffling": true, "shuffle_buffer_size": 128, "prefetch": true, "batch_size": 8, "batches_per_epoch": 200, "min_batches_per_epoch": 200, "val_batches_per_epoch": 18, "min_val_batches_per_epoch": 10, "epochs": 200, "optimizer": "adam", "initial_learning_rate": 0.0001, "learning_rate_schedule": { "reduce_on_plateau": true, "reduction_factor": 0.5, "plateau_min_delta": 1e-06, "plateau_patience": 5, "plateau_cooldown": 3, "min_learning_rate": 1e-08 }, "hard_keypoint_mining": { "online_mining": false, "hard_to_easy_ratio": 2.0, "min_hard_keypoints": 2, "max_hard_keypoints": null, "loss_scale": 5.0 }, "early_stopping": { "stop_training_on_plateau": true, "plateau_min_delta": 1e-08, "plateau_patience": 20 } }, "outputs": { "save_outputs": true, "run_name": "250121_134227.centroid.n=864", "run_name_prefix": "", "run_name_suffix": "", "runs_folder": "C:/Windows/System32\models", "tags": [ "" ], "save_visualizations": true, "delete_viz_images": true, "zip_outputs": false, "log_to_csv": true, "checkpointing": { "initial_model": false, "best_model": true, "every_epoch": false, "latest_model": false, "final_model": false }, "tensorboard": { "write_logs": false, "loss_frequency": "epoch", "architecture_graph": false, "profile_graph": false, "visualizations": true }, "zmq": { "subscribe_to_controller": true, "controller_address": "tcp://127.0.0.1:9000", "controller_polling_timeout": 10, "publish_updates": true, "publish_address": "tcp://127.0.0.1:9001" } }, "name": "", "description": "", "sleap_version": "1.3.3", "filename": "C:\Users\cviel\AppData\Local\Temp\tmpasrp01rk\250121_134227_training_job.json" } INFO:sleap.nn.training: INFO:sleap.nn.training:Auto-selected GPU 0 with 16050 MiB of free memory. INFO:sleap.nn.training:Using GPU 0 for acceleration. INFO:sleap.nn.training:Disabled GPU memory pre-allocation. INFO:sleap.nn.training:System: GPUs: 1/1 available Device: /physical_device:GPU:0 Available: True Initialized: False Memory growth: True INFO:sleap.nn.training: INFO:sleap.nn.training:Initializing trainer... INFO:sleap.nn.training:Loading training labels from: C:/Windows/System32/topdownapproachvideomodif.v006.slp INFO:sleap.nn.training:Creating training and validation splits from validation fraction: 0.35 INFO:sleap.nn.training: Splits: Training = 562 / Validation = 302. INFO:sleap.nn.training:Setting up for training... INFO:sleap.nn.training:Setting up pipeline builders... INFO:sleap.nn.training:Setting up model... INFO:sleap.nn.training:Building test pipeline... 2025-01-21 13:42:36.419356: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX AVX2 To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. 2025-01-21 13:42:37.025550: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 13394 MB memory: -> device: 0, name: NVIDIA GeForce RTX 4090 Laptop GPU, pci bus id: 0000:01:00.0, compute capability: 8.9 INFO:sleap.nn.training:Loaded test example. [2.440s] INFO:sleap.nn.training: Input shape: (384, 512, 3) INFO:sleap.nn.training:Created Keras model. INFO:sleap.nn.training: Backbone: UNet(stacks=1, filters=16, filters_rate=2.0, kernel_size=3, stem_kernel_size=7, convs_per_block=2, stem_blocks=0, down_blocks=4, middle_block=True, up_blocks=3, up_interpolate=True, block_contraction=False) INFO:sleap.nn.training: Max stride: 16 INFO:sleap.nn.training: Parameters: 1,953,393 INFO:sleap.nn.training: Heads: INFO:sleap.nn.training: [0] = CentroidConfmapsHead(anchor_part='spine1', sigma=2.5, output_stride=2, loss_weight=1.0) INFO:sleap.nn.training: Outputs: INFO:sleap.nn.training: [0] = KerasTensor(type_spec=TensorSpec(shape=(None, 192, 256, 1), dtype=tf.float32, name=None), name='CentroidConfmapsHead/BiasAdd:0', description="created by layer 'CentroidConfmapsHead'") INFO:sleap.nn.training:Training from scratch INFO:sleap.nn.training:Setting up data pipelines... C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\albumentations\augmentations\transforms.py:1260: FutureWarning: This class has been deprecated. Please use RandomBrightnessContrast FutureWarning, INFO:sleap.nn.training:Training set: n = 562 INFO:sleap.nn.training:Validation set: n = 302 INFO:sleap.nn.training:Setting up optimization... INFO:sleap.nn.training: Learning rate schedule: LearningRateScheduleConfig(reduce_on_plateau=True, reduction_factor=0.5, plateau_min_delta=1e-06, plateau_patience=5, plateau_cooldown=3, min_learning_rate=1e-08) INFO:sleap.nn.training: Early stopping: EarlyStoppingConfig(stop_training_on_plateau=True, plateau_min_delta=1e-08, plateau_patience=20) INFO:sleap.nn.training:Setting up outputs... INFO:sleap.nn.callbacks:Training controller subscribed to: tcp://127.0.0.1:9000 (topic: ) INFO:sleap.nn.training: ZMQ controller subcribed to: tcp://127.0.0.1:9000 INFO:sleap.nn.callbacks:Progress reporter publishing on: tcp://127.0.0.1:9001 for: not_set INFO:sleap.nn.training: ZMQ progress reporter publish on: tcp://127.0.0.1:9001 INFO:sleap.nn.training:Created run path: C:/Windows/System32\models\250121_134227.centroid.n=864 INFO:sleap.nn.training:Setting up visualization... INFO:sleap.nn.training:Finished trainer set up. [5.9s] INFO:sleap.nn.training:Creating tf.data.Datasets for training data generation... 2025-01-21 13:43:41.008955: W tensorflow/core/framework/op_kernel.cc:1733] UNKNOWN: KeyError: dtype('float64') Traceback (most recent call last):

File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\tensorflow\python\ops\script_ops.py", line 273, in call return func(device, token, args)

File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\tensorflow\python\ops\script_ops.py", line 151, in call outputs = self._call(device, args)

File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\tensorflow\python\ops\script_ops.py", line 158, in _call ret = self._func(*args)

File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\tensorflow\python\autograph\impl\api.py", line 649, in wrapper return func(*args, **kwargs)

File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\sleap\nn\data\augmentation.py", line 240, in py_augment augmented = self.augmenter(image=img, keypoints=kps)

File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\albumentations\core\composition.py", line 210, in call data = t(**data)

File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\albumentations\core\transforms_interface.py", line 118, in call return self.apply_with_params(params, **kwargs)

File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\albumentations\core\transforms_interface.py", line 131, in apply_with_params res[key] = target_function(arg, **dict(params, **target_dependencies))

File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\albumentations\augmentations\transforms.py", line 1229, in apply return F.brightness_contrast_adjust(img, alpha, beta, self.brightness_by_max)

File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\albumentations\augmentations\functional.py", line 877, in brightness_contrast_adjust return _brightness_contrast_adjust_non_uint(img, alpha, beta, beta_by_max)

File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\albumentations\augmentations\utils.py", line 68, in wrapped_function return clip(func(img, *args, **kwargs), dtype, maxval)

File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\albumentations\augmentations\functional.py", line 845, in _brightness_contrast_adjust_non_uint max_value = MAX_VALUES_BY_DTYPE[dtype]

KeyError: dtype('float64')

Traceback (most recent call last): File "C:\Users\cviel\anaconda3\envs\sleap\Scripts\sleap-train-script.py", line 33, in sys.exit(load_entry_point('sleap==1.4.1a2', 'console_scripts', 'sleap-train')()) File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\sleap\nn\training.py", line 2030, in main trainer.train() File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\sleap\nn\training.py", line 928, in train training_ds = self.training_pipeline.make_dataset() File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\sleap\nn\data\pipelines.py", line 287, in make_dataset ds = transformer.transform_dataset(ds) File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\sleap\nn\data\normalization.py", line 342, in transform_dataset test_ex = next(iter(ds_input)) File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\tensorflow\python\data\ops\iterator_ops.py", line 800, in next return self._next_internal() File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\tensorflow\python\data\ops\iterator_ops.py", line 786, in _next_internal output_shapes=self._flat_output_shapes) File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\tensorflow\python\ops\gen_dataset_ops.py", line 2844, in iterator_get_next _ops.raise_from_not_ok_status(e, name) File "C:\Users\cviel\anaconda3\envs\sleap\lib\site-packages\tensorflow\python\framework\ops.py", line 7107, in raise_from_not_ok_status raise core._status_to_exception(e) from None # pylint: disable=protected-access tensorflow.python.framework.errors_impl.UnknownError: KeyError: dtype('float64') Traceback (most recent call last):