小目标检测训练过程中报错

[CashBai]

使用PP-YOLOE_plus_SOD-L，运行到验证集评估的阶段就开始报错

[05/21 14:21:36] ppdet.engine.callbacks INFO: Epoch: [0] [380/396] learning_rate: 0.000025 loss: 0.864219 loss_cls: 0.473560 loss_iou: 0.049636 loss_dfl: 0.528526 loss_l1: 0.049658 eta: 0:32:15 batch_cost: 0.2519 data_cost: 0.0014 ips: 7.9396 images/s, max_mem_reserved: 8056 MB, max_mem_allocated: 6962 MB
[05/21 14:21:39] ppdet.engine.callbacks INFO: Epoch: [0] [390/396] learning_rate: 0.000025 loss: 0.852183 loss_cls: 0.471614 loss_iou: 0.046057 loss_dfl: 0.529075 loss_l1: 0.049179 eta: 0:32:10 batch_cost: 0.2429 data_cost: 0.0010 ips: 8.2349 images/s, max_mem_reserved: 8056 MB, max_mem_allocated: 6962 MB
[05/21 14:21:44] ppdet.utils.checkpoint INFO: Save checkpoint: D:\Model\Second3\0
Traceback (most recent call last):
  File "C:\Paddle\PaddleX-release-3.0-rc1\paddlex\repo_manager\repos\PaddleDetection\tools\train.py", line 212, in <module>
    main()
  File "C:\Paddle\PaddleX-release-3.0-rc1\paddlex\repo_manager\repos\PaddleDetection\tools\train.py", line 208, in main
    run(FLAGS, cfg)
  File "C:\Paddle\PaddleX-release-3.0-rc1\paddlex\repo_manager\repos\PaddleDetection\tools\train.py", line 161, in run
    trainer.train(FLAGS.eval)
  File "C:\Paddle\PaddleX-release-3.0-rc1\paddlex\repo_manager\repos\PaddleDetection\ppdet\engine\trainer.py", line 665, in train
    self._compose_callback.on_epoch_end(self.status)
  File "C:\Paddle\PaddleX-release-3.0-rc1\paddlex\repo_manager\repos\PaddleDetection\ppdet\engine\callbacks.py", line 93, in on_epoch_end
    c.on_epoch_end(status)
  File "C:\Paddle\PaddleX-release-3.0-rc1\paddlex\repo_manager\repos\PaddleDetection\ppdet\engine\callbacks.py", line 273, in on_epoch_end
    self.model.export(output_dir=os.path.join(self.save_dir, save_name, "inference"), for_fd=True)
  File "C:\Paddle\PaddleX-release-3.0-rc1\paddlex\repo_manager\repos\PaddleDetection\ppdet\engine\trainer.py", line 1413, in export
    static_model, pruned_input_spec, input_spec = self._get_infer_cfg_and_input_spec(
                                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Paddle\PaddleX-release-3.0-rc1\paddlex\repo_manager\repos\PaddleDetection\ppdet\engine\trainer.py", line 1357, in _get_infer_cfg_and_input_spec
    static_model, pruned_input_spec = self._model_to_static(model, input_spec, prune_input)
                                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Paddle\PaddleX-release-3.0-rc1\paddlex\repo_manager\repos\PaddleDetection\ppdet\engine\trainer.py", line 1272, in _model_to_static
    input_spec, static_model.forward.main_program,
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\ProgramData\anaconda3\envs\Paddlex30rc1\Lib\site-packages\paddle\jit\dy2static\program_translator.py", line 1118, in main_program
    concrete_program = self.concrete_program
                       ^^^^^^^^^^^^^^^^^^^^^
  File "C:\ProgramData\anaconda3\envs\Paddlex30rc1\Lib\site-packages\paddle\jit\dy2static\program_translator.py", line 1002, in concrete_program
    return self.concrete_program_specify_input_spec(input_spec=None)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\ProgramData\anaconda3\envs\Paddlex30rc1\Lib\site-packages\paddle\jit\dy2static\program_translator.py", line 1046, in concrete_program_specify_input_spec
    concrete_program, _ = self.get_concrete_program(
                          ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\ProgramData\anaconda3\envs\Paddlex30rc1\Lib\site-packages\paddle\jit\dy2static\program_translator.py", line 935, in get_concrete_program
    concrete_program, partial_program_layer = self._program_cache[
                                              ^^^^^^^^^^^^^^^^^^^^
  File "C:\ProgramData\anaconda3\envs\Paddlex30rc1\Lib\site-packages\paddle\jit\dy2static\program_translator.py", line 1694, in __getitem__
    self._caches[item_id] = self._build_once(item)
                            ^^^^^^^^^^^^^^^^^^^^^^
  File "C:\ProgramData\anaconda3\envs\Paddlex30rc1\Lib\site-packages\paddle\jit\dy2static\program_translator.py", line 1631, in _build_once
    concrete_program = ConcreteProgram.pir_from_func_spec(
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\ProgramData\anaconda3\envs\Paddlex30rc1\Lib\site-packages\decorator.py", line 232, in fun
    return caller(func, *(extras + args), **kw)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\ProgramData\anaconda3\envs\Paddlex30rc1\Lib\site-packages\paddle\base\wrapped_decorator.py", line 40, in __impl__
    return wrapped_func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\ProgramData\anaconda3\envs\Paddlex30rc1\Lib\site-packages\paddle\base\dygraph\base.py", line 101, in __impl__
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "C:\ProgramData\anaconda3\envs\Paddlex30rc1\Lib\site-packages\paddle\jit\dy2static\program_translator.py", line 1302, in pir_from_func_spec
    error_data.raise_new_exception()
  File "C:\ProgramData\anaconda3\envs\Paddlex30rc1\Lib\site-packages\paddle\jit\dy2static\error.py", line 454, in raise_new_exception
    raise new_exception from None
AttributeError: In transformed code:

    File "C:\Paddle\PaddleX-release-3.0-rc1\paddlex\repo_manager\repos\PaddleDetection\ppdet\modeling\architectures\meta_arch.py", line 49, in forward
        if self.fuse_norm:
    File "C:\Paddle\PaddleX-release-3.0-rc1\paddlex\repo_manager\repos\PaddleDetection\ppdet\modeling\architectures\meta_arch.py", line 51, in forward
        if self.fuse_norm:
            image = inputs['image']
            self.inputs['image'] = image * self.scale + self.bias
            ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ <--- HERE
            self.inputs['im_shape'] = inputs['im_shape']
            self.inputs['scale_factor'] = inputs['scale_factor']

    File "C:\ProgramData\anaconda3\envs\Paddlex30rc1\Lib\site-packages\paddle\nn\layer\layers.py", line 1789, in __getattr__
        return object.__getattribute__(self, name)

    AttributeError: 'YOLOv3' object has no attribute 'inputs'
. Did you mean: 'get_concrete_program'?
Traceback (most recent call last):
  File "C:\Paddle\PaddleX-release-3.0-rc1\paddlex\utils\result_saver.py", line 28, in wrap
    result = func(self, *args, **kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Paddle\PaddleX-release-3.0-rc1\paddlex\engine.py", line 41, in run
    self._model.train()
  File "C:\Paddle\PaddleX-release-3.0-rc1\paddlex\model.py", line 119, in train
    trainer.train()
  File "C:\Paddle\PaddleX-release-3.0-rc1\paddlex\modules\base\trainer.py", line 98, in train
    train_result = self.pdx_model.train(**train_args)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Paddle\PaddleX-release-3.0-rc1\paddlex\repo_apis\PaddleDetection_api\object_det\model.py", line 143, in train
    return self.runner.train(
           ^^^^^^^^^^^^^^^^^^
  File "C:\Paddle\PaddleX-release-3.0-rc1\paddlex\repo_apis\PaddleDetection_api\object_det\runner.py", line 54, in train
    return self.run_cmd(
           ^^^^^^^^^^^^^
  File "C:\Paddle\PaddleX-release-3.0-rc1\paddlex\repo_apis\base\runner.py", line 355, in run_cmd
    raise CalledProcessError(
paddlex.utils.errors.others.CalledProcessError: Command ['C:\\ProgramData\\anaconda3\\envs\\Paddlex30rc1\\python.exe', 'tools/train.py', '--eval', '--config', 'C:\\Users\\user\\.paddlex\\tmpo4ygtkwi\\detmodel_PP-YOLOE_plus_SOD-L.yml', '--use_vdl', 'True', '--vdl_log_dir', 'D:\\Model\\Second3'] returned non-zero exit status 1.