LLaVAModel finetuning with llava-phi-3-mini-xtuner

[shockjiang]

llava_name_or_path='hub/llava-phi-3-mini-xtuner'
model = dict(
    type=LLaVAModel,
    freeze_llm=True,
    freeze_visual_encoder=True,
    pretrained_pth=llava_name_or_path,
    llm=dict(
        type=AutoModelForCausalLM.from_pretrained,
        pretrained_model_name_or_path=llm_name_or_path, attn_implementation="eager",
        trust_remote_code=True),
    visual_encoder=dict(
        type=CLIPVisionModel.from_pretrained,
        pretrained_model_name_or_path=visual_encoder_name_or_path))

when loadding model, the following error occurs:

[rank7]:     obj = obj_cls(**args)  # type: ignore                                                                             
[rank7]:   File "/home/jiangxiaoke/workspace/xtuner/xtuner/model/obj_llava.py", line 96, in __init__                           
[rank7]:     pretrained_state_dict = guess_load_checkpoint(pretrained_pth)                                                     
[rank7]:   File "/home/jiangxiaoke/workspace/xtuner/xtuner/model/utils.py", line 305, in guess_load_checkpoint                 
[rank7]:     state_dict = get_fp32_state_dict_from_zero_checkpoint(                                                            
[rank7]:   File "/home/jiangxiaoke/anaconda3/envs/xtuner/lib/python3.10/site-packages/deepspeed/utils/zero_to_fp32.py", line 52
1, in get_fp32_state_dict_from_zero_checkpoint                                                                                 
[rank7]:     return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)                    
[rank7]:   File "/home/jiangxiaoke/anaconda3/envs/xtuner/lib/python3.10/site-packages/deepspeed/utils/zero_to_fp32.py", line 20
4, in _get_fp32_state_dict_from_zero_checkpoint                                                                                
[rank7]:     optim_files = get_optim_files(ds_checkpoint_dir)                                                                  
[rank7]:   File "/home/jiangxiaoke/anaconda3/envs/xtuner/lib/python3.10/site-packages/deepspeed/utils/zero_to_fp32.py", line 88
, in get_optim_files                                                                                                           
[rank7]:     return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")                                                  
[rank7]:   File "/home/jiangxiaoke/anaconda3/envs/xtuner/lib/python3.10/site-packages/deepspeed/utils/zero_to_fp32.py", line 82
, in get_checkpoint_files                                                                                                      
[rank7]:     raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")                       
[rank7]: FileNotFoundError: can't find *_optim_states.pt files in directory 'hub/llava-phi-3-mini-xtuner'

how can I fix it? thank you!

[LZHgrla]

@shockjiang Please use this model for continue fine-tuning.

https://huggingface.co/xtuner/llava-phi-3-mini-pth/blob/main/model.pth

[shockjiang]

thx!

[shockjiang]

@LZHgrla I trid this:

model = dict(
    freeze_llm=True,
    freeze_visual_encoder=True,
    llm=dict(
        attn_implementation='eager',
        pretrained_model_name_or_path='hub/Phi-3-mini-4k-instruct',
        trust_remote_code=True,
        type='transformers.AutoModelForCausalLM.from_pretrained'),
    pretrained_pth='hub/llava-phi-3-mini-pth',
    type='xtuner.model.LLaVAModel',
    visual_encoder=dict(
        pretrained_model_name_or_path='hub/clip-vit-large-patch14-336',
        type='transformers.CLIPVisionModel.from_pretrained'))

It still gives me the error:

ile "/opt/conda/lib/python3.10/site-packages/deepspeed/utils/zero_to_fp32.py", line 82, in get_checkpoint_files
    raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
FileNotFoundError: can't find *_optim_states.pt files in directory 'hub/llava-phi-3-mini-pth'

what's wrong here?

[LZHgrla]

@shockjiang Please first download this model (https://huggingface.co/xtuner/llava-phi-3-mini-pth/blob/main/model.pth) and set the local path to pretrained_pth

[shockjiang]

@LZHgrla problem solved. key setting here is assign the model.pth's path to pretrained_pth, instead of its parent directory.

[Mikael17125]

hi, can I knew the exact syntax mine is still error:

model = dict(
    freeze_llm=True,
    freeze_visual_encoder=True,
    llm=dict(
        attn_implementation='eager',
        pretrained_model_name_or_path='/home/oem/xtuner/pretrained/phi-3/',
        type='transformers.AutoModelForCausalLM.from_pretrained',
        trust_remote_code=True),
    pretrained_pth='/home/oem/xtuner/pretrained/phi-3/',
    type='xtuner.model.LLaVAModel',
    visual_encoder=dict(
        type=CLIPVisionModel.from_pretrained,
        pretrained_model_name_or_path=visual_encoder_name_or_path))

it said I have no config.json

Traceback (most recent call last):
  File "/home/oem/xtuner/xtuner/tools/train.py", line 360, in <module>
    main()
  File "/home/oem/xtuner/xtuner/tools/train.py", line 356, in main
    runner.train()
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/runner/_flexible_runner.py", line 1182, in train
    self.strategy.prepare(
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/_strategy/deepspeed.py", line 381, in prepare
    model = self.build_model(model)
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/_strategy/base.py", line 306, in build_model
    model = MODELS.build(model)
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/registry/registry.py", line 570, in build
    return self.build_func(cfg, *args, **kwargs, registry=self)
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/registry/build_functions.py", line 232, in build_model_from_cfg
    return build_from_cfg(cfg, registry, default_args)
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/registry/build_functions.py", line 121, in build_from_cfg
    obj = obj_cls(**args)  # type: ignore
  File "/home/oem/xtuner/xtuner/model/llava.py", line 61, in __init__
    llm = self._dispatch_lm_model_cfg(llm, max_position_embeddings)
  File "/home/oem/xtuner/xtuner/model/llava.py", line 269, in _dispatch_lm_model_cfg
    llm_cfg = AutoConfig.from_pretrained(
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/transformers/models/auto/configuration_auto.py", line 965, in from_pretrained
    config_dict, unused_kwargs = PretrainedConfig.get_config_dict(pretrained_model_name_or_path, **kwargs)
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/transformers/configuration_utils.py", line 632, in get_config_dict
    config_dict, kwargs = cls._get_config_dict(pretrained_model_name_or_path, **kwargs)
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/transformers/configuration_utils.py", line 689, in _get_config_dict
    resolved_config_file = cached_file(
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/transformers/utils/hub.py", line 373, in cached_file
    raise EnvironmentError(
OSError: /home/oem/xtuner/pretrained/phi-3/ does not appear to have a file named config.json. Checkout 'https://huggingface.co//home/oem/xtuner/pretrained/phi-3//tree/None' for available files.

even when I specified the .pth it still give me error:

model = dict(
    freeze_llm=True,
    freeze_visual_encoder=True,
    llm=dict(
        attn_implementation='eager',
        pretrained_model_name_or_path='/home/oem/xtuner/pretrained/phi-3/model.pth',
        type='transformers.AutoModelForCausalLM.from_pretrained',
        trust_remote_code=True),
    pretrained_pth='/home/oem/xtuner/pretrained/phi-3/model.pth',
    type='xtuner.model.LLaVAModel',
    visual_encoder=dict(
        type=CLIPVisionModel.from_pretrained,
        pretrained_model_name_or_path=visual_encoder_name_or_path))

Traceback (most recent call last):
  File "/home/oem/xtuner/xtuner/tools/train.py", line 360, in <module>
    main()
  File "/home/oem/xtuner/xtuner/tools/train.py", line 356, in main
    runner.train()
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/runner/_flexible_runner.py", line 1182, in train
    self.strategy.prepare(
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/_strategy/deepspeed.py", line 381, in prepare
    model = self.build_model(model)
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/_strategy/base.py", line 306, in build_model
    model = MODELS.build(model)
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/registry/registry.py", line 570, in build
    return self.build_func(cfg, *args, **kwargs, registry=self)
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/registry/build_functions.py", line 232, in build_model_from_cfg
    return build_from_cfg(cfg, registry, default_args)
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/registry/build_functions.py", line 121, in build_from_cfg
    obj = obj_cls(**args)  # type: ignore
  File "/home/oem/xtuner/xtuner/model/llava.py", line 61, in __init__
    llm = self._dispatch_lm_model_cfg(llm, max_position_embeddings)
  File "/home/oem/xtuner/xtuner/model/llava.py", line 269, in _dispatch_lm_model_cfg
    llm_cfg = AutoConfig.from_pretrained(
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/transformers/models/auto/configuration_auto.py", line 965, in from_pretrained
    config_dict, unused_kwargs = PretrainedConfig.get_config_dict(pretrained_model_name_or_path, **kwargs)
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/transformers/configuration_utils.py", line 632, in get_config_dict
    config_dict, kwargs = cls._get_config_dict(pretrained_model_name_or_path, **kwargs)
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/transformers/configuration_utils.py", line 726, in _get_config_dict
    raise EnvironmentError(
OSError: It looks like the config file at '/home/oem/xtuner/pretrained/phi-3/model.pth' is not a valid JSON file.
Traceback (most recent call last):
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/transformers/configuration_utils.py", line 722, in _get_config_dict
    config_dict = cls._dict_from_json_file(resolved_config_file)
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/transformers/configuration_utils.py", line 825, in _dict_from_json_file
    text = reader.read()
  File "/home/oem/anaconda3/envs/xtuner-env/lib/python3.10/codecs.py", line 322, in decode
    (result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 64: invalid start byte