13B-chat量化之后推理报错

[LinXin04]

model = AutoPeftModelForCausalLM.from_pretrained("./output", device_map="auto", torch_dtype=torch.float16, trust_remote_code=True) model = model.quantize(4).cuda() model.save_pretrained('./quan') print("quan done")

model = AutoModelForCausalLM.from_pretrained("./quan", device_map="auto", trust_remote_code=True)

报错：

Traceback (most recent call last): File "/ossfs/node_38300464/workspace/Baichuan2-main/fine-tune/test.py", line 26, in #model = AutoModelForCausalLM.from_pretrained("./quan", device_map="auto", trust_remote_code=True) File "/root/miniconda3/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py", line 558, in from_pretrained return model_class.from_pretrained( File "/root/.cache/huggingface/modules/transformers_modules/quan/modeling_baichuan.py", line 670, in from_pretrained return super(BaichuanForCausalLM, cls).from_pretrained(pretrained_model_name_or_path, *model_args, File "/root/miniconda3/lib/python3.10/site-packages/transformers/modeling_utils.py", line 3180, in from_pretrained ) = cls._load_pretrained_model( File "/root/miniconda3/lib/python3.10/site-packages/transformers/modeling_utils.py", line 3568, in _load_pretrained_model new_error_msgs, offload_index, state_dict_index = _load_state_dict_into_meta_model( File "/root/miniconda3/lib/python3.10/site-packages/transformers/modeling_utils.py", line 745, in _load_state_dict_into_meta_model set_module_tensor_to_device(model, param_name, param_device, **set_module_kwargs) File "/root/miniconda3/lib/python3.10/site-packages/accelerate/utils/modeling.py", line 285, in set_module_tensor_to_device raise ValueError( ValueError: Trying to set a tensor of shape torch.Size([39321600, 1]) in "weight" (which has shape torch.Size([15360, 5120])), this look incorrect.

[LinXin04]

@baichuan-assistant