rust-bert icon indicating copy to clipboard operation
rust-bert copied to clipboard

Error: Tokenizer error: expected value at line 1 column 1

Open lijingrs opened this issue 1 year ago • 0 comments

let model_resource = RemoteResource::from_pretrained(("distiluse-base-multilingual-cased","https://huggingface.co/sentence-transformers/distiluse-base-multilingual-cased/blob/main/rust_model.ot")); let config_resource = RemoteResource::from_pretrained(("distiluse-base-multilingual-cased","https://huggingface.co/sentence-transformers/distiluse-base-multilingual-cased/blob/main/config.json")); let vocab_resource = RemoteResource::from_pretrained(("distiluse-base-multilingual-cased","https://huggingface.co/sentence-transformers/distiluse-base-multilingual-cased/blob/main/vocab.txt")); let config = ZeroShotClassificationConfig::new(ModelType::DistilBert, ModelResource::Torch(Box::new(model_resource)), config_resource, vocab_resource, None, true, None, None ); let tokenizer_file_path = PathBuf::from("/root/rust-bert/model/tokenizer.json"); let special_token_map_path = PathBuf::from("/root/rust-bert/model/special_token_map.json"); let tokenizer = HFTokenizer::from_file(tokenizer_file_path, special_token_map_path)?; let sequence_classification_model = ZeroShotClassificationModel::new_with_tokenizer(config,TokenizerOption::HFTokenizer(tokenizer))?;

cargo run --package rust-bert --example zero_shot_classification --features="hf-tokenizers" Error: Tokenizer error: expected value at line 1 column 1

lijingrs avatar Aug 08 '24 01:08 lijingrs