rust-bert
rust-bert copied to clipboard
Error: Tokenizer error: expected value at line 1 column 1
let model_resource = RemoteResource::from_pretrained(("distiluse-base-multilingual-cased","https://huggingface.co/sentence-transformers/distiluse-base-multilingual-cased/blob/main/rust_model.ot")); let config_resource = RemoteResource::from_pretrained(("distiluse-base-multilingual-cased","https://huggingface.co/sentence-transformers/distiluse-base-multilingual-cased/blob/main/config.json")); let vocab_resource = RemoteResource::from_pretrained(("distiluse-base-multilingual-cased","https://huggingface.co/sentence-transformers/distiluse-base-multilingual-cased/blob/main/vocab.txt")); let config = ZeroShotClassificationConfig::new(ModelType::DistilBert, ModelResource::Torch(Box::new(model_resource)), config_resource, vocab_resource, None, true, None, None ); let tokenizer_file_path = PathBuf::from("/root/rust-bert/model/tokenizer.json"); let special_token_map_path = PathBuf::from("/root/rust-bert/model/special_token_map.json"); let tokenizer = HFTokenizer::from_file(tokenizer_file_path, special_token_map_path)?; let sequence_classification_model = ZeroShotClassificationModel::new_with_tokenizer(config,TokenizerOption::HFTokenizer(tokenizer))?;
cargo run --package rust-bert --example zero_shot_classification --features="hf-tokenizers" Error: Tokenizer error: expected value at line 1 column 1