R5byte

If you are using Quantized model then you can load varbuilder like: let vb =candle_transformers::quantized_var_builder::VarBuilder::from_gguf(filename, &device)?; let model = QMistral::new(&config, vb)?; And for safetensors: let vb = unsafe { VarBuilder::from_mmaped_safetensors(&filenames,...