fix: fetch llm tool no longer depends on system libssl.so (#1404)

crates/avante-tokenizers/Cargo.toml

@@ -14,7 +14,7 @@ workspace = true
 [dependencies]
 dirs = "5.0.1"
 regex = "1.11.1"
-hf-hub = { version = "0.3.2", features = ["default"] }
+hf-hub = { git = "https://github.com/yetone/hf-hub", branch='main', features = ["default", "ureq"] }
 ureq = { version = "2.10.1", features = ["json", "socks-proxy"] }
 mlua = { workspace = true }
 tiktoken-rs = { workspace = true }

crates/avante-tokenizers/src/lib.rs

@@ -81,7 +81,7 @@ impl HuggingFaceTokenizer {
 
 enum TokenizerType {
     Tiktoken(Tiktoken),
-    HuggingFace(HuggingFaceTokenizer),
+    HuggingFace(Box<HuggingFaceTokenizer>),
 }
 
 struct State {
@@ -111,7 +111,7 @@ fn from_pretrained(state: &State, model: &str) {
     let mut tokenizer_mutex = state.tokenizer.lock().unwrap();
     *tokenizer_mutex = Some(match model {
         "gpt-4o" => TokenizerType::Tiktoken(Tiktoken::new(model)),
-        _ => TokenizerType::HuggingFace(HuggingFaceTokenizer::new(model)),
+        _ => TokenizerType::HuggingFace(Box::new(HuggingFaceTokenizer::new(model))),
     });
 }