fix: fetch llm tool no longer depends on system libssl.so (#1404)

This commit is contained in:
yetone
2025-02-26 17:36:01 +08:00
committed by GitHub
parent ef78c36e98
commit 6f98cb0ed6
11 changed files with 169 additions and 36 deletions

View File

@@ -12,7 +12,7 @@ version.workspace = true
htmd = "0.1.6"
html2md = "0.2.15"
mlua.workspace = true
reqwest = { version = "0.12.12", features = ["blocking"] }
reqwest = { version = "0.12.12", features = ["blocking", "native-tls-vendored"] }
[lints]
workspace = true

View File

@@ -1,3 +1,5 @@
#![allow(clippy::unnecessary_map_or)]
use mlua::prelude::*;
use std::cell::RefCell;
use std::collections::BTreeMap;

View File

@@ -7,7 +7,7 @@ struct State<'a> {
environment: Mutex<Option<Environment<'a>>>,
}
impl<'a> State<'a> {
impl State<'_> {
fn new() -> Self {
State {
environment: Mutex::new(None),

View File

@@ -14,7 +14,7 @@ workspace = true
[dependencies]
dirs = "5.0.1"
regex = "1.11.1"
hf-hub = { version = "0.3.2", features = ["default"] }
hf-hub = { git = "https://github.com/yetone/hf-hub", branch='main', features = ["default", "ureq"] }
ureq = { version = "2.10.1", features = ["json", "socks-proxy"] }
mlua = { workspace = true }
tiktoken-rs = { workspace = true }

View File

@@ -81,7 +81,7 @@ impl HuggingFaceTokenizer {
enum TokenizerType {
Tiktoken(Tiktoken),
HuggingFace(HuggingFaceTokenizer),
HuggingFace(Box<HuggingFaceTokenizer>),
}
struct State {
@@ -111,7 +111,7 @@ fn from_pretrained(state: &State, model: &str) {
let mut tokenizer_mutex = state.tokenizer.lock().unwrap();
*tokenizer_mutex = Some(match model {
"gpt-4o" => TokenizerType::Tiktoken(Tiktoken::new(model)),
_ => TokenizerType::HuggingFace(HuggingFaceTokenizer::new(model)),
_ => TokenizerType::HuggingFace(Box::new(HuggingFaceTokenizer::new(model))),
});
}