fix: fetch llm tool no longer depends on system libssl.so (#1404)

This commit is contained in:
yetone
2025-02-26 17:36:01 +08:00
committed by GitHub
parent ef78c36e98
commit 6f98cb0ed6
11 changed files with 169 additions and 36 deletions

View File

@@ -81,7 +81,7 @@ impl HuggingFaceTokenizer {
enum TokenizerType {
Tiktoken(Tiktoken),
HuggingFace(HuggingFaceTokenizer),
HuggingFace(Box<HuggingFaceTokenizer>),
}
struct State {
@@ -111,7 +111,7 @@ fn from_pretrained(state: &State, model: &str) {
let mut tokenizer_mutex = state.tokenizer.lock().unwrap();
*tokenizer_mutex = Some(match model {
"gpt-4o" => TokenizerType::Tiktoken(Tiktoken::new(model)),
_ => TokenizerType::HuggingFace(HuggingFaceTokenizer::new(model)),
_ => TokenizerType::HuggingFace(Box::new(HuggingFaceTokenizer::new(model))),
});
}