fix: fetch llm tool no longer depends on system libssl.so (#1404)

.github/workflows/rust.yaml

@@ -49,7 +49,7 @@ jobs:
       - uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # ratchet:Swatinem/rust-cache@v2
       - uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a # ratchet:dtolnay/rust-toolchain@master
         with:
-          toolchain: "1.80.1"
+          toolchain: "1.85.0"
           components: clippy, rustfmt
       - name: Run rustfmt
         run: make rustlint

Cargo.lock

@@ -93,7 +93,7 @@ name = "avante-tokenizers"
 version = "0.1.0"
 dependencies = [
  "dirs",
- "hf-hub",
+ "hf-hub 0.4.1",
  "mlua",
  "regex",
  "tiktoken-rs",
@@ -494,6 +494,21 @@ dependencies = [
  "new_debug_unreachable",
 ]
 
+[[package]]
+name = "futures"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
 [[package]]
 name = "futures-channel"
 version = "0.3.31"
@@ -510,12 +525,34 @@ version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
 
+[[package]]
+name = "futures-executor"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
 [[package]]
 name = "futures-io"
 version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
 
+[[package]]
+name = "futures-macro"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "futures-sink"
 version = "0.3.31"
@@ -534,8 +571,10 @@ version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
 dependencies = [
+ "futures-channel",
  "futures-core",
  "futures-io",
+ "futures-macro",
  "futures-sink",
  "futures-task",
  "memchr",
@@ -607,10 +646,33 @@ dependencies = [
  "rand",
  "serde",
  "serde_json",
- "thiserror",
+ "thiserror 1.0.65",
  "ureq",
 ]
 
+[[package]]
+name = "hf-hub"
+version = "0.4.1"
+source = "git+https://github.com/yetone/hf-hub?branch=main#9db367044d11cac5ffe29d241b378e4578f01e72"
+dependencies = [
+ "dirs",
+ "futures",
+ "http",
+ "indicatif",
+ "libc",
+ "log",
+ "native-tls",
+ "num_cpus",
+ "rand",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.11",
+ "tokio",
+ "ureq",
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "htmd"
 version = "0.1.6"
@@ -849,7 +911,7 @@ dependencies = [
  "combine",
  "jni-sys",
  "log",
- "thiserror",
+ "thiserror 1.0.65",
  "walkdir",
 ]
 
@@ -1124,6 +1186,16 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "num_cpus"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
 [[package]]
 name = "number_prefix"
 version = "0.4.0"
@@ -1199,6 +1271,15 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
+[[package]]
+name = "openssl-src"
+version = "300.4.2+3.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "168ce4e058f975fe43e89d9ccf78ca668601887ae736090aacc23ae353c298e2"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "openssl-sys"
 version = "0.9.103"
@@ -1207,6 +1288,7 @@ checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6"
 dependencies = [
  "cc",
  "libc",
+ "openssl-src",
  "pkg-config",
  "vcpkg",
 ]
@@ -1340,9 +1422,9 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.86"
+version = "1.0.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
+checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99"
 dependencies = [
  "unicode-ident",
 ]
@@ -1434,7 +1516,7 @@ checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"
 dependencies = [
  "getrandom",
  "libredox",
- "thiserror",
+ "thiserror 1.0.65",
 ]
 
 [[package]]
@@ -1502,11 +1584,13 @@ dependencies = [
  "system-configuration",
  "tokio",
  "tokio-native-tls",
+ "tokio-util",
  "tower",
  "tower-service",
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
+ "wasm-streams",
  "web-sys",
  "windows-registry",
 ]
@@ -1816,9 +1900,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
 [[package]]
 name = "syn"
-version = "2.0.77"
+version = "2.0.98"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed"
+checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1885,7 +1969,16 @@ version = "1.0.65"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5"
 dependencies = [
- "thiserror-impl",
+ "thiserror-impl 1.0.65",
+]
+
+[[package]]
+name = "thiserror"
+version = "2.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc"
+dependencies = [
+ "thiserror-impl 2.0.11",
 ]
 
 [[package]]
@@ -1899,6 +1992,17 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "thiserror-impl"
+version = "2.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "tiktoken-rs"
 version = "0.6.0"
@@ -1941,7 +2045,7 @@ dependencies = [
  "esaxx-rs",
  "fancy-regex",
  "getrandom",
- "hf-hub",
+ "hf-hub 0.3.2",
  "itertools 0.12.1",
  "lazy_static",
  "log",
@@ -1957,7 +2061,7 @@ dependencies = [
  "serde",
  "serde_json",
  "spm_precompiled",
- "thiserror",
+ "thiserror 1.0.65",
  "unicode-normalization-alignments",
  "unicode-segmentation",
  "unicode_categories",
@@ -1975,9 +2079,21 @@ dependencies = [
  "mio",
  "pin-project-lite",
  "socket2",
+ "tokio-macros",
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "tokio-macros"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "tokio-native-tls"
 version = "0.3.1"
@@ -2427,6 +2543,19 @@ version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
 
+[[package]]
+name = "wasm-streams"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd"
+dependencies = [
+ "futures-util",
+ "js-sys",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
 [[package]]
 name = "web-sys"
 version = "0.3.70"

crates/avante-html2md/Cargo.toml

@@ -12,7 +12,7 @@ version.workspace = true
 htmd = "0.1.6"
 html2md = "0.2.15"
 mlua.workspace = true
-reqwest = { version = "0.12.12", features = ["blocking"] }
+reqwest = { version = "0.12.12", features = ["blocking", "native-tls-vendored"] }
 
 [lints]
 workspace = true

crates/avante-repo-map/src/lib.rs

@@ -1,3 +1,5 @@
+#![allow(clippy::unnecessary_map_or)]
+
 use mlua::prelude::*;
 use std::cell::RefCell;
 use std::collections::BTreeMap;

crates/avante-templates/src/lib.rs

@@ -7,7 +7,7 @@ struct State<'a> {
     environment: Mutex<Option<Environment<'a>>>,
 }
 
-impl<'a> State<'a> {
+impl State<'_> {
     fn new() -> Self {
         State {
             environment: Mutex::new(None),

crates/avante-tokenizers/Cargo.toml

@@ -14,7 +14,7 @@ workspace = true
 [dependencies]
 dirs = "5.0.1"
 regex = "1.11.1"
-hf-hub = { version = "0.3.2", features = ["default"] }
+hf-hub = { git = "https://github.com/yetone/hf-hub", branch='main', features = ["default", "ureq"] }
 ureq = { version = "2.10.1", features = ["json", "socks-proxy"] }
 mlua = { workspace = true }
 tiktoken-rs = { workspace = true }

crates/avante-tokenizers/src/lib.rs

@@ -81,7 +81,7 @@ impl HuggingFaceTokenizer {
 
 enum TokenizerType {
     Tiktoken(Tiktoken),
-    HuggingFace(HuggingFaceTokenizer),
+    HuggingFace(Box<HuggingFaceTokenizer>),
 }
 
 struct State {
@@ -111,7 +111,7 @@ fn from_pretrained(state: &State, model: &str) {
     let mut tokenizer_mutex = state.tokenizer.lock().unwrap();
     *tokenizer_mutex = Some(match model {
         "gpt-4o" => TokenizerType::Tiktoken(Tiktoken::new(model)),
-        _ => TokenizerType::HuggingFace(HuggingFaceTokenizer::new(model)),
+        _ => TokenizerType::HuggingFace(Box::new(HuggingFaceTokenizer::new(model))),
     });
 }
 

lua/avante/html2md.lua

@@ -19,7 +19,7 @@ function M.setup() vim.defer_fn(M._init_html2md_lib, 1000) end
 
 function M.fetch_md(url)
   local html2md_lib = M._init_html2md_lib()
-  if not html2md_lib then return "", "Failed to load avante_html2md" end
+  if not html2md_lib then return nil, "Failed to load avante_html2md" end
 
   return html2md_lib.fetch_md(url)
 end

lua/avante/init.lua

@@ -28,6 +28,7 @@ function H.load_path()
   local ok, LazyConfig = pcall(require, "lazy.core.config")
 
   if ok then
+    Utils.debug("LazyConfig loaded")
     local name = "avante.nvim"
     local function load_path() require("avante_lib").load() end
 
@@ -406,7 +407,6 @@ function M.setup(opts)
         local uri = "file://" .. Utils.get_project_root()
         if uri:sub(-1) ~= "/" then uri = uri .. "/" end
         RagService.add_resource(uri)
-        Utils.info("Added project root to Rag Service")
       end, 5000)
     end
     add_resource_with_delay = function()

lua/avante/llm_tools.lua

@@ -380,8 +380,8 @@ end
 function M.fetch(opts, on_log)
   if on_log then on_log("url: " .. opts.url) end
   local Html2Md = require("avante.html2md")
-  local res = Html2Md.fetch_md(opts.url)
+  local res, err = Html2Md.fetch_md(opts.url)
-  if res == nil then return nil, "Failed to fetch markdown" end
+  if err then return nil, err end
   return res, nil
 end
 

lua/avante/rag_service.lua

@@ -135,7 +135,6 @@ end
 ---@field message string
 
 ---@param uri string
----@return AvanteRagServiceAddResourceResponse | nil
 function M.add_resource(uri)
   uri = M.to_container_uri(uri)
   local resource_name = uri:match("([^/]+)/$")
@@ -171,20 +170,23 @@ function M.add_resource(uri)
       end
     end
   end
-  local resp = curl.post(M.get_rag_service_url() .. "/api/v1/add_resource", {
+  local cmd = {
-    headers = {
+    "curl",
-      ["Content-Type"] = "application/json",
+    "-X",
-    },
+    "POST",
-    body = vim.json.encode({
+    M.get_rag_service_url() .. "/api/v1/add_resource",
-      name = resource_name,
+    "-H",
-      uri = uri,
+    "Content-Type: application/json",
-    }),
+    "-d",
-  })
+    vim.json.encode({ name = resource_name, uri = uri }),
-  if resp.status ~= 200 then
+  }
-    Utils.error("failed to add resource: " .. resp.body)
+  vim.system(cmd, { text = true }, function(output)
-    return
+    if output.code == 0 then
+      Utils.debug(string.format("Added resource: %s", uri))
+    else
+      Utils.error(string.format("Failed to add resource: %s; output: %s", uri, output.stderr))
     end
-  return vim.json.decode(resp.body)
+  end)
 end
 
 function M.remove_resource(uri)