diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index 4ae1f4e..36406b1 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -49,7 +49,7 @@ jobs: - uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # ratchet:Swatinem/rust-cache@v2 - uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a # ratchet:dtolnay/rust-toolchain@master with: - toolchain: "1.80.1" + toolchain: "1.85.0" components: clippy, rustfmt - name: Run rustfmt run: make rustlint diff --git a/Cargo.lock b/Cargo.lock index d36afed..6ee1bb7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -93,7 +93,7 @@ name = "avante-tokenizers" version = "0.1.0" dependencies = [ "dirs", - "hf-hub", + "hf-hub 0.4.1", "mlua", "regex", "tiktoken-rs", @@ -494,6 +494,21 @@ dependencies = [ "new_debug_unreachable", ] +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.31" @@ -510,12 +525,34 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.31" @@ -534,8 +571,10 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -607,10 +646,33 @@ dependencies = [ "rand", "serde", "serde_json", - "thiserror", + "thiserror 1.0.65", "ureq", ] +[[package]] +name = "hf-hub" +version = "0.4.1" +source = "git+https://github.com/yetone/hf-hub?branch=main#9db367044d11cac5ffe29d241b378e4578f01e72" +dependencies = [ + "dirs", + "futures", + "http", + "indicatif", + "libc", + "log", + "native-tls", + "num_cpus", + "rand", + "reqwest", + "serde", + "serde_json", + "thiserror 2.0.11", + "tokio", + "ureq", + "windows-sys 0.59.0", +] + [[package]] name = "htmd" version = "0.1.6" @@ -849,7 +911,7 @@ dependencies = [ "combine", "jni-sys", "log", - "thiserror", + "thiserror 1.0.65", "walkdir", ] @@ -1124,6 +1186,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "number_prefix" version = "0.4.0" @@ -1199,6 +1271,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-src" +version = "300.4.2+3.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168ce4e058f975fe43e89d9ccf78ca668601887ae736090aacc23ae353c298e2" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.103" @@ -1207,6 +1288,7 @@ checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" dependencies = [ "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] @@ -1340,9 +1422,9 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] @@ -1434,7 +1516,7 @@ checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom", "libredox", - "thiserror", + "thiserror 1.0.65", ] [[package]] @@ -1502,11 +1584,13 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", + "tokio-util", "tower", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "windows-registry", ] @@ -1816,9 +1900,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.77" +version = "2.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" dependencies = [ "proc-macro2", "quote", @@ -1885,7 +1969,16 @@ version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.65", +] + +[[package]] +name = "thiserror" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +dependencies = [ + "thiserror-impl 2.0.11", ] [[package]] @@ -1899,6 +1992,17 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror-impl" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tiktoken-rs" version = "0.6.0" @@ -1941,7 +2045,7 @@ dependencies = [ "esaxx-rs", "fancy-regex", "getrandom", - "hf-hub", + "hf-hub 0.3.2", "itertools 0.12.1", "lazy_static", "log", @@ -1957,7 +2061,7 @@ dependencies = [ "serde", "serde_json", "spm_precompiled", - "thiserror", + "thiserror 1.0.65", "unicode-normalization-alignments", "unicode-segmentation", "unicode_categories", @@ -1975,9 +2079,21 @@ dependencies = [ "mio", "pin-project-lite", "socket2", + "tokio-macros", "windows-sys 0.52.0", ] +[[package]] +name = "tokio-macros" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio-native-tls" version = "0.3.1" @@ -2427,6 +2543,19 @@ version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +[[package]] +name = "wasm-streams" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.70" diff --git a/crates/avante-html2md/Cargo.toml b/crates/avante-html2md/Cargo.toml index 4ae1119..f710544 100644 --- a/crates/avante-html2md/Cargo.toml +++ b/crates/avante-html2md/Cargo.toml @@ -12,7 +12,7 @@ version.workspace = true htmd = "0.1.6" html2md = "0.2.15" mlua.workspace = true -reqwest = { version = "0.12.12", features = ["blocking"] } +reqwest = { version = "0.12.12", features = ["blocking", "native-tls-vendored"] } [lints] workspace = true diff --git a/crates/avante-repo-map/src/lib.rs b/crates/avante-repo-map/src/lib.rs index cef8e2d..42e4ee4 100644 --- a/crates/avante-repo-map/src/lib.rs +++ b/crates/avante-repo-map/src/lib.rs @@ -1,3 +1,5 @@ +#![allow(clippy::unnecessary_map_or)] + use mlua::prelude::*; use std::cell::RefCell; use std::collections::BTreeMap; diff --git a/crates/avante-templates/src/lib.rs b/crates/avante-templates/src/lib.rs index 8a4d23f..b4d275b 100644 --- a/crates/avante-templates/src/lib.rs +++ b/crates/avante-templates/src/lib.rs @@ -7,7 +7,7 @@ struct State<'a> { environment: Mutex>>, } -impl<'a> State<'a> { +impl State<'_> { fn new() -> Self { State { environment: Mutex::new(None), diff --git a/crates/avante-tokenizers/Cargo.toml b/crates/avante-tokenizers/Cargo.toml index fcecb63..dd072f3 100644 --- a/crates/avante-tokenizers/Cargo.toml +++ b/crates/avante-tokenizers/Cargo.toml @@ -14,7 +14,7 @@ workspace = true [dependencies] dirs = "5.0.1" regex = "1.11.1" -hf-hub = { version = "0.3.2", features = ["default"] } +hf-hub = { git = "https://github.com/yetone/hf-hub", branch='main', features = ["default", "ureq"] } ureq = { version = "2.10.1", features = ["json", "socks-proxy"] } mlua = { workspace = true } tiktoken-rs = { workspace = true } diff --git a/crates/avante-tokenizers/src/lib.rs b/crates/avante-tokenizers/src/lib.rs index d89cf1c..2a91004 100644 --- a/crates/avante-tokenizers/src/lib.rs +++ b/crates/avante-tokenizers/src/lib.rs @@ -81,7 +81,7 @@ impl HuggingFaceTokenizer { enum TokenizerType { Tiktoken(Tiktoken), - HuggingFace(HuggingFaceTokenizer), + HuggingFace(Box), } struct State { @@ -111,7 +111,7 @@ fn from_pretrained(state: &State, model: &str) { let mut tokenizer_mutex = state.tokenizer.lock().unwrap(); *tokenizer_mutex = Some(match model { "gpt-4o" => TokenizerType::Tiktoken(Tiktoken::new(model)), - _ => TokenizerType::HuggingFace(HuggingFaceTokenizer::new(model)), + _ => TokenizerType::HuggingFace(Box::new(HuggingFaceTokenizer::new(model))), }); } diff --git a/lua/avante/html2md.lua b/lua/avante/html2md.lua index 58d99b1..adccb59 100644 --- a/lua/avante/html2md.lua +++ b/lua/avante/html2md.lua @@ -19,7 +19,7 @@ function M.setup() vim.defer_fn(M._init_html2md_lib, 1000) end function M.fetch_md(url) local html2md_lib = M._init_html2md_lib() - if not html2md_lib then return "", "Failed to load avante_html2md" end + if not html2md_lib then return nil, "Failed to load avante_html2md" end return html2md_lib.fetch_md(url) end diff --git a/lua/avante/init.lua b/lua/avante/init.lua index eb1b669..e807362 100644 --- a/lua/avante/init.lua +++ b/lua/avante/init.lua @@ -28,6 +28,7 @@ function H.load_path() local ok, LazyConfig = pcall(require, "lazy.core.config") if ok then + Utils.debug("LazyConfig loaded") local name = "avante.nvim" local function load_path() require("avante_lib").load() end @@ -406,7 +407,6 @@ function M.setup(opts) local uri = "file://" .. Utils.get_project_root() if uri:sub(-1) ~= "/" then uri = uri .. "/" end RagService.add_resource(uri) - Utils.info("Added project root to Rag Service") end, 5000) end add_resource_with_delay = function() diff --git a/lua/avante/llm_tools.lua b/lua/avante/llm_tools.lua index e994403..a88e1ec 100644 --- a/lua/avante/llm_tools.lua +++ b/lua/avante/llm_tools.lua @@ -380,8 +380,8 @@ end function M.fetch(opts, on_log) if on_log then on_log("url: " .. opts.url) end local Html2Md = require("avante.html2md") - local res = Html2Md.fetch_md(opts.url) - if res == nil then return nil, "Failed to fetch markdown" end + local res, err = Html2Md.fetch_md(opts.url) + if err then return nil, err end return res, nil end diff --git a/lua/avante/rag_service.lua b/lua/avante/rag_service.lua index a97324a..4205258 100644 --- a/lua/avante/rag_service.lua +++ b/lua/avante/rag_service.lua @@ -135,7 +135,6 @@ end ---@field message string ---@param uri string ----@return AvanteRagServiceAddResourceResponse | nil function M.add_resource(uri) uri = M.to_container_uri(uri) local resource_name = uri:match("([^/]+)/$") @@ -171,20 +170,23 @@ function M.add_resource(uri) end end end - local resp = curl.post(M.get_rag_service_url() .. "/api/v1/add_resource", { - headers = { - ["Content-Type"] = "application/json", - }, - body = vim.json.encode({ - name = resource_name, - uri = uri, - }), - }) - if resp.status ~= 200 then - Utils.error("failed to add resource: " .. resp.body) - return - end - return vim.json.decode(resp.body) + local cmd = { + "curl", + "-X", + "POST", + M.get_rag_service_url() .. "/api/v1/add_resource", + "-H", + "Content-Type: application/json", + "-d", + vim.json.encode({ name = resource_name, uri = uri }), + } + vim.system(cmd, { text = true }, function(output) + if output.code == 0 then + Utils.debug(string.format("Added resource: %s", uri)) + else + Utils.error(string.format("Failed to add resource: %s; output: %s", uri, output.stderr)) + end + end) end function M.remove_resource(uri)