feat(tokenizers): support parsing from public URL (#765)

2024-10-27 02:17:35 -04:00
parent a8e2b9a00c
commit bdbbdec88c
7 changed files with 236 additions and 89 deletions
--- a/crates/avante-tokenizers/Cargo.toml
+++ b/crates/avante-tokenizers/Cargo.toml
@@ -12,6 +12,10 @@ license = { workspace = true }
 workspace = true

 [dependencies]
+dirs = "5.0.1"
+regex = "1.11.1"
+hf-hub = { version = "0.3.2", features = ["default"] }
+ureq = { version = "2.10.1", features = ["json", "socks-proxy"] }
 mlua = { workspace = true }
 tiktoken-rs = { workspace = true }
 tokenizers = { workspace = true }