feat: tokenizers (#429)

* feat: tokenizers This reverts commit d5a4db8321. * fix(inputs): #422 Signed-off-by: Aaron Pham <contact@aarnphm.xyz> --------- Signed-off-by: Aaron Pham <contact@aarnphm.xyz>
2024-08-31 13:39:50 -04:00
parent 534b1e6bec
commit 0557deeab7
28 changed files with 3553 additions and 27 deletions
--- a/lua/avante/config.lua
+++ b/lua/avante/config.lua
@@ -285,6 +285,7 @@ M.BASE_PROVIDER_KEYS = {
  -- internal
  "local",
  "_shellenv",
+  "tokenizer_id",
 }

 ---@return {width: integer, height: integer}