feat: tokenizers (#429)

* feat: tokenizers This reverts commit d5a4db8321. * fix(inputs): #422 Signed-off-by: Aaron Pham <contact@aarnphm.xyz> --------- Signed-off-by: Aaron Pham <contact@aarnphm.xyz>
2024-08-31 13:39:50 -04:00
parent 534b1e6bec
commit 0557deeab7
28 changed files with 3553 additions and 27 deletions
--- a/lua/avante/providers/cohere.lua
+++ b/lua/avante/providers/cohere.lua
@@ -29,6 +29,7 @@ local P = require("avante.providers")
 local M = {}

 M.api_key_name = "CO_API_KEY"
+M.tokenizer_id = "CohereForAI/c4ai-command-r-plus-08-2024"

 M.parse_message = function(opts)
  local user_prompt = table.concat(opts.user_prompts, "\n\n")