feat: tokenizers (#429)

* feat: tokenizers

This reverts commit d5a4db8321.

* fix(inputs): #422

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

---------

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

lua/avante/providers/azure.lua

@@ -12,6 +12,7 @@ local O = require("avante.providers").openai
 local M = {}
 
 M.api_key_name = "AZURE_OPENAI_API_KEY"
+M.tokenizer_id = "gpt-4o"
 
 M.parse_message = O.parse_message
 M.parse_response = O.parse_response

lua/avante/providers/claude.lua

@@ -6,6 +6,7 @@ local P = require("avante.providers")
 local M = {}
 
 M.api_key_name = "ANTHROPIC_API_KEY"
+M.tokenizer_id = "gpt-4o"
 
 ---@param prompt_opts AvantePromptOptions
 M.parse_message = function(prompt_opts)
@@ -26,7 +27,7 @@ M.parse_message = function(prompt_opts)
 
   local user_prompts_with_length = {}
   for idx, user_prompt in ipairs(prompt_opts.user_prompts) do
-    table.insert(user_prompts_with_length, { idx = idx, length = #user_prompt })
+    table.insert(user_prompts_with_length, { idx = idx, length = Utils.tokens.calculate_tokens(user_prompt) })
   end
 
   table.sort(user_prompts_with_length, function(a, b)

lua/avante/providers/cohere.lua

@@ -29,6 +29,7 @@ local P = require("avante.providers")
 local M = {}
 
 M.api_key_name = "CO_API_KEY"
+M.tokenizer_id = "CohereForAI/c4ai-command-r-plus-08-2024"
 
 M.parse_message = function(opts)
   local user_prompt = table.concat(opts.user_prompts, "\n\n")

lua/avante/providers/copilot.lua

@@ -127,6 +127,7 @@ end
 M.state = nil
 
 M.api_key_name = P.AVANTE_INTERNAL_KEY
+M.tokenizer_id = "gpt-4o"
 
 M.parse_message = function(opts)
   return {
@@ -166,6 +167,7 @@ M.setup = function()
     M.state = { github_token = nil, oauth_token = H.get_oauth_token() }
     H.refresh_token()
   end
+  require("avante.tokenizers").setup(M.tokenizer_id)
   vim.g.avante_login = true
 end
 

lua/avante/providers/gemini.lua

@@ -6,6 +6,7 @@ local Clipboard = require("avante.clipboard")
 local M = {}
 
 M.api_key_name = "GEMINI_API_KEY"
+M.tokenizer_id = "google/gemma-2b"
 
 M.parse_message = function(opts)
   local message_content = {}

lua/avante/providers/init.lua

@@ -69,6 +69,7 @@ local Dressing = require("avante.ui.dressing")
 ---@field setup fun(): nil
 ---@field has fun(): boolean
 ---@field api_key_name string
+---@field tokenizer_id string | "gpt-4o"
 ---@field model? string
 ---@field parse_api_key fun(): string | nil
 ---@field parse_stream_data? AvanteStreamParser
@@ -269,6 +270,11 @@ M = setmetatable(M, {
       return E.parse_envvar(t[k])
     end
 
+    -- default to gpt-4o as tokenizer
+    if t[k].tokenizer_id == nil then
+      t[k].tokenizer_id = "gpt-4o"
+    end
+
     if t[k].has == nil then
       t[k].has = function()
         return E.parse_envvar(t[k]) ~= nil
@@ -280,6 +286,7 @@ M = setmetatable(M, {
         if not E.is_local(k) then
           t[k].parse_api_key()
         end
+        require("avante.tokenizers").setup(t[k].tokenizer_id)
       end
     end
 

lua/avante/providers/openai.lua

@@ -26,6 +26,7 @@ local P = require("avante.providers")
 local M = {}
 
 M.api_key_name = "OPENAI_API_KEY"
+M.tokenizer_id = "gpt-4o"
 
 ---@param opts AvantePromptOptions
 M.get_user_message = function(opts)