Revert tokenizers (#423)

* Revert "fix: add missing "with" in README"

This reverts commit bf1e7f1f17.

* Revert "feat: tokenizers (#407)"

This reverts commit d2095ba267.

lua/avante/providers/azure.lua

@@ -12,7 +12,6 @@ local O = require("avante.providers").openai
 local M = {}
 
 M.api_key_name = "AZURE_OPENAI_API_KEY"
-M.tokenizer_id = "gpt-4o"
 
 M.parse_message = O.parse_message
 M.parse_response = O.parse_response

lua/avante/providers/claude.lua

@@ -6,7 +6,6 @@ local P = require("avante.providers")
 local M = {}
 
 M.api_key_name = "ANTHROPIC_API_KEY"
-M.tokenizer_id = "gpt-4o"
 
 ---@param prompt_opts AvantePromptOptions
 M.parse_message = function(prompt_opts)
@@ -29,10 +28,8 @@ M.parse_message = function(prompt_opts)
     local user_prompt_obj = {
       type = "text",
       text = user_prompt,
+      cache_control = { type = "ephemeral" },
     }
-    if Utils.tokens.calculate_tokens(user_prompt_obj.text) > 1024 then
-      user_prompt_obj.cache_control = { type = "ephemeral" }
-    end
 
     table.insert(message_content, user_prompt_obj)
   end

lua/avante/providers/cohere.lua

@@ -29,7 +29,6 @@ local P = require("avante.providers")
 local M = {}
 
 M.api_key_name = "CO_API_KEY"
-M.tokenizer_id = "CohereForAI/c4ai-command-r-plus-08-2024"
 
 M.parse_message = function(opts)
   local user_prompt = table.concat(opts.user_prompts, "\n\n")

lua/avante/providers/copilot.lua

@@ -127,7 +127,6 @@ end
 M.state = nil
 
 M.api_key_name = P.AVANTE_INTERNAL_KEY
-M.tokenizer_id = "gpt-4o"
 
 M.parse_message = function(opts)
   return {
@@ -167,7 +166,6 @@ M.setup = function()
     M.state = { github_token = nil, oauth_token = H.get_oauth_token() }
     H.refresh_token()
   end
-  require("avante.tokenizers").setup(M.tokenizer_id)
   vim.g.avante_login = true
 end
 

lua/avante/providers/gemini.lua

@@ -6,7 +6,6 @@ local Clipboard = require("avante.clipboard")
 local M = {}
 
 M.api_key_name = "GEMINI_API_KEY"
-M.tokenizer_id = "google/gemma-2b"
 
 M.parse_message = function(opts)
   local message_content = {}

lua/avante/providers/init.lua

@@ -69,7 +69,6 @@ local Dressing = require("avante.ui.dressing")
 ---@field setup fun(): nil
 ---@field has fun(): boolean
 ---@field api_key_name string
----@field tokenizer_id string | "gpt-4o"
 ---@field model? string
 ---@field parse_api_key fun(): string | nil
 ---@field parse_stream_data? AvanteStreamParser
@@ -270,11 +269,6 @@ M = setmetatable(M, {
       return E.parse_envvar(t[k])
     end
 
-    -- default to gpt-4o as tokenizer
-    if t[k].tokenizer_id == nil then
-      t[k].tokenizer_id = "gpt-4o"
-    end
-
     if t[k].has == nil then
       t[k].has = function()
         return E.parse_envvar(t[k]) ~= nil
@@ -286,7 +280,6 @@ M = setmetatable(M, {
         if not E.is_local(k) then
           t[k].parse_api_key()
         end
-        require("avante.tokenizers").setup(t[k].tokenizer_id)
       end
     end
 

lua/avante/providers/openai.lua

@@ -26,7 +26,6 @@ local P = require("avante.providers")
 local M = {}
 
 M.api_key_name = "OPENAI_API_KEY"
-M.tokenizer_id = "gpt-4o"
 
 ---@param opts AvantePromptOptions
 M.get_user_message = function(opts)