Revert tokenizers (#423)

* Revert "fix: add missing "with" in README"

This reverts commit bf1e7f1f17.

* Revert "feat: tokenizers (#407)"

This reverts commit d2095ba267.
This commit is contained in:
yetone
2024-08-31 22:45:31 +08:00
committed by GitHub
parent bf1e7f1f17
commit d5a4db8321
27 changed files with 27 additions and 3554 deletions

View File

@@ -6,7 +6,6 @@ local P = require("avante.providers")
local M = {}
M.api_key_name = "ANTHROPIC_API_KEY"
M.tokenizer_id = "gpt-4o"
---@param prompt_opts AvantePromptOptions
M.parse_message = function(prompt_opts)
@@ -29,10 +28,8 @@ M.parse_message = function(prompt_opts)
local user_prompt_obj = {
type = "text",
text = user_prompt,
cache_control = { type = "ephemeral" },
}
if Utils.tokens.calculate_tokens(user_prompt_obj.text) > 1024 then
user_prompt_obj.cache_control = { type = "ephemeral" }
end
table.insert(message_content, user_prompt_obj)
end