CarGDev/avante.nvim
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: tokenizers (#429)

Browse Source 
* feat: tokenizers

This reverts commit d5a4db8321.

* fix(inputs): #422

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

---------

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>
This commit is contained in:
Aaron Pham
2024-08-31 13:39:50 -04:00
committed by GitHub
parent 534b1e6bec
commit 0557deeab7
28 changed files with 3553 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
lua/avante/providers/claude.lua
View File

@@ -6,6 +6,7 @@ local P = require("avante.providers")
local M = {}
M.api_key_name = "ANTHROPIC_API_KEY"
M.tokenizer_id = "gpt-4o"
---@param prompt_opts AvantePromptOptions
M.parse_message = function(prompt_opts)
@@ -26,7 +27,7 @@ M.parse_message = function(prompt_opts)
local user_prompts_with_length = {}
for idx, user_prompt in ipairs(prompt_opts.user_prompts) do
table.insert(user_prompts_with_length, { idx = idx, length = #user_prompt })
table.insert(user_prompts_with_length, { idx = idx, length = Utils.tokens.calculate_tokens(user_prompt) })
end
table.sort(user_prompts_with_length, function(a, b)