feat: tokenizers (#429)

* feat: tokenizers This reverts commit d5a4db8321. * fix(inputs): #422 Signed-off-by: Aaron Pham <contact@aarnphm.xyz> --------- Signed-off-by: Aaron Pham <contact@aarnphm.xyz>
2024-08-31 13:39:50 -04:00
parent 534b1e6bec
commit 0557deeab7
28 changed files with 3553 additions and 27 deletions
--- a/lua/avante/providers/openai.lua
+++ b/lua/avante/providers/openai.lua
@@ -26,6 +26,7 @@ local P = require("avante.providers")
 local M = {}

 M.api_key_name = "OPENAI_API_KEY"
+M.tokenizer_id = "gpt-4o"

 ---@param opts AvantePromptOptions
 M.get_user_message = function(opts)