CarGDev/avante.nvim
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert tokenizers (#423)

Browse Source 
* Revert "fix: add missing "with" in README"

This reverts commit bf1e7f1f17.

* Revert "feat: tokenizers (#407)"

This reverts commit d2095ba267.
This commit is contained in:
yetone
2024-08-31 22:45:31 +08:00
committed by GitHub
parent bf1e7f1f17
commit d5a4db8321
27 changed files with 27 additions and 3554 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
lua/avante/tokenizers.lua
View File

@@ -1,66 +0,0 @@
local Utils = require("avante.utils")
---@class AvanteTokenizer
---@field from_pretrained fun(model: string): nil
---@field encode fun(string): integer[]
local tokenizers = nil
local M = {}
---@param model "gpt-4o" | string
M.setup = function(model)
local ok, core = pcall(require, "avante_tokenizers")
if not ok then
return
end
---@cast core AvanteTokenizer
if tokenizers == nil then
tokenizers = core
end
local HF_TOKEN = os.getenv("HF_TOKEN")
if HF_TOKEN == nil and model ~= "gpt-4o" then
Utils.warn(
"Please set HF_TOKEN environment variable to use HuggingFace tokenizer if " .. model .. " is gated",
{ once = true }
)
end
vim.env.HF_HUB_DISABLE_PROGRESS_BARS = 1
---@cast core AvanteTokenizer
core.from_pretrained(model)
end
M.available = function()
return tokenizers ~= nil
end
---@param prompt string
M.encode = function(prompt)
if not tokenizers then
return nil
end
if not prompt or prompt == "" then
return nil
end
if type(prompt) ~= "string" then
error("Prompt is not type string", 2)
end
return tokenizers.encode(prompt)
end
---@param prompt string
M.count = function(prompt)
if not tokenizers then
return math.ceil(#prompt * 0.5)
end
local tokens = M.encode(prompt)
if not tokens then
return 0
end
return #tokens
end
return M