Revert tokenizers (#423)

* Revert "fix: add missing "with" in README"

This reverts commit bf1e7f1f17.

* Revert "feat: tokenizers (#407)"

This reverts commit d2095ba267.
This commit is contained in:
yetone
2024-08-31 22:45:31 +08:00
committed by GitHub
parent bf1e7f1f17
commit d5a4db8321
27 changed files with 27 additions and 3554 deletions

View File

@@ -1,6 +1,4 @@
--Taken from https://github.com/jackMort/ChatGPT.nvim/blob/main/lua/chatgpt/flows/chat/tokens.lua
local Tokenizer = require("avante.tokenizers")
---@class avante.utils.tokens
local Tokens = {}
@@ -13,10 +11,6 @@ local cost_per_token = {
---@param text string The text to calculate the number of tokens for.
---@return integer The number of tokens in the given text.
function Tokens.calculate_tokens(text)
if Tokenizer.available() then
return Tokenizer.count(text)
end
local tokens = 0
local current_token = ""
for char in text:gmatch(".") do