Revert tokenizers (#423)
* Revert "fix: add missing "with" in README" This reverts commitbf1e7f1f17. * Revert "feat: tokenizers (#407)" This reverts commitd2095ba267.
This commit is contained in:
@@ -1,6 +1,4 @@
|
||||
--Taken from https://github.com/jackMort/ChatGPT.nvim/blob/main/lua/chatgpt/flows/chat/tokens.lua
|
||||
local Tokenizer = require("avante.tokenizers")
|
||||
|
||||
---@class avante.utils.tokens
|
||||
local Tokens = {}
|
||||
|
||||
@@ -13,10 +11,6 @@ local cost_per_token = {
|
||||
---@param text string The text to calculate the number of tokens for.
|
||||
---@return integer The number of tokens in the given text.
|
||||
function Tokens.calculate_tokens(text)
|
||||
if Tokenizer.available() then
|
||||
return Tokenizer.count(text)
|
||||
end
|
||||
|
||||
local tokens = 0
|
||||
local current_token = ""
|
||||
for char in text:gmatch(".") do
|
||||
|
||||
Reference in New Issue
Block a user