refactor & fix: improve libraries initialization (#921)

* refactor(libs): extract libraries initialization

Extract initialization logic into separate functions
for better error handling and reusability.

* fix(libs): improve core libraries init

This change helps prevent runtime errors from uninitialized libraries.

lua/avante/tokenizers.lua

@@ -5,20 +5,38 @@ local Utils = require("avante.utils")
 ---@field encode fun(string): integer[]
 local tokenizers = nil
 
+---@type "gpt-4o" | string
+local current_model = "gpt-4o"
+
 local M = {}
 
+---@param model "gpt-4o" | string
+---@return AvanteTokenizer|nil
+M._init_tokenizers_lib = function(model)
+  if tokenizers ~= nil then
+    return tokenizers
+  end
+
+  local ok, core = pcall(require, "avante_tokenizers")
+  if not ok then
+    return nil
+  end
+
+  ---@cast core AvanteTokenizer
+  tokenizers = core
+
+  core.from_pretrained(model)
+
+  return tokenizers
+end
+
 ---@param model "gpt-4o" | string
 ---@param warning? boolean
 M.setup = function(model, warning)
+  current_model = model
   warning = warning or true
   vim.defer_fn(function()
-    local ok, core = pcall(require, "avante_tokenizers")
-    if not ok then return end
-
-    ---@cast core AvanteTokenizer
-    if tokenizers == nil then tokenizers = core end
-
-    core.from_pretrained(model)
+    M._init_tokenizers_lib(model)
   end, 1000)
 
   if warning then
@@ -32,11 +50,11 @@ M.setup = function(model, warning)
   end
 end
 
-M.available = function() return tokenizers ~= nil end
+M.available = function() return M._init_tokenizers_lib(current_model) ~= nil end
 
 ---@param prompt string
 M.encode = function(prompt)
-  if not tokenizers then return nil end
+  if not M.available() then return nil end
   if not prompt or prompt == "" then return nil end
   if type(prompt) ~= "string" then error("Prompt is not type string", 2) end
 
@@ -45,7 +63,7 @@ end
 
 ---@param prompt string
 M.count = function(prompt)
-  if not tokenizers then return math.ceil(#prompt * 0.5) end
+  if not M.available() then return math.ceil(#prompt * 0.5) end
 
   local tokens = M.encode(prompt)
   if not tokens then return 0 end