feat: tokenizers (#407)

* feat: autobuild tiktoken lib and schenanigans

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* chore: revert readme changes

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* fix(build): windows

Signed-off-by: Hanchin Hsieh <me@yuchanns.xyz>

* chore(plugin): early load commands and base setup

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* fix(build): make sync

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* feat: rust go vroom vroom

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* feat: scuffed afaf implementation binding go brrrr

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* chore: remove dups

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* fix(tokens): calculate whether we should do prompt_caching (fixes #416)

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* chore: ignore lockfiles

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* Update README.md

* Update crates/avante-tokenizers/README.md

* chore: remove unused

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* chore: remove auto build

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

---------

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>
Signed-off-by: Hanchin Hsieh <me@yuchanns.xyz>
Co-authored-by: yuchanns <me@yuchanns.xyz>

plugin/avante.lua

@@ -0,0 +1,35 @@
+--- NOTE: We will override vim.paste if img-clip.nvim is available to work with avante.nvim internal logic paste
+
+require("avante").setup()
+
+local Clipboard = require("avante.clipboard")
+local Config = require("avante.config")
+
+if Config.support_paste_image(true) then
+  vim.paste = (function(overriden)
+    ---@param lines string[]
+    ---@param phase -1|1|2|3
+    return function(lines, phase)
+      require("img-clip.util").verbose = false
+
+      local bufnr = vim.api.nvim_get_current_buf()
+      local filetype = vim.api.nvim_get_option_value("filetype", { buf = bufnr })
+      if filetype ~= "AvanteInput" then
+        return overriden(lines, phase)
+      end
+
+      ---@type string
+      local line = lines[1]
+
+      local ok = Clipboard.paste_image(line)
+      if not ok then
+        return overriden(lines, phase)
+      end
+
+      -- After pasting, insert a new line and set cursor to this line
+      vim.api.nvim_buf_set_lines(bufnr, -1, -1, false, { "" })
+      local last_line = vim.api.nvim_buf_line_count(bufnr)
+      vim.api.nvim_win_set_cursor(0, { last_line, 0 })
+    end
+  end)(vim.paste)
+end