feat: tokenizers (#407)

* feat: autobuild tiktoken lib and schenanigans

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* chore: revert readme changes

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* fix(build): windows

Signed-off-by: Hanchin Hsieh <me@yuchanns.xyz>

* chore(plugin): early load commands and base setup

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* fix(build): make sync

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* feat: rust go vroom vroom

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* feat: scuffed afaf implementation binding go brrrr

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* chore: remove dups

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* fix(tokens): calculate whether we should do prompt_caching (fixes #416)

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* chore: ignore lockfiles

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* Update README.md

* Update crates/avante-tokenizers/README.md

* chore: remove unused

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* chore: remove auto build

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

---------

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>
Signed-off-by: Hanchin Hsieh <me@yuchanns.xyz>
Co-authored-by: yuchanns <me@yuchanns.xyz>
This commit is contained in:
Aaron Pham
2024-08-31 07:19:59 -04:00
committed by GitHub
parent 81b44e4533
commit d2095ba267
27 changed files with 3554 additions and 27 deletions

View File

@@ -30,11 +30,7 @@ https://github.com/user-attachments/assets/86140bfd-08b4-483d-a887-1b701d9e37dd
opts = {
-- add any opts here
},
keys = {
{ "<leader>aa", function() require("avante.api").ask() end, desc = "avante: ask", mode = { "n", "v" } },
{ "<leader>ar", function() require("avante.api").refresh() end, desc = "avante: refresh" },
{ "<leader>ae", function() require("avante.api").edit() end, desc = "avante: edit", mode = "v" },
},
build = ":AvanteBuild", -- This is optional, recommended tho. Also note that this will block the startup for a bit since we are compiling bindings in Rust.
dependencies = {
"stevearc/dressing.nvim",
"nvim-lua/plenary.nvim",
@@ -90,9 +86,14 @@ Plug 'HakonHarnes/img-clip.nvim'
Plug 'zbirenbaum/copilot.lua'
" Yay
Plug 'yetone/avante.nvim'
Plug 'yetone/avante.nvim', { 'branch': 'main', 'do': ':AvanteBuild', 'on': 'AvanteAsk' }
```
> [!important]
>
> For `avante.tokenizers` to work, make sure to call `require('avante_lib').load()` somewhere when entering the editor.
> We will leave the users to decide where it fits to do this, as this varies among configurations. (But we do recommend running this after where you set your colorscheme)
</details>
<details>
@@ -100,32 +101,32 @@ Plug 'yetone/avante.nvim'
<summary><a href="https://github.com/echasnovski/mini.deps">mini.deps</a></summary>
```lua
local add, later = MiniDeps.add, MiniDeps.later
local add, later, now = MiniDeps.add, MiniDeps.later, MiniDeps.now
add({
source = 'yetone/avante.nvim',
monitor = 'main',
depends = {
'stevearc/dressing.nvim',
'nvim-lua/plenary.nvim',
'MunifTanjim/nui.nvim',
'echasnovski/mini.icons'
},
hooks = { post_checkout = function() vim.cmd('AvanteBuild') end }
})
--- optional
add({ source = 'zbirenbaum/copilot.lua' })
add({ source = 'HakonHarnes/img-clip.nvim' })
add({ source = 'MeanderingProgrammer/render-markdown.nvim' })
now(function() require('avante_lib').load() end)
later(function() require('render-markdown').setup({...}) end)
later(function()
require('img-clip').setup({...}) -- config img-clip
require("copilot").setup({...}) -- setup copilot to your liking
require("avante").setup({...}) -- config for avante.nvim
end)
```
```
```
</details>
@@ -171,6 +172,10 @@ require('avante').setup ({
> }
> ```
> [!TIP]
>
> Any rendering plugins that support markdown should work Avante as long as you add the supported filetype `Avante`. See https://github.com/yetone/avante.nvim/issues/175 and [this comment](https://github.com/yetone/avante.nvim/issues/175#issuecomment-2313749363) for more information.
### Default setup configuration
_See [config.lua#L9](./lua/avante/config.lua) for the full config_