feat: tokenizers (#407)

* feat: autobuild tiktoken lib and schenanigans

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* chore: revert readme changes

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* fix(build): windows

Signed-off-by: Hanchin Hsieh <me@yuchanns.xyz>

* chore(plugin): early load commands and base setup

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* fix(build): make sync

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* feat: rust go vroom vroom

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* feat: scuffed afaf implementation binding go brrrr

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* chore: remove dups

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* fix(tokens): calculate whether we should do prompt_caching (fixes #416)

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* chore: ignore lockfiles

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* Update README.md

* Update crates/avante-tokenizers/README.md

* chore: remove unused

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

* chore: remove auto build

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

---------

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>
Signed-off-by: Hanchin Hsieh <me@yuchanns.xyz>
Co-authored-by: yuchanns <me@yuchanns.xyz>

Makefile

@@ -1,5 +1,48 @@
-hello:
-	@echo Hello avante.nvim!
+UNAME := $(shell uname)
+ARCH := $(shell uname -m)
+
+ifeq ($(UNAME), Linux)
+	OS := linux
+	EXT := so
+else ifeq ($(UNAME), Darwin)
+	OS := macOS
+	EXT := dylib
+else
+	$(error Unsupported operating system: $(UNAME))
+endif
+
+LUA_VERSIONS := luajit lua51
+BUILD_DIR := build
+
+all: luajit
+
+luajit: $(BUILD_DIR)/libavante_tokenizers.$(EXT)
+lua51: $(BUILD_DIR)/libavante_tokenizers-lua51.$(EXT)
+lua52: $(BUILD_DIR)/libavante_tokenizers-lua52.$(EXT)
+lua53: $(BUILD_DIR)/libavante_tokenizers-lua53.$(EXT)
+lua54: $(BUILD_DIR)/libavante_tokenizers-lua54.$(EXT)
+
+define build_from_source
+	cargo build --release --features=$1
+	cp target/release/libavante_tokenizers.$(EXT) $(BUILD_DIR)/avante_tokenizers.$(EXT)
+endef
+
+$(BUILD_DIR)/libavante_tokenizers.$(EXT): $(BUILD_DIR)
+	$(call build_from_source,luajit)
+$(BUILD_DIR)/libavante_tokenizers-lua51.$(EXT): $(BUILD_DIR)
+	$(call build_from_source,lua51)
+$(BUILD_DIR)/libavante_tokenizers-lua52.$(EXT): $(BUILD_DIR)
+	$(call build_from_source,lua52)
+$(BUILD_DIR)/libavante_tokenizers-lua53.$(EXT): $(BUILD_DIR)
+	$(call build_from_source,lua53)
+$(BUILD_DIR)/libavante_tokenizers-lua54.$(EXT): $(BUILD_DIR)
+	$(call build_from_source,lua54)
+
+$(BUILD_DIR):
+	mkdir -p $(BUILD_DIR)
+
+clean:
+	rm -rf $(BUILD_DIR)
 
 luacheck:
 	luacheck `find -name "*.lua"` --codes