Revert tokenizers (#423)

* Revert "fix: add missing "with" in README"

This reverts commit bf1e7f1f17.

* Revert "feat: tokenizers (#407)"

This reverts commit d2095ba267.

crates/avante-tokenizers/Cargo.toml

@@ -1,32 +0,0 @@
-[lib]
-crate-type = ["cdylib"]
-
-[package]
-name = "avante-tokenizers"
-edition = { workspace = true }
-version = { workspace = true }
-rust-version = { workspace = true }
-license = { workspace = true }
-
-[lints]
-workspace = true
-
-[dependencies]
-mlua = { version = "0.10.0-beta.1", features = [
-  "module",
-  "serialize",
-], git = "https://github.com/mlua-rs/mlua.git", branch = "main" }
-tiktoken-rs = "0.5.9"
-tokenizers = { version = "0.20.0", features = [
-  "esaxx_fast",
-  "http",
-  "unstable_wasm",
-  "onig",
-], default-features = false }
-
-[features]
-lua51 = ["mlua/lua51"]
-lua52 = ["mlua/lua52"]
-lua53 = ["mlua/lua53"]
-lua54 = ["mlua/lua54"]
-luajit = ["mlua/luajit"]