Revert tokenizers (#423)

* Revert "fix: add missing "with" in README"

This reverts commit bf1e7f1f17.

* Revert "feat: tokenizers (#407)"

This reverts commit d2095ba267.

Build.ps1

@@ -1,29 +0,0 @@
-param (
-    [string]$Version = "luajit"
-)
-
-$BuildDir = "build"
-$BuildFromSource = $true
-
-function Build-FromSource($feature) {
-    if (-not (Test-Path $BuildDir)) {
-        New-Item -ItemType Directory -Path $BuildDir | Out-Null
-    }
-
-    cargo build --release --features=$feature
-
-    $targetFile = "avante_tokenizers.dll"
-    Copy-Item (Join-Path "target\release\libavante_tokenizers.dll") (Join-Path $BuildDir $targetFile)
-
-    Remove-Item -Recurse -Force "target"
-}
-
-function Main {
-    Set-Location $PSScriptRoot
-    Write-Host "Building for $Version..."
-    Build-FromSource $Version
-    Write-Host "Completed!"
-}
-
-# Run the main function
-Main