CarGDev/codetyper.nvim
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adding autocomplete and copilot suggestions

Browse Source
This commit is contained in:
Carlos
2026-01-14 21:43:56 -05:00
parent 5493a5ec38
commit 84c8bcf92c
55 changed files with 11823 additions and 550 deletions
Download Patch File Download Diff File Expand all files Collapse all files

582
lua/codetyper/indexer/analyzer.lua Normal file
View File

@@ -0,0 +1,582 @@
---@mod codetyper.indexer.analyzer Code analyzer using Tree-sitter
---@brief [[
--- Analyzes source files to extract functions, classes, exports, and imports.
--- Uses Tree-sitter when available, falls back to pattern matching.
---@brief ]]
local M = {}
local utils = require("codetyper.utils")
local scanner = require("codetyper.indexer.scanner")
--- Language-specific query patterns for Tree-sitter
local TS_QUERIES = {
lua = {
functions = [[
(function_declaration name: (identifier) @name) @func
(function_definition) @func
(local_function name: (identifier) @name) @func
(assignment_statement
(variable_list name: (identifier) @name)
(expression_list value: (function_definition) @func))
]],
exports = [[
(return_statement (expression_list (table_constructor))) @export
]],
},
typescript = {
functions = [[
(function_declaration name: (identifier) @name) @func
(method_definition name: (property_identifier) @name) @func
(arrow_function) @func
(lexical_declaration
(variable_declarator name: (identifier) @name value: (arrow_function) @func))
]],
exports = [[
(export_statement) @export
]],
imports = [[
(import_statement) @import
]],
},
javascript = {
functions = [[
(function_declaration name: (identifier) @name) @func
(method_definition name: (property_identifier) @name) @func
(arrow_function) @func
]],
exports = [[
(export_statement) @export
]],
imports = [[
(import_statement) @import
]],
},
python = {
functions = [[
(function_definition name: (identifier) @name) @func
]],
classes = [[
(class_definition name: (identifier) @name) @class
]],
imports = [[
(import_statement) @import
(import_from_statement) @import
]],
},
go = {
functions = [[
(function_declaration name: (identifier) @name) @func
(method_declaration name: (field_identifier) @name) @func
]],
imports = [[
(import_declaration) @import
]],
},
rust = {
functions = [[
(function_item name: (identifier) @name) @func
]],
imports = [[
(use_declaration) @import
]],
},
}
--- Hash file content for change detection
---@param content string
---@return string
local function hash_content(content)
local hash = 0
for i = 1, math.min(#content, 10000) do
hash = (hash * 31 + string.byte(content, i)) % 2147483647
end
return string.format("%08x", hash)
end
--- Try to get Tree-sitter parser for a language
---@param lang string
---@return boolean
local function has_ts_parser(lang)
local ok = pcall(vim.treesitter.language.inspect, lang)
return ok
end
--- Analyze file using Tree-sitter
---@param filepath string
---@param lang string
---@param content string
---@return table|nil
local function analyze_with_treesitter(filepath, lang, content)
if not has_ts_parser(lang) then
return nil
end
local result = {
functions = {},
classes = {},
exports = {},
imports = {},
}
-- Create a temporary buffer for parsing
local bufnr = vim.api.nvim_create_buf(false, true)
vim.api.nvim_buf_set_lines(bufnr, 0, -1, false, vim.split(content, "\n"))
local ok, parser = pcall(vim.treesitter.get_parser, bufnr, lang)
if not ok or not parser then
vim.api.nvim_buf_delete(bufnr, { force = true })
return nil
end
local tree = parser:parse()[1]
if not tree then
vim.api.nvim_buf_delete(bufnr, { force = true })
return nil
end
local root = tree:root()
local queries = TS_QUERIES[lang]
if not queries then
-- Fallback: walk tree manually for common patterns
result = analyze_tree_generic(root, bufnr)
else
-- Use language-specific queries
if queries.functions then
local query_ok, query = pcall(vim.treesitter.query.parse, lang, queries.functions)
if query_ok then
for id, node in query:iter_captures(root, bufnr, 0, -1) do
local capture_name = query.captures[id]
if capture_name == "func" or capture_name == "name" then
local start_row, _, end_row, _ = node:range()
local name = nil
-- Try to get name from sibling capture or child
if capture_name == "func" then
local name_node = node:field("name")[1]
if name_node then
name = vim.treesitter.get_node_text(name_node, bufnr)
end
else
name = vim.treesitter.get_node_text(node, bufnr)
end
if name and not vim.tbl_contains(vim.tbl_map(function(f)
return f.name
end, result.functions), name) then
table.insert(result.functions, {
name = name,
line = start_row + 1,
end_line = end_row + 1,
params = {},
})
end
end
end
end
end
if queries.classes then
local query_ok, query = pcall(vim.treesitter.query.parse, lang, queries.classes)
if query_ok then
for id, node in query:iter_captures(root, bufnr, 0, -1) do
local capture_name = query.captures[id]
if capture_name == "class" then
local start_row, _, end_row, _ = node:range()
local name_node = node:field("name")[1]
local name = name_node and vim.treesitter.get_node_text(name_node, bufnr) or "anonymous"
table.insert(result.classes, {
name = name,
line = start_row + 1,
end_line = end_row + 1,
methods = {},
})
end
end
end
end
if queries.exports then
local query_ok, query = pcall(vim.treesitter.query.parse, lang, queries.exports)
if query_ok then
for _, node in query:iter_captures(root, bufnr, 0, -1) do
local text = vim.treesitter.get_node_text(node, bufnr)
local start_row, _, _, _ = node:range()
-- Extract export names (simplified)
local names = {}
for name in text:gmatch("export%s+[%w_]+%s+([%w_]+)") do
table.insert(names, name)
end
for name in text:gmatch("export%s*{([^}]+)}") do
for n in name:gmatch("([%w_]+)") do
table.insert(names, n)
end
end
for _, name in ipairs(names) do
table.insert(result.exports, {
name = name,
type = "unknown",
line = start_row + 1,
})
end
end
end
end
if queries.imports then
local query_ok, query = pcall(vim.treesitter.query.parse, lang, queries.imports)
if query_ok then
for _, node in query:iter_captures(root, bufnr, 0, -1) do
local text = vim.treesitter.get_node_text(node, bufnr)
local start_row, _, _, _ = node:range()
-- Extract import source
local source = text:match('["\']([^"\']+)["\']')
if source then
table.insert(result.imports, {
source = source,
names = {},
line = start_row + 1,
})
end
end
end
end
end
vim.api.nvim_buf_delete(bufnr, { force = true })
return result
end
--- Generic tree analysis for unsupported languages
---@param root TSNode
---@param bufnr number
---@return table
local function analyze_tree_generic(root, bufnr)
local result = {
functions = {},
classes = {},
exports = {},
imports = {},
}
local function visit(node)
local node_type = node:type()
-- Common function patterns
if
node_type:match("function")
or node_type:match("method")
or node_type == "arrow_function"
or node_type == "func_literal"
then
local start_row, _, end_row, _ = node:range()
local name_node = node:field("name")[1]
local name = name_node and vim.treesitter.get_node_text(name_node, bufnr) or "anonymous"
table.insert(result.functions, {
name = name,
line = start_row + 1,
end_line = end_row + 1,
params = {},
})
end
-- Common class patterns
if node_type:match("class") or node_type == "struct_item" or node_type == "impl_item" then
local start_row, _, end_row, _ = node:range()
local name_node = node:field("name")[1]
local name = name_node and vim.treesitter.get_node_text(name_node, bufnr) or "anonymous"
table.insert(result.classes, {
name = name,
line = start_row + 1,
end_line = end_row + 1,
methods = {},
})
end
-- Recurse into children
for child in node:iter_children() do
visit(child)
end
end
visit(root)
return result
end
--- Analyze file using pattern matching (fallback)
---@param content string
---@param lang string
---@return table
local function analyze_with_patterns(content, lang)
local result = {
functions = {},
classes = {},
exports = {},
imports = {},
}
local lines = vim.split(content, "\n")
-- Language-specific patterns
local patterns = {
lua = {
func_start = "^%s*local?%s*function%s+([%w_%.]+)",
func_assign = "^%s*([%w_%.]+)%s*=%s*function",
module_return = "^return%s+M",
},
javascript = {
func_start = "^%s*function%s+([%w_]+)",
func_arrow = "^%s*const%s+([%w_]+)%s*=%s*",
class_start = "^%s*class%s+([%w_]+)",
export_line = "^%s*export%s+",
import_line = "^%s*import%s+",
},
typescript = {
func_start = "^%s*function%s+([%w_]+)",
func_arrow = "^%s*const%s+([%w_]+)%s*=%s*",
class_start = "^%s*class%s+([%w_]+)",
export_line = "^%s*export%s+",
import_line = "^%s*import%s+",
},
python = {
func_start = "^%s*def%s+([%w_]+)",
class_start = "^%s*class%s+([%w_]+)",
import_line = "^%s*import%s+",
from_import = "^%s*from%s+",
},
go = {
func_start = "^func%s+([%w_]+)",
method_start = "^func%s+%([^%)]+%)%s+([%w_]+)",
import_line = "^import%s+",
},
rust = {
func_start = "^%s*pub?%s*fn%s+([%w_]+)",
struct_start = "^%s*pub?%s*struct%s+([%w_]+)",
impl_start = "^%s*impl%s+([%w_<>]+)",
use_line = "^%s*use%s+",
},
}
local lang_patterns = patterns[lang] or patterns.javascript
for i, line in ipairs(lines) do
-- Functions
if lang_patterns.func_start then
local name = line:match(lang_patterns.func_start)
if name then
table.insert(result.functions, {
name = name,
line = i,
end_line = i,
params = {},
})
end
end
if lang_patterns.func_arrow then
local name = line:match(lang_patterns.func_arrow)
if name and line:match("=>") then
table.insert(result.functions, {
name = name,
line = i,
end_line = i,
params = {},
})
end
end
if lang_patterns.func_assign then
local name = line:match(lang_patterns.func_assign)
if name then
table.insert(result.functions, {
name = name,
line = i,
end_line = i,
params = {},
})
end
end
if lang_patterns.method_start then
local name = line:match(lang_patterns.method_start)
if name then
table.insert(result.functions, {
name = name,
line = i,
end_line = i,
params = {},
})
end
end
-- Classes
if lang_patterns.class_start then
local name = line:match(lang_patterns.class_start)
if name then
table.insert(result.classes, {
name = name,
line = i,
end_line = i,
methods = {},
})
end
end
if lang_patterns.struct_start then
local name = line:match(lang_patterns.struct_start)
if name then
table.insert(result.classes, {
name = name,
line = i,
end_line = i,
methods = {},
})
end
end
-- Exports
if lang_patterns.export_line and line:match(lang_patterns.export_line) then
local name = line:match("export%s+[%w_]+%s+([%w_]+)")
or line:match("export%s+default%s+([%w_]+)")
or line:match("export%s+{%s*([%w_]+)")
if name then
table.insert(result.exports, {
name = name,
type = "unknown",
line = i,
})
end
end
-- Imports
if lang_patterns.import_line and line:match(lang_patterns.import_line) then
local source = line:match('["\']([^"\']+)["\']')
if source then
table.insert(result.imports, {
source = source,
names = {},
line = i,
})
end
end
if lang_patterns.from_import and line:match(lang_patterns.from_import) then
local source = line:match("from%s+([%w_%.]+)")
if source then
table.insert(result.imports, {
source = source,
names = {},
line = i,
})
end
end
if lang_patterns.use_line and line:match(lang_patterns.use_line) then
local source = line:match("use%s+([%w_:]+)")
if source then
table.insert(result.imports, {
source = source,
names = {},
line = i,
})
end
end
end
-- For Lua, infer exports from module table
if lang == "lua" then
for _, func in ipairs(result.functions) do
if func.name:match("^M%.") then
local name = func.name:gsub("^M%.", "")
table.insert(result.exports, {
name = name,
type = "function",
line = func.line,
})
end
end
end
return result
end
--- Analyze a single file
---@param filepath string Full path to file
---@return FileIndex|nil
function M.analyze_file(filepath)
local content = utils.read_file(filepath)
if not content then
return nil
end
local lang = scanner.get_language(filepath)
-- Map to Tree-sitter language names
local ts_lang_map = {
typescript = "typescript",
typescriptreact = "tsx",
javascript = "javascript",
javascriptreact = "javascript",
python = "python",
go = "go",
rust = "rust",
lua = "lua",
}
local ts_lang = ts_lang_map[lang] or lang
-- Try Tree-sitter first
local analysis = analyze_with_treesitter(filepath, ts_lang, content)
-- Fallback to pattern matching
if not analysis then
analysis = analyze_with_patterns(content, lang)
end
return {
path = filepath,
language = lang,
hash = hash_content(content),
exports = analysis.exports,
imports = analysis.imports,
functions = analysis.functions,
classes = analysis.classes,
last_indexed = os.time(),
}
end
--- Extract exports from a buffer
---@param bufnr number
---@return Export[]
function M.extract_exports(bufnr)
local filepath = vim.api.nvim_buf_get_name(bufnr)
local analysis = M.analyze_file(filepath)
return analysis and analysis.exports or {}
end
--- Extract functions from a buffer
---@param bufnr number
---@return FunctionInfo[]
function M.extract_functions(bufnr)
local filepath = vim.api.nvim_buf_get_name(bufnr)
local analysis = M.analyze_file(filepath)
return analysis and analysis.functions or {}
end
--- Extract imports from a buffer
---@param bufnr number
---@return Import[]
function M.extract_imports(bufnr)
local filepath = vim.api.nvim_buf_get_name(bufnr)
local analysis = M.analyze_file(filepath)
return analysis and analysis.imports or {}
end
return M

604
lua/codetyper/indexer/init.lua Normal file
View File

@@ -0,0 +1,604 @@
---@mod codetyper.indexer Project indexer for Codetyper.nvim
---@brief [[
--- Indexes project structure, dependencies, and code symbols.
--- Stores knowledge in .coder/ directory for enriching LLM context.
---@brief ]]
local M = {}
local utils = require("codetyper.utils")
--- Index schema version for migrations
local INDEX_VERSION = 1
--- Index file name
local INDEX_FILE = "index.json"
--- Debounce timer for file indexing
local index_timer = nil
local INDEX_DEBOUNCE_MS = 500
--- Default indexer configuration
local default_config = {
enabled = true,
auto_index = true,
index_on_open = false,
max_file_size = 100000,
excluded_dirs = { "node_modules", "dist", "build", ".git", ".coder", "__pycache__", "vendor", "target" },
index_extensions = { "lua", "ts", "tsx", "js", "jsx", "py", "go", "rs", "rb", "java", "c", "cpp", "h", "hpp" },
memory = {
enabled = true,
max_memories = 1000,
prune_threshold = 0.1,
},
}
--- Current configuration
---@type table
local config = vim.deepcopy(default_config)
--- Cached project index
---@type table<string, ProjectIndex>
local index_cache = {}
---@class ProjectIndex
---@field version number Index schema version
---@field project_root string Absolute path to project
---@field project_name string Project name
---@field project_type string "node"|"rust"|"go"|"python"|"lua"|"unknown"
---@field dependencies table<string, string> name -> version
---@field dev_dependencies table<string, string> name -> version
---@field files table<string, FileIndex> path -> FileIndex
---@field symbols table<string, string[]> symbol -> [file paths]
---@field last_indexed number Timestamp
---@field stats {files: number, functions: number, classes: number, exports: number}
---@class FileIndex
---@field path string Relative path from project root
---@field language string Detected language
---@field hash string Content hash for change detection
---@field exports Export[] Exported symbols
---@field imports Import[] Dependencies
---@field functions FunctionInfo[]
---@field classes ClassInfo[]
---@field last_indexed number Timestamp
---@class Export
---@field name string Symbol name
---@field type string "function"|"class"|"constant"|"type"|"variable"
---@field line number Line number
---@class Import
---@field source string Import source/module
---@field names string[] Imported names
---@field line number Line number
---@class FunctionInfo
---@field name string Function name
---@field params string[] Parameter names
---@field line number Start line
---@field end_line number End line
---@field docstring string|nil Documentation
---@class ClassInfo
---@field name string Class name
---@field methods string[] Method names
---@field line number Start line
---@field end_line number End line
---@field docstring string|nil Documentation
--- Get the index file path
---@return string|nil
local function get_index_path()
local root = utils.get_project_root()
if not root then
return nil
end
return root .. "/.coder/" .. INDEX_FILE
end
--- Create empty index structure
---@return ProjectIndex
local function create_empty_index()
local root = utils.get_project_root()
return {
version = INDEX_VERSION,
project_root = root or "",
project_name = root and vim.fn.fnamemodify(root, ":t") or "",
project_type = "unknown",
dependencies = {},
dev_dependencies = {},
files = {},
symbols = {},
last_indexed = os.time(),
stats = {
files = 0,
functions = 0,
classes = 0,
exports = 0,
},
}
end
--- Load index from disk
---@return ProjectIndex|nil
function M.load_index()
local root = utils.get_project_root()
if not root then
return nil
end
-- Check cache first
if index_cache[root] then
return index_cache[root]
end
local path = get_index_path()
if not path then
return nil
end
local content = utils.read_file(path)
if not content then
return nil
end
local ok, index = pcall(vim.json.decode, content)
if not ok or not index then
return nil
end
-- Validate version
if index.version ~= INDEX_VERSION then
-- Index needs migration or rebuild
return nil
end
-- Cache it
index_cache[root] = index
return index
end
--- Save index to disk
---@param index ProjectIndex
---@return boolean
function M.save_index(index)
local root = utils.get_project_root()
if not root then
return false
end
-- Ensure .coder directory exists
local coder_dir = root .. "/.coder"
utils.ensure_dir(coder_dir)
local path = get_index_path()
if not path then
return false
end
local ok, encoded = pcall(vim.json.encode, index)
if not ok then
return false
end
local success = utils.write_file(path, encoded)
if success then
-- Update cache
index_cache[root] = index
end
return success
end
--- Index the entire project
---@param callback? fun(index: ProjectIndex)
---@return ProjectIndex|nil
function M.index_project(callback)
local scanner = require("codetyper.indexer.scanner")
local analyzer = require("codetyper.indexer.analyzer")
local index = create_empty_index()
local root = utils.get_project_root()
if not root then
if callback then
callback(index)
end
return index
end
-- Detect project type and parse dependencies
index.project_type = scanner.detect_project_type(root)
local deps = scanner.parse_dependencies(root, index.project_type)
index.dependencies = deps.dependencies or {}
index.dev_dependencies = deps.dev_dependencies or {}
-- Get all indexable files
local files = scanner.get_indexable_files(root, config)
-- Index each file
local total_functions = 0
local total_classes = 0
local total_exports = 0
for _, filepath in ipairs(files) do
local relative_path = filepath:gsub("^" .. vim.pesc(root) .. "/", "")
local file_index = analyzer.analyze_file(filepath)
if file_index then
file_index.path = relative_path
index.files[relative_path] = file_index
-- Update symbol index
for _, exp in ipairs(file_index.exports or {}) do
if not index.symbols[exp.name] then
index.symbols[exp.name] = {}
end
table.insert(index.symbols[exp.name], relative_path)
total_exports = total_exports + 1
end
total_functions = total_functions + #(file_index.functions or {})
total_classes = total_classes + #(file_index.classes or {})
end
end
-- Update stats
index.stats = {
files = #files,
functions = total_functions,
classes = total_classes,
exports = total_exports,
}
index.last_indexed = os.time()
-- Save to disk
M.save_index(index)
-- Store memories
local memory = require("codetyper.indexer.memory")
memory.store_index_summary(index)
-- Sync project summary to brain
M.sync_project_to_brain(index, files, root)
if callback then
callback(index)
end
return index
end
--- Sync project index to brain
---@param index ProjectIndex
---@param files string[] List of file paths
---@param root string Project root
function M.sync_project_to_brain(index, files, root)
local ok_brain, brain = pcall(require, "codetyper.brain")
if not ok_brain or not brain.is_initialized or not brain.is_initialized() then
return
end
-- Store project-level pattern
brain.learn({
type = "pattern",
file = root,
content = {
summary = "Project: "
.. index.project_name
.. " ("
.. index.project_type
.. ") - "
.. index.stats.files
.. " files",
detail = string.format(
"%d functions, %d classes, %d exports",
index.stats.functions,
index.stats.classes,
index.stats.exports
),
},
context = {
file = root,
project_type = index.project_type,
dependencies = index.dependencies,
},
})
-- Store key file patterns (files with most functions/classes)
local key_files = {}
for path, file_index in pairs(index.files) do
local score = #(file_index.functions or {}) + (#(file_index.classes or {}) * 2)
if score >= 3 then
table.insert(key_files, { path = path, index = file_index, score = score })
end
end
table.sort(key_files, function(a, b)
return a.score > b.score
end)
-- Store top 20 key files in brain
for i, kf in ipairs(key_files) do
if i > 20 then
break
end
M.sync_to_brain(root .. "/" .. kf.path, kf.index)
end
end
--- Index a single file (incremental update)
---@param filepath string
---@return FileIndex|nil
function M.index_file(filepath)
local analyzer = require("codetyper.indexer.analyzer")
local memory = require("codetyper.indexer.memory")
local root = utils.get_project_root()
if not root then
return nil
end
-- Load existing index
local index = M.load_index() or create_empty_index()
-- Analyze file
local file_index = analyzer.analyze_file(filepath)
if not file_index then
return nil
end
local relative_path = filepath:gsub("^" .. vim.pesc(root) .. "/", "")
file_index.path = relative_path
-- Remove old symbol references for this file
for symbol, paths in pairs(index.symbols) do
for i = #paths, 1, -1 do
if paths[i] == relative_path then
table.remove(paths, i)
end
end
if #paths == 0 then
index.symbols[symbol] = nil
end
end
-- Add new file index
index.files[relative_path] = file_index
-- Update symbol index
for _, exp in ipairs(file_index.exports or {}) do
if not index.symbols[exp.name] then
index.symbols[exp.name] = {}
end
table.insert(index.symbols[exp.name], relative_path)
end
-- Recalculate stats
local total_functions = 0
local total_classes = 0
local total_exports = 0
local file_count = 0
for _, f in pairs(index.files) do
file_count = file_count + 1
total_functions = total_functions + #(f.functions or {})
total_classes = total_classes + #(f.classes or {})
total_exports = total_exports + #(f.exports or {})
end
index.stats = {
files = file_count,
functions = total_functions,
classes = total_classes,
exports = total_exports,
}
index.last_indexed = os.time()
-- Save to disk
M.save_index(index)
-- Store file memory
memory.store_file_memory(relative_path, file_index)
-- Sync to brain if available
M.sync_to_brain(filepath, file_index)
return file_index
end
--- Sync file analysis to brain system
---@param filepath string Full file path
---@param file_index FileIndex File analysis
function M.sync_to_brain(filepath, file_index)
local ok_brain, brain = pcall(require, "codetyper.brain")
if not ok_brain or not brain.is_initialized or not brain.is_initialized() then
return
end
-- Only store if file has meaningful content
local funcs = file_index.functions or {}
local classes = file_index.classes or {}
if #funcs == 0 and #classes == 0 then
return
end
-- Build summary
local parts = {}
if #funcs > 0 then
local func_names = {}
for i, f in ipairs(funcs) do
if i <= 5 then
table.insert(func_names, f.name)
end
end
table.insert(parts, "functions: " .. table.concat(func_names, ", "))
if #funcs > 5 then
table.insert(parts, "(+" .. (#funcs - 5) .. " more)")
end
end
if #classes > 0 then
local class_names = {}
for _, c in ipairs(classes) do
table.insert(class_names, c.name)
end
table.insert(parts, "classes: " .. table.concat(class_names, ", "))
end
local filename = vim.fn.fnamemodify(filepath, ":t")
local summary = filename .. " - " .. table.concat(parts, "; ")
-- Learn this pattern in brain
brain.learn({
type = "pattern",
file = filepath,
content = {
summary = summary,
detail = #funcs .. " functions, " .. #classes .. " classes",
},
context = {
file = file_index.path or filepath,
language = file_index.language,
functions = funcs,
classes = classes,
exports = file_index.exports,
imports = file_index.imports,
},
})
end
--- Schedule file indexing with debounce
---@param filepath string
function M.schedule_index_file(filepath)
if not config.enabled or not config.auto_index then
return
end
-- Check if file should be indexed
local scanner = require("codetyper.indexer.scanner")
if not scanner.should_index(filepath, config) then
return
end
-- Cancel existing timer
if index_timer then
index_timer:stop()
end
-- Schedule new index
index_timer = vim.defer_fn(function()
M.index_file(filepath)
index_timer = nil
end, INDEX_DEBOUNCE_MS)
end
--- Get relevant context for a prompt
---@param opts {file: string, intent: table|nil, prompt: string, scope: string|nil}
---@return table Context information
function M.get_context_for(opts)
local memory = require("codetyper.indexer.memory")
local index = M.load_index()
local context = {
project_type = "unknown",
dependencies = {},
relevant_files = {},
relevant_symbols = {},
patterns = {},
}
if not index then
return context
end
context.project_type = index.project_type
context.dependencies = index.dependencies
-- Find relevant symbols from prompt
local words = {}
for word in opts.prompt:gmatch("%w+") do
if #word > 2 then
words[word:lower()] = true
end
end
-- Match symbols
for symbol, files in pairs(index.symbols) do
if words[symbol:lower()] then
context.relevant_symbols[symbol] = files
end
end
-- Get file context if available
if opts.file then
local root = utils.get_project_root()
if root then
local relative_path = opts.file:gsub("^" .. vim.pesc(root) .. "/", "")
local file_index = index.files[relative_path]
if file_index then
context.current_file = file_index
end
end
end
-- Get relevant memories
context.patterns = memory.get_relevant(opts.prompt, 5)
return context
end
--- Get index status
---@return table Status information
function M.get_status()
local index = M.load_index()
if not index then
return {
indexed = false,
stats = nil,
last_indexed = nil,
}
end
return {
indexed = true,
stats = index.stats,
last_indexed = index.last_indexed,
project_type = index.project_type,
}
end
--- Clear the project index
function M.clear()
local root = utils.get_project_root()
if root then
index_cache[root] = nil
end
local path = get_index_path()
if path and utils.file_exists(path) then
os.remove(path)
end
end
--- Setup the indexer with configuration
---@param opts? table Configuration options
function M.setup(opts)
if opts then
config = vim.tbl_deep_extend("force", config, opts)
end
-- Index on startup if configured
if config.index_on_open then
vim.defer_fn(function()
M.index_project()
end, 1000)
end
end
--- Get current configuration
---@return table
function M.get_config()
return vim.deepcopy(config)
end
return M

539
lua/codetyper/indexer/memory.lua Normal file
View File

@@ -0,0 +1,539 @@
---@mod codetyper.indexer.memory Memory persistence manager
---@brief [[
--- Stores and retrieves learned patterns and memories in .coder/memories/.
--- Supports session history for learning from interactions.
---@brief ]]
local M = {}
local utils = require("codetyper.utils")
--- Memory directories
local MEMORIES_DIR = "memories"
local SESSIONS_DIR = "sessions"
local FILES_DIR = "files"
--- Memory files
local PATTERNS_FILE = "patterns.json"
local CONVENTIONS_FILE = "conventions.json"
local SYMBOLS_FILE = "symbols.json"
--- In-memory cache
local cache = {
patterns = nil,
conventions = nil,
symbols = nil,
}
---@class Memory
---@field id string Unique identifier
---@field type "pattern"|"convention"|"session"|"interaction"
---@field content string The learned information
---@field context table Where/when learned
---@field weight number Importance score (0.0-1.0)
---@field created_at number Timestamp
---@field updated_at number Last update timestamp
---@field used_count number Times referenced
--- Get the memories base directory
---@return string|nil
local function get_memories_dir()
local root = utils.get_project_root()
if not root then
return nil
end
return root .. "/.coder/" .. MEMORIES_DIR
end
--- Get the sessions directory
---@return string|nil
local function get_sessions_dir()
local root = utils.get_project_root()
if not root then
return nil
end
return root .. "/.coder/" .. SESSIONS_DIR
end
--- Ensure memories directory exists
---@return boolean
local function ensure_memories_dir()
local dir = get_memories_dir()
if not dir then
return false
end
utils.ensure_dir(dir)
utils.ensure_dir(dir .. "/" .. FILES_DIR)
return true
end
--- Ensure sessions directory exists
---@return boolean
local function ensure_sessions_dir()
local dir = get_sessions_dir()
if not dir then
return false
end
return utils.ensure_dir(dir)
end
--- Generate a unique ID
---@return string
local function generate_id()
return string.format("mem_%d_%s", os.time(), string.sub(tostring(math.random()), 3, 8))
end
--- Load a memory file
---@param filename string
---@return table
local function load_memory_file(filename)
local dir = get_memories_dir()
if not dir then
return {}
end
local path = dir .. "/" .. filename
local content = utils.read_file(path)
if not content then
return {}
end
local ok, data = pcall(vim.json.decode, content)
if not ok or not data then
return {}
end
return data
end
--- Save a memory file
---@param filename string
---@param data table
---@return boolean
local function save_memory_file(filename, data)
if not ensure_memories_dir() then
return false
end
local dir = get_memories_dir()
if not dir then
return false
end
local path = dir .. "/" .. filename
local ok, encoded = pcall(vim.json.encode, data)
if not ok then
return false
end
return utils.write_file(path, encoded)
end
--- Hash a file path for storage
---@param filepath string
---@return string
local function hash_path(filepath)
local hash = 0
for i = 1, #filepath do
hash = (hash * 31 + string.byte(filepath, i)) % 2147483647
end
return string.format("%08x", hash)
end
--- Load patterns from cache or disk
---@return table
function M.load_patterns()
if cache.patterns then
return cache.patterns
end
cache.patterns = load_memory_file(PATTERNS_FILE)
return cache.patterns
end
--- Load conventions from cache or disk
---@return table
function M.load_conventions()
if cache.conventions then
return cache.conventions
end
cache.conventions = load_memory_file(CONVENTIONS_FILE)
return cache.conventions
end
--- Load symbols from cache or disk
---@return table
function M.load_symbols()
if cache.symbols then
return cache.symbols
end
cache.symbols = load_memory_file(SYMBOLS_FILE)
return cache.symbols
end
--- Store a new memory
---@param memory Memory
---@return boolean
function M.store_memory(memory)
memory.id = memory.id or generate_id()
memory.created_at = memory.created_at or os.time()
memory.updated_at = os.time()
memory.used_count = memory.used_count or 0
memory.weight = memory.weight or 0.5
local filename
if memory.type == "pattern" then
filename = PATTERNS_FILE
cache.patterns = nil
elseif memory.type == "convention" then
filename = CONVENTIONS_FILE
cache.conventions = nil
else
filename = PATTERNS_FILE
cache.patterns = nil
end
local data = load_memory_file(filename)
data[memory.id] = memory
return save_memory_file(filename, data)
end
--- Store file-specific memory
---@param relative_path string Relative file path
---@param file_index table FileIndex data
---@return boolean
function M.store_file_memory(relative_path, file_index)
if not ensure_memories_dir() then
return false
end
local dir = get_memories_dir()
if not dir then
return false
end
local hash = hash_path(relative_path)
local path = dir .. "/" .. FILES_DIR .. "/" .. hash .. ".json"
local data = {
path = relative_path,
indexed_at = os.time(),
functions = file_index.functions or {},
classes = file_index.classes or {},
exports = file_index.exports or {},
imports = file_index.imports or {},
}
local ok, encoded = pcall(vim.json.encode, data)
if not ok then
return false
end
return utils.write_file(path, encoded)
end
--- Load file-specific memory
---@param relative_path string
---@return table|nil
function M.load_file_memory(relative_path)
local dir = get_memories_dir()
if not dir then
return nil
end
local hash = hash_path(relative_path)
local path = dir .. "/" .. FILES_DIR .. "/" .. hash .. ".json"
local content = utils.read_file(path)
if not content then
return nil
end
local ok, data = pcall(vim.json.decode, content)
if not ok then
return nil
end
return data
end
--- Store index summary as memories
---@param index ProjectIndex
function M.store_index_summary(index)
-- Store project type convention
if index.project_type and index.project_type ~= "unknown" then
M.store_memory({
type = "convention",
content = "Project uses " .. index.project_type .. " ecosystem",
context = {
project_root = index.project_root,
detected_at = os.time(),
},
weight = 0.9,
})
end
-- Store dependency patterns
local dep_count = 0
for _ in pairs(index.dependencies or {}) do
dep_count = dep_count + 1
end
if dep_count > 0 then
local deps_list = {}
for name, _ in pairs(index.dependencies) do
table.insert(deps_list, name)
end
M.store_memory({
type = "pattern",
content = "Project dependencies: " .. table.concat(deps_list, ", "),
context = {
dependency_count = dep_count,
},
weight = 0.7,
})
end
-- Update symbol cache
cache.symbols = nil
save_memory_file(SYMBOLS_FILE, index.symbols or {})
end
--- Store session interaction
---@param interaction {prompt: string, response: string, file: string|nil, success: boolean}
function M.store_session(interaction)
if not ensure_sessions_dir() then
return
end
local dir = get_sessions_dir()
if not dir then
return
end
-- Use date-based session files
local date = os.date("%Y-%m-%d")
local path = dir .. "/" .. date .. ".json"
local sessions = {}
local content = utils.read_file(path)
if content then
local ok, data = pcall(vim.json.decode, content)
if ok and data then
sessions = data
end
end
table.insert(sessions, {
timestamp = os.time(),
prompt = interaction.prompt,
response = string.sub(interaction.response or "", 1, 500), -- Truncate
file = interaction.file,
success = interaction.success,
})
-- Limit session size
if #sessions > 100 then
sessions = { unpack(sessions, #sessions - 99) }
end
local ok, encoded = pcall(vim.json.encode, sessions)
if ok then
utils.write_file(path, encoded)
end
end
--- Get relevant memories for a query
---@param query string Search query
---@param limit number Maximum results
---@return Memory[]
function M.get_relevant(query, limit)
limit = limit or 10
local results = {}
-- Tokenize query
local query_words = {}
for word in query:lower():gmatch("%w+") do
if #word > 2 then
query_words[word] = true
end
end
-- Search patterns
local patterns = M.load_patterns()
for _, memory in pairs(patterns) do
local score = 0
local content_lower = (memory.content or ""):lower()
for word in pairs(query_words) do
if content_lower:find(word, 1, true) then
score = score + 1
end
end
if score > 0 then
memory.relevance_score = score * (memory.weight or 0.5)
table.insert(results, memory)
end
end
-- Search conventions
local conventions = M.load_conventions()
for _, memory in pairs(conventions) do
local score = 0
local content_lower = (memory.content or ""):lower()
for word in pairs(query_words) do
if content_lower:find(word, 1, true) then
score = score + 1
end
end
if score > 0 then
memory.relevance_score = score * (memory.weight or 0.5)
table.insert(results, memory)
end
end
-- Sort by relevance
table.sort(results, function(a, b)
return (a.relevance_score or 0) > (b.relevance_score or 0)
end)
-- Limit results
local limited = {}
for i = 1, math.min(limit, #results) do
limited[i] = results[i]
end
return limited
end
--- Update memory usage count
---@param memory_id string
function M.update_usage(memory_id)
local patterns = M.load_patterns()
if patterns[memory_id] then
patterns[memory_id].used_count = (patterns[memory_id].used_count or 0) + 1
patterns[memory_id].updated_at = os.time()
save_memory_file(PATTERNS_FILE, patterns)
cache.patterns = nil
return
end
local conventions = M.load_conventions()
if conventions[memory_id] then
conventions[memory_id].used_count = (conventions[memory_id].used_count or 0) + 1
conventions[memory_id].updated_at = os.time()
save_memory_file(CONVENTIONS_FILE, conventions)
cache.conventions = nil
end
end
--- Get all memories
---@return {patterns: table, conventions: table, symbols: table}
function M.get_all()
return {
patterns = M.load_patterns(),
conventions = M.load_conventions(),
symbols = M.load_symbols(),
}
end
--- Clear all memories
---@param pattern? string Optional pattern to match memory IDs
function M.clear(pattern)
if not pattern then
-- Clear all
cache = { patterns = nil, conventions = nil, symbols = nil }
save_memory_file(PATTERNS_FILE, {})
save_memory_file(CONVENTIONS_FILE, {})
save_memory_file(SYMBOLS_FILE, {})
return
end
-- Clear matching pattern
local patterns = M.load_patterns()
for id in pairs(patterns) do
if id:match(pattern) then
patterns[id] = nil
end
end
save_memory_file(PATTERNS_FILE, patterns)
cache.patterns = nil
local conventions = M.load_conventions()
for id in pairs(conventions) do
if id:match(pattern) then
conventions[id] = nil
end
end
save_memory_file(CONVENTIONS_FILE, conventions)
cache.conventions = nil
end
--- Prune low-weight memories
---@param threshold number Weight threshold (default: 0.1)
function M.prune(threshold)
threshold = threshold or 0.1
local patterns = M.load_patterns()
local pruned = 0
for id, memory in pairs(patterns) do
if (memory.weight or 0) < threshold and (memory.used_count or 0) == 0 then
patterns[id] = nil
pruned = pruned + 1
end
end
if pruned > 0 then
save_memory_file(PATTERNS_FILE, patterns)
cache.patterns = nil
end
local conventions = M.load_conventions()
for id, memory in pairs(conventions) do
if (memory.weight or 0) < threshold and (memory.used_count or 0) == 0 then
conventions[id] = nil
pruned = pruned + 1
end
end
if pruned > 0 then
save_memory_file(CONVENTIONS_FILE, conventions)
cache.conventions = nil
end
return pruned
end
--- Get memory statistics
---@return table
function M.get_stats()
local patterns = M.load_patterns()
local conventions = M.load_conventions()
local symbols = M.load_symbols()
local pattern_count = 0
for _ in pairs(patterns) do
pattern_count = pattern_count + 1
end
local convention_count = 0
for _ in pairs(conventions) do
convention_count = convention_count + 1
end
local symbol_count = 0
for _ in pairs(symbols) do
symbol_count = symbol_count + 1
end
return {
patterns = pattern_count,
conventions = convention_count,
symbols = symbol_count,
total = pattern_count + convention_count,
}
end
return M

409
lua/codetyper/indexer/scanner.lua Normal file
View File

@@ -0,0 +1,409 @@
---@mod codetyper.indexer.scanner File scanner for project indexing
---@brief [[
--- Discovers indexable files, detects project type, and parses dependencies.
---@brief ]]
local M = {}
local utils = require("codetyper.utils")
--- Project type markers
local PROJECT_MARKERS = {
node = { "package.json" },
rust = { "Cargo.toml" },
go = { "go.mod" },
python = { "pyproject.toml", "setup.py", "requirements.txt" },
lua = { "init.lua", ".luarc.json" },
ruby = { "Gemfile" },
java = { "pom.xml", "build.gradle" },
csharp = { "*.csproj", "*.sln" },
}
--- File extension to language mapping
local EXTENSION_LANGUAGE = {
lua = "lua",
ts = "typescript",
tsx = "typescriptreact",
js = "javascript",
jsx = "javascriptreact",
py = "python",
go = "go",
rs = "rust",
rb = "ruby",
java = "java",
c = "c",
cpp = "cpp",
h = "c",
hpp = "cpp",
cs = "csharp",
}
--- Default ignore patterns
local DEFAULT_IGNORES = {
"^%.", -- Hidden files/folders
"^node_modules$",
"^__pycache__$",
"^%.git$",
"^%.coder$",
"^dist$",
"^build$",
"^target$",
"^vendor$",
"^%.next$",
"^%.nuxt$",
"^coverage$",
"%.min%.js$",
"%.min%.css$",
"%.map$",
"%.lock$",
"%-lock%.json$",
}
--- Detect project type from root markers
---@param root string Project root path
---@return string Project type
function M.detect_project_type(root)
for project_type, markers in pairs(PROJECT_MARKERS) do
for _, marker in ipairs(markers) do
local path = root .. "/" .. marker
if marker:match("^%*") then
-- Glob pattern
local pattern = marker:gsub("^%*", "")
local entries = vim.fn.glob(root .. "/*" .. pattern, false, true)
if #entries > 0 then
return project_type
end
else
if utils.file_exists(path) then
return project_type
end
end
end
end
return "unknown"
end
--- Parse project dependencies
---@param root string Project root path
---@param project_type string Project type
---@return {dependencies: table<string, string>, dev_dependencies: table<string, string>}
function M.parse_dependencies(root, project_type)
local deps = {
dependencies = {},
dev_dependencies = {},
}
if project_type == "node" then
deps = M.parse_package_json(root)
elseif project_type == "rust" then
deps = M.parse_cargo_toml(root)
elseif project_type == "go" then
deps = M.parse_go_mod(root)
elseif project_type == "python" then
deps = M.parse_python_deps(root)
end
return deps
end
--- Parse package.json for Node.js projects
---@param root string Project root path
---@return {dependencies: table, dev_dependencies: table}
function M.parse_package_json(root)
local path = root .. "/package.json"
local content = utils.read_file(path)
if not content then
return { dependencies = {}, dev_dependencies = {} }
end
local ok, pkg = pcall(vim.json.decode, content)
if not ok or not pkg then
return { dependencies = {}, dev_dependencies = {} }
end
return {
dependencies = pkg.dependencies or {},
dev_dependencies = pkg.devDependencies or {},
}
end
--- Parse Cargo.toml for Rust projects
---@param root string Project root path
---@return {dependencies: table, dev_dependencies: table}
function M.parse_cargo_toml(root)
local path = root .. "/Cargo.toml"
local content = utils.read_file(path)
if not content then
return { dependencies = {}, dev_dependencies = {} }
end
local deps = {}
local dev_deps = {}
local in_deps = false
local in_dev_deps = false
for line in content:gmatch("[^\n]+") do
if line:match("^%[dependencies%]") then
in_deps = true
in_dev_deps = false
elseif line:match("^%[dev%-dependencies%]") then
in_deps = false
in_dev_deps = true
elseif line:match("^%[") then
in_deps = false
in_dev_deps = false
elseif in_deps or in_dev_deps then
local name, version = line:match('^([%w_%-]+)%s*=%s*"([^"]+)"')
if not name then
name = line:match("^([%w_%-]+)%s*=")
version = "workspace"
end
if name then
if in_deps then
deps[name] = version or "unknown"
else
dev_deps[name] = version or "unknown"
end
end
end
end
return { dependencies = deps, dev_dependencies = dev_deps }
end
--- Parse go.mod for Go projects
---@param root string Project root path
---@return {dependencies: table, dev_dependencies: table}
function M.parse_go_mod(root)
local path = root .. "/go.mod"
local content = utils.read_file(path)
if not content then
return { dependencies = {}, dev_dependencies = {} }
end
local deps = {}
local in_require = false
for line in content:gmatch("[^\n]+") do
if line:match("^require%s*%(") then
in_require = true
elseif line:match("^%)") then
in_require = false
elseif in_require then
local module, version = line:match("^%s*([%w%.%-%_/]+)%s+([%w%.%-]+)")
if module then
deps[module] = version
end
else
local module, version = line:match("^require%s+([%w%.%-%_/]+)%s+([%w%.%-]+)")
if module then
deps[module] = version
end
end
end
return { dependencies = deps, dev_dependencies = {} }
end
--- Parse Python dependencies (pyproject.toml or requirements.txt)
---@param root string Project root path
---@return {dependencies: table, dev_dependencies: table}
function M.parse_python_deps(root)
local deps = {}
local dev_deps = {}
-- Try pyproject.toml first
local pyproject = root .. "/pyproject.toml"
local content = utils.read_file(pyproject)
if content then
-- Simple parsing for dependencies
local in_deps = false
local in_dev = false
for line in content:gmatch("[^\n]+") do
if line:match("^%[project%.dependencies%]") or line:match("^dependencies%s*=") then
in_deps = true
in_dev = false
elseif line:match("dev") and line:match("dependencies") then
in_deps = false
in_dev = true
elseif line:match("^%[") then
in_deps = false
in_dev = false
elseif in_deps or in_dev then
local name = line:match('"([%w_%-]+)')
if name then
if in_deps then
deps[name] = "latest"
else
dev_deps[name] = "latest"
end
end
end
end
end
-- Fallback to requirements.txt
local req_file = root .. "/requirements.txt"
content = utils.read_file(req_file)
if content then
for line in content:gmatch("[^\n]+") do
if not line:match("^#") and not line:match("^%s*$") then
local name, version = line:match("^([%w_%-]+)==([%d%.]+)")
if not name then
name = line:match("^([%w_%-]+)")
version = "latest"
end
if name then
deps[name] = version or "latest"
end
end
end
end
return { dependencies = deps, dev_dependencies = dev_deps }
end
--- Check if a file/directory should be ignored
---@param name string File or directory name
---@param config table Indexer configuration
---@return boolean
function M.should_ignore(name, config)
-- Check default patterns
for _, pattern in ipairs(DEFAULT_IGNORES) do
if name:match(pattern) then
return true
end
end
-- Check config excluded dirs
if config and config.excluded_dirs then
for _, dir in ipairs(config.excluded_dirs) do
if name == dir then
return true
end
end
end
return false
end
--- Check if a file should be indexed
---@param filepath string Full file path
---@param config table Indexer configuration
---@return boolean
function M.should_index(filepath, config)
local name = vim.fn.fnamemodify(filepath, ":t")
local ext = vim.fn.fnamemodify(filepath, ":e")
-- Check if it's a coder file
if utils.is_coder_file(filepath) then
return false
end
-- Check file size
if config and config.max_file_size then
local stat = vim.loop.fs_stat(filepath)
if stat and stat.size > config.max_file_size then
return false
end
end
-- Check extension
if config and config.index_extensions then
local valid_ext = false
for _, allowed_ext in ipairs(config.index_extensions) do
if ext == allowed_ext then
valid_ext = true
break
end
end
if not valid_ext then
return false
end
end
-- Check ignore patterns
if M.should_ignore(name, config) then
return false
end
return true
end
--- Get all indexable files in the project
---@param root string Project root path
---@param config table Indexer configuration
---@return string[] List of file paths
function M.get_indexable_files(root, config)
local files = {}
local function scan_dir(path)
local handle = vim.loop.fs_scandir(path)
if not handle then
return
end
while true do
local name, type = vim.loop.fs_scandir_next(handle)
if not name then
break
end
local full_path = path .. "/" .. name
if M.should_ignore(name, config) then
goto continue
end
if type == "directory" then
scan_dir(full_path)
elseif type == "file" then
if M.should_index(full_path, config) then
table.insert(files, full_path)
end
end
::continue::
end
end
scan_dir(root)
return files
end
--- Get language from file extension
---@param filepath string File path
---@return string Language name
function M.get_language(filepath)
local ext = vim.fn.fnamemodify(filepath, ":e")
return EXTENSION_LANGUAGE[ext] or ext
end
--- Read .gitignore patterns
---@param root string Project root
---@return string[] Patterns
function M.read_gitignore(root)
local patterns = {}
local path = root .. "/.gitignore"
local content = utils.read_file(path)
if not content then
return patterns
end
for line in content:gmatch("[^\n]+") do
-- Skip comments and empty lines
if not line:match("^#") and not line:match("^%s*$") then
-- Convert gitignore pattern to Lua pattern (simplified)
local pattern = line:gsub("^/", "^"):gsub("%*%*", ".*"):gsub("%*", "[^/]*"):gsub("%?", ".")
table.insert(patterns, pattern)
end
end
return patterns
end
return M