fix: fixing the multi files

This commit is contained in:
Carlos
2025-05-24 15:49:11 -04:00
parent 34b5a54daf
commit 9a78ec0c7c

View File

@ -3,6 +3,31 @@ const fs = require("fs");
const path = require("path");
const { savePrompt, saveError } = require("../controller/generate");
// Constants for file handling
const MAX_FILE_SIZE = 30000; // ~30KB per file
const MAX_TOTAL_FILES = 5; // Maximum number of files to inject
const MAX_TOTAL_SIZE = 100000; // ~100KB total for all files
/**
* Saves an error with consistent formatting
* @param {Error} error - The error to save
* @param {Object} context - Additional context about where the error occurred
*/
async function logError(error, context = {}) {
const errorDetails = {
message: error.message,
stack: error.stack,
...context
};
await saveError({
error_message: error.message,
details: errorDetails
});
console.warn(`⚠️ ${context.operation || 'Operation'} failed:`, error.message);
}
/**
* Injects file contents into message content when file tags are present
* @param {string} content - The message content to process
@ -10,90 +35,124 @@ const { savePrompt, saveError } = require("../controller/generate");
*/
async function injectFileContent(content) {
let processedContent = content;
let totalSize = 0;
let fileCount = 0;
// First, handle <selected_files> block
const selectedFilesMatch = processedContent.match(
/<selected_files>([\s\S]*?)<\/selected_files>/,
);
if (selectedFilesMatch) {
const fileListBlock = selectedFilesMatch[1];
// Skip injection if it already contains <file> tags
const containsFileTags = fileListBlock.includes("<file path=");
if (!containsFileTags) {
const filePaths = fileListBlock
.split("\n")
.map((line) => line.trim())
.filter((line) => line && line.startsWith("- "))
.map((line) => line.replace(/^- /, ""));
const fileBlocks = await Promise.all(
filePaths.map(async (filePath) => {
try {
const resolvedPath = path.resolve(process.cwd(), filePath);
const fileContent = fs.readFileSync(resolvedPath, "utf-8");
const ext = path.extname(filePath).slice(1) || "text";
return `<file path="${filePath}" language="${ext}">\n${fileContent}\n</file>`;
} catch (error) {
console.warn(
`⚠️ Failed to read file: ${filePath}${error.message}`,
);
await saveError({
error_message: error.message,
details: {
message: error.message,
request: null,
response: error.response?.data,
stack: error.stack,
},
});
return `<file path="${filePath}" language="text">\n[Error: File not found or unreadable]\n</file>`;
}
}),
);
// Replace the <selected_files> block with the injected file tags
processedContent = processedContent.replace(
selectedFilesMatch[0],
fileBlocks.join("\n\n"),
);
} else {
console.log(
" Skipping injection: <selected_files> already includes <file> tags.",
);
// Helper function to check file size limits
const checkFileLimits = (fileSize) => {
if (fileCount >= MAX_TOTAL_FILES) {
throw new Error(`Maximum number of files (${MAX_TOTAL_FILES}) exceeded`);
}
}
if (totalSize + fileSize > MAX_TOTAL_SIZE) {
throw new Error(`Total file size limit (${MAX_TOTAL_SIZE} bytes) exceeded`);
}
totalSize += fileSize;
fileCount++;
};
// Inject content into empty <file> tags
const fileTagRegex = /<file path="(.+?)"(?: language="(.+?)")?>\s*<\/file>/g;
const matches = [...processedContent.matchAll(fileTagRegex)];
for (const match of matches) {
const [fullTag, filePath, lang] = match;
// Helper function to read and process a single file
const processFile = async (filePath, lang = null) => {
try {
const resolvedPath = path.resolve(process.cwd(), filePath);
const fileContent = fs.readFileSync(resolvedPath, "utf-8");
const replacement = `<file path="${filePath}" language="${lang || "text"}">\n${fileContent}\n</file>`;
processedContent = processedContent.replace(fullTag, replacement);
} catch (error) {
console.warn(
`⚠️ Failed to inject <file> tag for ${filePath}: ${error.message}`,
);
await saveError({
error_message: error.message,
details: {
message: error.message,
request: null,
response: error.response?.data,
stack: error.stack,
},
});
const replacement = `<file path="${filePath}" language="${lang || "text"}">\n[Error: File not found or unreadable]\n</file>`;
processedContent = processedContent.replace(fullTag, replacement);
}
}
const stats = fs.statSync(resolvedPath);
// Check file size before reading
if (stats.size > MAX_FILE_SIZE) {
const error = new Error(`File too large (${stats.size} bytes) - maximum size is ${MAX_FILE_SIZE} bytes`);
await logError(error, {
operation: 'File size check',
filePath,
fileSize: stats.size,
maxSize: MAX_FILE_SIZE
});
return `<file path="${filePath}" language="${lang || 'text'}">\n[File too large (${stats.size} bytes) - maximum size is ${MAX_FILE_SIZE} bytes]\n</file>`;
}
return processedContent;
const fileContent = fs.readFileSync(resolvedPath, "utf-8");
checkFileLimits(fileContent.length);
const ext = lang || path.extname(filePath).slice(1) || "text";
return `<file path="${filePath}" language="${ext}">\n${fileContent}\n</file>`;
} catch (error) {
if (error.message.includes('Maximum number of files') ||
error.message.includes('Total file size limit')) {
throw error;
}
await logError(error, {
operation: 'File processing',
filePath,
language: lang
});
return `<file path="${filePath}" language="${lang || 'text'}">\n[Error: File not found or unreadable]\n</file>`;
}
};
try {
// First, handle <selected_files> block
const selectedFilesMatch = processedContent.match(
/<selected_files>([\s\S]*?)<\/selected_files>/,
);
if (selectedFilesMatch) {
const fileListBlock = selectedFilesMatch[1];
// Skip injection if it already contains <file> tags
const containsFileTags = fileListBlock.includes("<file path=");
if (!containsFileTags) {
const filePaths = fileListBlock
.split("\n")
.map((line) => line.trim())
.filter((line) => line && line.startsWith("- "))
.map((line) => line.replace(/^- /, ""));
const fileBlocks = await Promise.all(
filePaths.map(async (filePath) => await processFile(filePath))
);
// Replace the <selected_files> block with the injected file tags
// Add extra newlines for better model parsing
processedContent = processedContent.replace(
selectedFilesMatch[0],
"\n\n" + fileBlocks.join("\n\n") + "\n\n"
);
} else {
console.log(
" Skipping injection: <selected_files> already includes <file> tags.",
);
}
}
// Inject content into empty <file> tags
const fileTagRegex = /<file path="(.+?)"(?: language="(.+?)")?>\s*<\/file>/g;
const matches = [...processedContent.matchAll(fileTagRegex)];
for (const match of matches) {
const [fullTag, filePath, lang] = match;
const fileBlock = await processFile(filePath, lang);
// Add extra newlines for better model parsing
processedContent = processedContent.replace(fullTag, "\n" + fileBlock + "\n");
}
return processedContent;
} catch (error) {
if (error.message.includes('Maximum number of files') ||
error.message.includes('Total file size limit')) {
await logError(error, {
operation: 'File limit check',
totalSize,
fileCount,
maxFiles: MAX_TOTAL_FILES,
maxSize: MAX_TOTAL_SIZE
});
// Add a note about the limit to the content
return processedContent + `\n\n[Note: ${error.message} - some files were not included]`;
}
// Log any other unexpected errors
await logError(error, {
operation: 'File content injection',
contentLength: content.length
});
throw error;
}
}
/**