fix: fixing the multi files
This commit is contained in:
Carlos
@ -3,6 +3,31 @@ const fs = require("fs");
|
||||
const path = require("path");
|
||||
const { savePrompt, saveError } = require("../controller/generate");
|
||||
|
||||
// Constants for file handling
|
||||
const MAX_FILE_SIZE = 30000; // ~30KB per file
|
||||
const MAX_TOTAL_FILES = 5; // Maximum number of files to inject
|
||||
const MAX_TOTAL_SIZE = 100000; // ~100KB total for all files
|
||||
|
||||
/**
|
||||
* Saves an error with consistent formatting
|
||||
* @param {Error} error - The error to save
|
||||
* @param {Object} context - Additional context about where the error occurred
|
||||
*/
|
||||
async function logError(error, context = {}) {
|
||||
const errorDetails = {
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
...context
|
||||
};
|
||||
|
||||
await saveError({
|
||||
error_message: error.message,
|
||||
details: errorDetails
|
||||
});
|
||||
|
||||
console.warn(`⚠️ ${context.operation || 'Operation'} failed:`, error.message);
|
||||
}
|
||||
|
||||
/**
|
||||
* Injects file contents into message content when file tags are present
|
||||
* @param {string} content - The message content to process
|
||||
@ -10,90 +35,124 @@ const { savePrompt, saveError } = require("../controller/generate");
|
||||
*/
|
||||
async function injectFileContent(content) {
|
||||
let processedContent = content;
|
||||
let totalSize = 0;
|
||||
let fileCount = 0;
|
||||
|
||||
// First, handle <selected_files> block
|
||||
const selectedFilesMatch = processedContent.match(
|
||||
/<selected_files>([\s\S]*?)<\/selected_files>/,
|
||||
);
|
||||
if (selectedFilesMatch) {
|
||||
const fileListBlock = selectedFilesMatch[1];
|
||||
|
||||
// Skip injection if it already contains <file> tags
|
||||
const containsFileTags = fileListBlock.includes("<file path=");
|
||||
if (!containsFileTags) {
|
||||
const filePaths = fileListBlock
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line && line.startsWith("- "))
|
||||
.map((line) => line.replace(/^- /, ""));
|
||||
|
||||
const fileBlocks = await Promise.all(
|
||||
filePaths.map(async (filePath) => {
|
||||
try {
|
||||
const resolvedPath = path.resolve(process.cwd(), filePath);
|
||||
const fileContent = fs.readFileSync(resolvedPath, "utf-8");
|
||||
const ext = path.extname(filePath).slice(1) || "text";
|
||||
return `<file path="${filePath}" language="${ext}">\n${fileContent}\n</file>`;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`⚠️ Failed to read file: ${filePath} → ${error.message}`,
|
||||
);
|
||||
await saveError({
|
||||
error_message: error.message,
|
||||
details: {
|
||||
message: error.message,
|
||||
request: null,
|
||||
response: error.response?.data,
|
||||
stack: error.stack,
|
||||
},
|
||||
});
|
||||
return `<file path="${filePath}" language="text">\n[Error: File not found or unreadable]\n</file>`;
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
// Replace the <selected_files> block with the injected file tags
|
||||
processedContent = processedContent.replace(
|
||||
selectedFilesMatch[0],
|
||||
fileBlocks.join("\n\n"),
|
||||
);
|
||||
} else {
|
||||
console.log(
|
||||
"ℹ️ Skipping injection: <selected_files> already includes <file> tags.",
|
||||
);
|
||||
// Helper function to check file size limits
|
||||
const checkFileLimits = (fileSize) => {
|
||||
if (fileCount >= MAX_TOTAL_FILES) {
|
||||
throw new Error(`Maximum number of files (${MAX_TOTAL_FILES}) exceeded`);
|
||||
}
|
||||
}
|
||||
if (totalSize + fileSize > MAX_TOTAL_SIZE) {
|
||||
throw new Error(`Total file size limit (${MAX_TOTAL_SIZE} bytes) exceeded`);
|
||||
}
|
||||
totalSize += fileSize;
|
||||
fileCount++;
|
||||
};
|
||||
|
||||
// Inject content into empty <file> tags
|
||||
const fileTagRegex = /<file path="(.+?)"(?: language="(.+?)")?>\s*<\/file>/g;
|
||||
const matches = [...processedContent.matchAll(fileTagRegex)];
|
||||
|
||||
for (const match of matches) {
|
||||
const [fullTag, filePath, lang] = match;
|
||||
// Helper function to read and process a single file
|
||||
const processFile = async (filePath, lang = null) => {
|
||||
try {
|
||||
const resolvedPath = path.resolve(process.cwd(), filePath);
|
||||
const fileContent = fs.readFileSync(resolvedPath, "utf-8");
|
||||
const replacement = `<file path="${filePath}" language="${lang || "text"}">\n${fileContent}\n</file>`;
|
||||
processedContent = processedContent.replace(fullTag, replacement);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`⚠️ Failed to inject <file> tag for ${filePath}: ${error.message}`,
|
||||
);
|
||||
await saveError({
|
||||
error_message: error.message,
|
||||
details: {
|
||||
message: error.message,
|
||||
request: null,
|
||||
response: error.response?.data,
|
||||
stack: error.stack,
|
||||
},
|
||||
});
|
||||
const replacement = `<file path="${filePath}" language="${lang || "text"}">\n[Error: File not found or unreadable]\n</file>`;
|
||||
processedContent = processedContent.replace(fullTag, replacement);
|
||||
}
|
||||
}
|
||||
const stats = fs.statSync(resolvedPath);
|
||||
|
||||
// Check file size before reading
|
||||
if (stats.size > MAX_FILE_SIZE) {
|
||||
const error = new Error(`File too large (${stats.size} bytes) - maximum size is ${MAX_FILE_SIZE} bytes`);
|
||||
await logError(error, {
|
||||
operation: 'File size check',
|
||||
filePath,
|
||||
fileSize: stats.size,
|
||||
maxSize: MAX_FILE_SIZE
|
||||
});
|
||||
return `<file path="${filePath}" language="${lang || 'text'}">\n[File too large (${stats.size} bytes) - maximum size is ${MAX_FILE_SIZE} bytes]\n</file>`;
|
||||
}
|
||||
|
||||
return processedContent;
|
||||
const fileContent = fs.readFileSync(resolvedPath, "utf-8");
|
||||
checkFileLimits(fileContent.length);
|
||||
|
||||
const ext = lang || path.extname(filePath).slice(1) || "text";
|
||||
return `<file path="${filePath}" language="${ext}">\n${fileContent}\n</file>`;
|
||||
} catch (error) {
|
||||
if (error.message.includes('Maximum number of files') ||
|
||||
error.message.includes('Total file size limit')) {
|
||||
throw error;
|
||||
}
|
||||
await logError(error, {
|
||||
operation: 'File processing',
|
||||
filePath,
|
||||
language: lang
|
||||
});
|
||||
return `<file path="${filePath}" language="${lang || 'text'}">\n[Error: File not found or unreadable]\n</file>`;
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
// First, handle <selected_files> block
|
||||
const selectedFilesMatch = processedContent.match(
|
||||
/<selected_files>([\s\S]*?)<\/selected_files>/,
|
||||
);
|
||||
if (selectedFilesMatch) {
|
||||
const fileListBlock = selectedFilesMatch[1];
|
||||
|
||||
// Skip injection if it already contains <file> tags
|
||||
const containsFileTags = fileListBlock.includes("<file path=");
|
||||
if (!containsFileTags) {
|
||||
const filePaths = fileListBlock
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line && line.startsWith("- "))
|
||||
.map((line) => line.replace(/^- /, ""));
|
||||
|
||||
const fileBlocks = await Promise.all(
|
||||
filePaths.map(async (filePath) => await processFile(filePath))
|
||||
);
|
||||
|
||||
// Replace the <selected_files> block with the injected file tags
|
||||
// Add extra newlines for better model parsing
|
||||
processedContent = processedContent.replace(
|
||||
selectedFilesMatch[0],
|
||||
"\n\n" + fileBlocks.join("\n\n") + "\n\n"
|
||||
);
|
||||
} else {
|
||||
console.log(
|
||||
"ℹ️ Skipping injection: <selected_files> already includes <file> tags.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Inject content into empty <file> tags
|
||||
const fileTagRegex = /<file path="(.+?)"(?: language="(.+?)")?>\s*<\/file>/g;
|
||||
const matches = [...processedContent.matchAll(fileTagRegex)];
|
||||
|
||||
for (const match of matches) {
|
||||
const [fullTag, filePath, lang] = match;
|
||||
const fileBlock = await processFile(filePath, lang);
|
||||
// Add extra newlines for better model parsing
|
||||
processedContent = processedContent.replace(fullTag, "\n" + fileBlock + "\n");
|
||||
}
|
||||
|
||||
return processedContent;
|
||||
} catch (error) {
|
||||
if (error.message.includes('Maximum number of files') ||
|
||||
error.message.includes('Total file size limit')) {
|
||||
await logError(error, {
|
||||
operation: 'File limit check',
|
||||
totalSize,
|
||||
fileCount,
|
||||
maxFiles: MAX_TOTAL_FILES,
|
||||
maxSize: MAX_TOTAL_SIZE
|
||||
});
|
||||
// Add a note about the limit to the content
|
||||
return processedContent + `\n\n[Note: ${error.message} - some files were not included]`;
|
||||
}
|
||||
// Log any other unexpected errors
|
||||
await logError(error, {
|
||||
operation: 'File content injection',
|
||||
contentLength: content.length
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user