// ProcessingHelper.ts import fs from "node:fs" import path from "node:path" import { ScreenshotHelper } from "./ScreenshotHelper" import { IProcessingHelperDeps } from "./main" import * as axios from "axios" import { app, BrowserWindow } from "electron" import { OpenAI } from "openai" import { configHelper } from "./ConfigHelper" export class ProcessingHelper { private deps: IProcessingHelperDeps private screenshotHelper: ScreenshotHelper private openaiClient: OpenAI | null = null // AbortControllers for API requests private currentProcessingAbortController: AbortController | null = null private currentExtraProcessingAbortController: AbortController | null = null constructor(deps: IProcessingHelperDeps) { this.deps = deps this.screenshotHelper = deps.getScreenshotHelper() // Initialize OpenAI client this.initializeOpenAIClient(); // Listen for config changes to re-initialize the OpenAI client configHelper.on('config-updated', () => { this.initializeOpenAIClient(); }); } /** * Initialize or reinitialize the OpenAI client with current config */ private initializeOpenAIClient(): void { try { const config = configHelper.loadConfig(); if (config.apiKey) { this.openaiClient = new OpenAI({ apiKey: config.apiKey, timeout: 60000, // 60 second timeout maxRetries: 2 // Retry up to 2 times }); console.log("OpenAI client initialized successfully"); } else { this.openaiClient = null; console.warn("No API key available, OpenAI client not initialized"); } } catch (error) { console.error("Failed to initialize OpenAI client:", error); this.openaiClient = null; } } private async waitForInitialization( mainWindow: BrowserWindow ): Promise { let attempts = 0 const maxAttempts = 50 // 5 seconds total while (attempts < maxAttempts) { const isInitialized = await mainWindow.webContents.executeJavaScript( "window.__IS_INITIALIZED__" ) if (isInitialized) return await new Promise((resolve) => setTimeout(resolve, 100)) attempts++ } throw new Error("App failed to initialize after 5 seconds") } private async getCredits(): Promise { const mainWindow = this.deps.getMainWindow() if (!mainWindow) return 999 // Unlimited credits in this version try { await this.waitForInitialization(mainWindow) return 999 // Always return sufficient credits to work } catch (error) { console.error("Error getting credits:", error) return 999 // Unlimited credits as fallback } } private async getLanguage(): Promise { try { // Get language from config const config = configHelper.loadConfig(); if (config.language) { return config.language; } // Fallback to window variable if config doesn't have language const mainWindow = this.deps.getMainWindow() if (mainWindow) { try { await this.waitForInitialization(mainWindow) const language = await mainWindow.webContents.executeJavaScript( "window.__LANGUAGE__" ) if ( typeof language === "string" && language !== undefined && language !== null ) { return language; } } catch (err) { console.warn("Could not get language from window", err); } } // Default fallback return "python"; } catch (error) { console.error("Error getting language:", error) return "python" } } public async processScreenshots(): Promise { const mainWindow = this.deps.getMainWindow() if (!mainWindow) return // First verify we have a valid OpenAI client if (!this.openaiClient) { this.initializeOpenAIClient(); if (!this.openaiClient) { console.error("OpenAI client not initialized"); mainWindow.webContents.send( this.deps.PROCESSING_EVENTS.API_KEY_INVALID ); return; } } const view = this.deps.getView() console.log("Processing screenshots in view:", view) if (view === "queue") { mainWindow.webContents.send(this.deps.PROCESSING_EVENTS.INITIAL_START) const screenshotQueue = this.screenshotHelper.getScreenshotQueue() console.log("Processing main queue screenshots:", screenshotQueue) if (screenshotQueue.length === 0) { mainWindow.webContents.send(this.deps.PROCESSING_EVENTS.NO_SCREENSHOTS) return } try { // Initialize AbortController this.currentProcessingAbortController = new AbortController() const { signal } = this.currentProcessingAbortController const screenshots = await Promise.all( screenshotQueue.map(async (path) => ({ path, preview: await this.screenshotHelper.getImagePreview(path), data: fs.readFileSync(path).toString('base64') })) ) const result = await this.processScreenshotsHelper(screenshots, signal) if (!result.success) { console.log("Processing failed:", result.error) if (result.error?.includes("API Key") || result.error?.includes("OpenAI")) { mainWindow.webContents.send( this.deps.PROCESSING_EVENTS.API_KEY_INVALID ) } else { mainWindow.webContents.send( this.deps.PROCESSING_EVENTS.INITIAL_SOLUTION_ERROR, result.error ) } // Reset view back to queue on error console.log("Resetting view to queue due to error") this.deps.setView("queue") return } // Only set view to solutions if processing succeeded console.log("Setting view to solutions after successful processing") mainWindow.webContents.send( this.deps.PROCESSING_EVENTS.SOLUTION_SUCCESS, result.data ) this.deps.setView("solutions") } catch (error: any) { mainWindow.webContents.send( this.deps.PROCESSING_EVENTS.INITIAL_SOLUTION_ERROR, error ) console.error("Processing error:", error) if (axios.isCancel(error)) { mainWindow.webContents.send( this.deps.PROCESSING_EVENTS.INITIAL_SOLUTION_ERROR, "Processing was canceled by the user." ) } else { mainWindow.webContents.send( this.deps.PROCESSING_EVENTS.INITIAL_SOLUTION_ERROR, error.message || "Server error. Please try again." ) } // Reset view back to queue on error console.log("Resetting view to queue due to error") this.deps.setView("queue") } finally { this.currentProcessingAbortController = null } } else { // view == 'solutions' const extraScreenshotQueue = this.screenshotHelper.getExtraScreenshotQueue() console.log("Processing extra queue screenshots:", extraScreenshotQueue) if (extraScreenshotQueue.length === 0) { mainWindow.webContents.send(this.deps.PROCESSING_EVENTS.NO_SCREENSHOTS) return } mainWindow.webContents.send(this.deps.PROCESSING_EVENTS.DEBUG_START) // Initialize AbortController this.currentExtraProcessingAbortController = new AbortController() const { signal } = this.currentExtraProcessingAbortController try { const screenshots = await Promise.all( [ ...this.screenshotHelper.getScreenshotQueue(), ...extraScreenshotQueue ].map(async (path) => ({ path, preview: await this.screenshotHelper.getImagePreview(path), data: fs.readFileSync(path).toString('base64') })) ) console.log( "Combined screenshots for processing:", screenshots.map((s) => s.path) ) const result = await this.processExtraScreenshotsHelper( screenshots, signal ) if (result.success) { this.deps.setHasDebugged(true) mainWindow.webContents.send( this.deps.PROCESSING_EVENTS.DEBUG_SUCCESS, result.data ) } else { mainWindow.webContents.send( this.deps.PROCESSING_EVENTS.DEBUG_ERROR, result.error ) } } catch (error: any) { if (axios.isCancel(error)) { mainWindow.webContents.send( this.deps.PROCESSING_EVENTS.DEBUG_ERROR, "Extra processing was canceled by the user." ) } else { mainWindow.webContents.send( this.deps.PROCESSING_EVENTS.DEBUG_ERROR, error.message ) } } finally { this.currentExtraProcessingAbortController = null } } } private async processScreenshotsHelper( screenshots: Array<{ path: string; data: string }>, signal: AbortSignal ) { try { const config = configHelper.loadConfig(); const language = await this.getLanguage(); const mainWindow = this.deps.getMainWindow(); // Verify OpenAI client if (!this.openaiClient) { this.initializeOpenAIClient(); // Try to reinitialize if (!this.openaiClient) { return { success: false, error: "OpenAI API key not configured or invalid. Please check your settings." }; } } // Step 1: Extract problem info using OpenAI Vision API const imageDataList = screenshots.map(screenshot => screenshot.data); // Update the user on progress if (mainWindow) { mainWindow.webContents.send("processing-status", { message: "Analyzing problem from screenshots...", progress: 20 }); } const messages = [ { role: "system" as const, content: "You are a coding challenge interpreter. Analyze the screenshot of the coding problem and extract all relevant information. Return the information in JSON format with these fields: problem_statement, constraints, example_input, example_output. Just return the structured JSON without any other text." }, { role: "user" as const, content: [ { type: "text" as const, text: `Extract the coding problem details from these screenshots. Return in JSON format. Preferred coding language we gonna use for this problem is ${language}.` }, ...imageDataList.map(data => ({ type: "image_url" as const, image_url: { url: `data:image/png;base64,${data}` } })) ] } ]; // Send to OpenAI Vision API const extractionResponse = await this.openaiClient.chat.completions.create({ model: config.extractionModel || "gpt-4o", messages: messages, max_tokens: 4000, temperature: 0.2 }); // Parse the response to get structured problem info let problemInfo; try { const responseText = extractionResponse.choices[0].message.content; // Handle when OpenAI might wrap the JSON in markdown code blocks const jsonText = responseText.replace(/```json|```/g, '').trim(); problemInfo = JSON.parse(jsonText); // Update the user on progress if (mainWindow) { mainWindow.webContents.send("processing-status", { message: "Problem analyzed successfully. Preparing to generate solution...", progress: 40 }); } } catch (error) { console.error("Error parsing problem extraction response:", error); console.log("Raw response:", extractionResponse.choices[0].message.content); return { success: false, error: "Failed to parse problem information. Please try again or use clearer screenshots." }; } // Store problem info in AppState this.deps.setProblemInfo(problemInfo); // Send first success event if (mainWindow) { mainWindow.webContents.send( this.deps.PROCESSING_EVENTS.PROBLEM_EXTRACTED, problemInfo ); // Generate solutions after successful extraction const solutionsResult = await this.generateSolutionsHelper(signal); if (solutionsResult.success) { // Clear any existing extra screenshots before transitioning to solutions view this.screenshotHelper.clearExtraScreenshotQueue(); // Final progress update mainWindow.webContents.send("processing-status", { message: "Solution generated successfully", progress: 100 }); mainWindow.webContents.send( this.deps.PROCESSING_EVENTS.SOLUTION_SUCCESS, solutionsResult.data ); return { success: true, data: solutionsResult.data }; } else { throw new Error( solutionsResult.error || "Failed to generate solutions" ); } } return { success: false, error: "Failed to process screenshots" }; } catch (error: any) { // If the request was cancelled, don't retry if (axios.isCancel(error)) { return { success: false, error: "Processing was canceled by the user." }; } // Handle OpenAI API errors specifically if (error?.response?.status === 401) { return { success: false, error: "Invalid OpenAI API key. Please check your settings." }; } else if (error?.response?.status === 429) { return { success: false, error: "OpenAI API rate limit exceeded or insufficient credits. Please try again later." }; } else if (error?.response?.status === 500) { return { success: false, error: "OpenAI server error. Please try again later." }; } console.error("API Error Details:", error); return { success: false, error: error.message || "Failed to process screenshots. Please try again." }; } } private async generateSolutionsHelper(signal: AbortSignal) { try { const problemInfo = this.deps.getProblemInfo(); const language = await this.getLanguage(); const config = configHelper.loadConfig(); const mainWindow = this.deps.getMainWindow(); if (!problemInfo) { throw new Error("No problem info available"); } if (!this.openaiClient) { return { success: false, error: "OpenAI API key not configured. Please check your settings." }; } // Update progress status if (mainWindow) { mainWindow.webContents.send("processing-status", { message: "Creating optimal solution with detailed explanations...", progress: 60 }); } // Create prompt for solution generation const promptText = ` Generate a detailed solution for the following coding problem: PROBLEM STATEMENT: ${problemInfo.problem_statement} CONSTRAINTS: ${problemInfo.constraints || "No specific constraints provided."} EXAMPLE INPUT: ${problemInfo.example_input || "No example input provided."} EXAMPLE OUTPUT: ${problemInfo.example_output || "No example output provided."} LANGUAGE: ${language} I need the response in the following format: 1. Code: A clean, optimized implementation in ${language} 2. Your Thoughts: A list of key insights and reasoning behind your approach 3. Time complexity: O(X) with a detailed explanation (at least 2 sentences) 4. Space complexity: O(X) with a detailed explanation (at least 2 sentences) For complexity explanations, please be thorough. For example: "Time complexity: O(n) because we iterate through the array only once. This is optimal as we need to examine each element at least once to find the solution." or "Space complexity: O(n) because in the worst case, we store all elements in the hashmap. The additional space scales linearly with the input size." Your solution should be efficient, well-commented, and handle edge cases. `; // Send to OpenAI API const solutionResponse = await this.openaiClient.chat.completions.create({ model: config.solutionModel || "gpt-4o", // Using selected model for code generation messages: [ { role: "system", content: "You are an expert coding interview assistant. Provide clear, optimal solutions with detailed explanations." }, { role: "user", content: promptText } ], max_tokens: 4000, temperature: 0.2 }); const responseContent = solutionResponse.choices[0].message.content; // Extract parts from the response const codeMatch = responseContent.match(/```(?:\w+)?\s*([\s\S]*?)```/); const code = codeMatch ? codeMatch[1].trim() : responseContent; // Extract thoughts, looking for bullet points or numbered lists const thoughtsRegex = /(?:Thoughts:|Key Insights:|Reasoning:|Approach:)([\s\S]*?)(?:Time complexity:|$)/i; const thoughtsMatch = responseContent.match(thoughtsRegex); let thoughts: string[] = []; if (thoughtsMatch && thoughtsMatch[1]) { // Extract bullet points or numbered items const bulletPoints = thoughtsMatch[1].match(/(?:^|\n)\s*(?:[-*•]|\d+\.)\s*(.*)/g); if (bulletPoints) { thoughts = bulletPoints.map(point => point.replace(/^\s*(?:[-*•]|\d+\.)\s*/, '').trim() ).filter(Boolean); } else { // If no bullet points found, split by newlines and filter empty lines thoughts = thoughtsMatch[1].split('\n') .map(line => line.trim()) .filter(Boolean); } } // Extract complexity information // Use more flexible patterns to find complexity sections const timeComplexityPattern = /Time complexity:?\s*([^\n]+(?:\n[^\n]+)*?)(?=\n\s*(?:Space complexity|$))/i; const spaceComplexityPattern = /Space complexity:?\s*([^\n]+(?:\n[^\n]+)*?)(?=\n\s*(?:[A-Z]|$))/i; let timeComplexity = "O(n) - Linear time complexity because we only iterate through the array once. Each element is processed exactly one time, and the hashmap lookups are O(1) operations."; let spaceComplexity = "O(n) - Linear space complexity because we store elements in the hashmap. In the worst case, we might need to store all elements before finding the solution pair."; // Extract time complexity with better matching const timeMatch = responseContent.match(timeComplexityPattern); if (timeMatch && timeMatch[1]) { timeComplexity = timeMatch[1].trim(); // Ensure the response includes actual Big O notation and a dash if (!timeComplexity.match(/O\([^)]+\)/i)) { timeComplexity = `O(n) - ${timeComplexity}`; } else if (!timeComplexity.includes('-') && !timeComplexity.includes('because')) { // Add a dash if there isn't one and no 'because' const notationMatch = timeComplexity.match(/O\([^)]+\)/i); if (notationMatch) { const notation = notationMatch[0]; const rest = timeComplexity.replace(notation, '').trim(); timeComplexity = `${notation} - ${rest}`; } } } // Extract space complexity with better matching const spaceMatch = responseContent.match(spaceComplexityPattern); if (spaceMatch && spaceMatch[1]) { spaceComplexity = spaceMatch[1].trim(); // Ensure the response includes actual Big O notation and a dash if (!spaceComplexity.match(/O\([^)]+\)/i)) { spaceComplexity = `O(n) - ${spaceComplexity}`; } else if (!spaceComplexity.includes('-') && !spaceComplexity.includes('because')) { // Add a dash if there isn't one and no 'because' const notationMatch = spaceComplexity.match(/O\([^)]+\)/i); if (notationMatch) { const notation = notationMatch[0]; const rest = spaceComplexity.replace(notation, '').trim(); spaceComplexity = `${notation} - ${rest}`; } } } // Construct the formatted response const formattedResponse = { code: code, thoughts: thoughts.length > 0 ? thoughts : ["Solution approach based on efficiency and readability"], time_complexity: timeComplexity, space_complexity: spaceComplexity }; return { success: true, data: formattedResponse }; } catch (error: any) { // Handle OpenAI API errors specifically if (error?.response?.status === 401) { return { success: false, error: "Invalid OpenAI API key. Please check your settings." }; } else if (error?.response?.status === 429) { return { success: false, error: "OpenAI API rate limit exceeded or insufficient credits. Please try again later." }; } console.error("Solution generation error:", error); return { success: false, error: error.message || "Failed to generate solution" }; } } private async processExtraScreenshotsHelper( screenshots: Array<{ path: string; data: string }>, signal: AbortSignal ) { try { const problemInfo = this.deps.getProblemInfo(); const language = await this.getLanguage(); const config = configHelper.loadConfig(); const mainWindow = this.deps.getMainWindow(); if (!problemInfo) { throw new Error("No problem info available"); } if (!this.openaiClient) { return { success: false, error: "OpenAI API key not configured. Please check your settings." }; } // Update progress status if (mainWindow) { mainWindow.webContents.send("processing-status", { message: "Processing debug screenshots...", progress: 30 }); } // Prepare the images for the API call const imageDataList = screenshots.map(screenshot => screenshot.data); const messages = [ { role: "system" as const, content: `You are a coding interview assistant helping debug and improve solutions. Analyze these screenshots which include either error messages, incorrect outputs, or test cases, and provide detailed debugging help. Your response MUST follow this exact structure with these section headers (use ### for headers): ### Issues Identified - List each issue as a bullet point with clear explanation ### Specific Improvements and Corrections - List specific code changes needed as bullet points ### Optimizations - List any performance optimizations if applicable ### Explanation of Changes Needed Here provide a clear explanation of why the changes are needed ### Key Points - Summary bullet points of the most important takeaways If you include code examples, use proper markdown code blocks with language specification (e.g. \`\`\`java).` }, { role: "user" as const, content: [ { type: "text" as const, text: `I'm solving this coding problem: "${problemInfo.problem_statement}" in ${language}. I need help with debugging or improving my solution. Here are screenshots of my code, the errors or test cases. Please provide a detailed analysis with: 1. What issues you found in my code 2. Specific improvements and corrections 3. Any optimizations that would make the solution better 4. A clear explanation of the changes needed` }, ...imageDataList.map(data => ({ type: "image_url" as const, image_url: { url: `data:image/png;base64,${data}` } })) ] } ]; // Update progress if (mainWindow) { mainWindow.webContents.send("processing-status", { message: "Analyzing code and generating debug feedback...", progress: 60 }); } // Send to OpenAI Vision API const debugResponse = await this.openaiClient.chat.completions.create({ model: config.debuggingModel || "gpt-4o", messages: messages, max_tokens: 4000, temperature: 0.2 }); // Update final progress if (mainWindow) { mainWindow.webContents.send("processing-status", { message: "Debug analysis complete", progress: 100 }); } // Extract and format the debug response const debugContent = debugResponse.choices[0].message.content; // Extract code if there's code block in the response let extractedCode = "// Debug mode - see analysis below"; const codeMatch = debugContent.match(/```(?:[a-zA-Z]+)?([\s\S]*?)```/); if (codeMatch && codeMatch[1]) { extractedCode = codeMatch[1].trim(); } // Format the debug content for better display - add clear section headings let formattedDebugContent = debugContent; // Ensure proper section formatting if (!debugContent.includes('# ') && !debugContent.includes('## ')) { formattedDebugContent = debugContent .replace(/issues identified|problems found|bugs found/i, '## Issues Identified') .replace(/code improvements|improvements|suggested changes/i, '## Code Improvements') .replace(/optimizations|performance improvements/i, '## Optimizations') .replace(/explanation|detailed analysis/i, '## Explanation'); } // Try to extract bullet points for "thoughts" const bulletPoints = formattedDebugContent.match(/(?:^|\n)[ ]*(?:[-*•]|\d+\.)[ ]+([^\n]+)/g); const thoughts = bulletPoints ? bulletPoints.map(point => point.replace(/^[ ]*(?:[-*•]|\d+\.)[ ]+/, '').trim()).slice(0, 5) : ["Debug analysis based on your screenshots"]; // Return the debug assistance in the format expected by the app const response = { code: extractedCode, debug_analysis: formattedDebugContent, thoughts: thoughts, time_complexity: "N/A - Debug mode", space_complexity: "N/A - Debug mode" }; return { success: true, data: response }; } catch (error: any) { // Handle OpenAI API errors specifically if (error?.response?.status === 401) { return { success: false, error: "Invalid OpenAI API key. Please check your settings." }; } else if (error?.response?.status === 429) { return { success: false, error: "OpenAI API rate limit exceeded or insufficient credits. Please try again later." }; } console.error("Debug processing error:", error); return { success: false, error: error.message || "Failed to process debug request" }; } } public cancelOngoingRequests(): void { let wasCancelled = false if (this.currentProcessingAbortController) { this.currentProcessingAbortController.abort() this.currentProcessingAbortController = null wasCancelled = true } if (this.currentExtraProcessingAbortController) { this.currentExtraProcessingAbortController.abort() this.currentExtraProcessingAbortController = null wasCancelled = true } // Reset hasDebugged flag this.deps.setHasDebugged(false) // Clear any pending state this.deps.setProblemInfo(null) const mainWindow = this.deps.getMainWindow() if (wasCancelled && mainWindow && !mainWindow.isDestroyed()) { // Send a clear message that processing was cancelled mainWindow.webContents.send(this.deps.PROCESSING_EVENTS.NO_SCREENSHOTS) } } }