Various improvements & Epub support

frontend/src/app/scrape/bulk/progress/route.ts

@@ -0,0 +1,93 @@
+import { NextRequest } from 'next/server';
+
+// Configure route timeout for long-running progress streams
+export const maxDuration = 900; // 15 minutes (900 seconds)
+
+interface ProgressUpdate {
+  type: 'progress' | 'completed' | 'error';
+  current: number;
+  total: number;
+  message: string;
+  url?: string;
+  title?: string;
+  author?: string;
+  wordCount?: number;
+  totalWordCount?: number;
+  error?: string;
+  combinedStory?: any;
+  results?: any[];
+  summary?: any;
+}
+
+// Global progress storage (in production, use Redis or database)
+const progressStore = new Map<string, ProgressUpdate[]>();
+
+export async function GET(request: NextRequest) {
+  const searchParams = request.nextUrl.searchParams;
+  const sessionId = searchParams.get('sessionId');
+  
+  if (!sessionId) {
+    return new Response('Session ID required', { status: 400 });
+  }
+
+  // Set up Server-Sent Events
+  const stream = new ReadableStream({
+    start(controller) {
+      const encoder = new TextEncoder();
+      
+      // Send initial connection message
+      const data = `data: ${JSON.stringify({ type: 'connected', sessionId })}\n\n`;
+      controller.enqueue(encoder.encode(data));
+
+      // Check for progress updates every 500ms
+      const interval = setInterval(() => {
+        const updates = progressStore.get(sessionId);
+        if (updates && updates.length > 0) {
+          // Send all pending updates
+          updates.forEach(update => {
+            const data = `data: ${JSON.stringify(update)}\n\n`;
+            controller.enqueue(encoder.encode(data));
+          });
+          
+          // Clear sent updates
+          progressStore.delete(sessionId);
+          
+          // If this was a completion or error, close the stream
+          const lastUpdate = updates[updates.length - 1];
+          if (lastUpdate.type === 'completed' || lastUpdate.type === 'error') {
+            clearInterval(interval);
+            controller.close();
+          }
+        }
+      }, 500);
+
+      // Cleanup after timeout
+      setTimeout(() => {
+        clearInterval(interval);
+        progressStore.delete(sessionId);
+        controller.close();
+      }, 900000); // 15 minutes
+    }
+  });
+
+  return new Response(stream, {
+    headers: {
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-cache',
+      'Connection': 'keep-alive',
+      'Access-Control-Allow-Origin': '*',
+      'Access-Control-Allow-Headers': 'Cache-Control',
+    },
+  });
+}
+
+// Helper function for other routes to send progress updates
+export function sendProgressUpdate(sessionId: string, update: ProgressUpdate) {
+  if (!progressStore.has(sessionId)) {
+    progressStore.set(sessionId, []);
+  }
+  progressStore.get(sessionId)!.push(update);
+}
+
+// Export the helper for other modules to use
+export { progressStore };