scraping and improvements

nginx.conf

@@ -28,7 +28,23 @@ http {
             proxy_cache_bypass $http_upgrade;
         }
 
-        # Backend API routes
+        # Scraping routes - completely separate from /api/ to avoid conflicts
+        location /scrape/ {
+            proxy_pass http://frontend/scrape/;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection 'upgrade';
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_cache_bypass $http_upgrade;
+            proxy_connect_timeout 60s;
+            proxy_send_timeout 60s;
+            proxy_read_timeout 60s;
+        }
+
+        # Backend API routes (fallback for all other /api/ routes)
         location /api/ {
             proxy_pass http://backend/api/;
             proxy_set_header Host $host;