scraping and improvements

frontend/next.config.js

@@ -1,12 +1,19 @@
 /** @type {import('next').NextConfig} */
 const nextConfig = {
-  async rewrites() {
-    return [
-      {
-        source: '/api/:path*',
-        destination: 'http://backend:8080/api/:path*',
-      },
-    ];
+  // Removed Next.js rewrites since nginx handles all API routing
+  webpack: (config, { isServer }) => {
+    // Exclude cheerio and its dependencies from client-side bundling
+    if (!isServer) {
+      config.resolve.fallback = {
+        ...config.resolve.fallback,
+        fs: false,
+        net: false,
+        tls: false,
+        'undici': false,
+      };
+      config.externals.push('cheerio', 'server-only');
+    }
+    return config;
   },
   images: {
     domains: ['localhost'],