shardegger/storycove
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Various improvements & Epub support

Browse Source
This commit is contained in:
Stefan Hardegger
2025-08-08 14:09:14 +02:00
parent 090b858a54
commit 379c8c170f
37 changed files with 4069 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
frontend/src/app/add-story/page.tsx
View File

@@ -64,6 +64,32 @@ export default function AddStoryPage() {
}
}, [searchParams]);
// Load pending story data from bulk combine operation
useEffect(() => {
const fromBulkCombine = searchParams.get('from') === 'bulk-combine';
if (fromBulkCombine) {
const pendingStoryData = localStorage.getItem('pendingStory');
if (pendingStoryData) {
try {
const storyData = JSON.parse(pendingStoryData);
setFormData(prev => ({
...prev,
title: storyData.title || '',
authorName: storyData.author || '',
contentHtml: storyData.content || '',
sourceUrl: storyData.sourceUrl || '',
summary: storyData.summary || '',
tags: storyData.tags || []
}));
// Clear the pending data
localStorage.removeItem('pendingStory');
} catch (error) {
console.error('Failed to load pending story data:', error);
}
}
}
}, [searchParams]);
// Check for duplicates when title and author are both present
useEffect(() => {
const checkDuplicates = async () => {
@@ -442,7 +468,7 @@ export default function AddStoryPage() {
</label>
<ImageUpload
onImageSelect={setCoverImage}
accept="image/jpeg,image/png,image/webp"
accept="image/jpeg,image/png"
maxSizeMB={5}
aspectRatio="3:4"
placeholder="Drop a cover image here or click to select"

2
frontend/src/app/authors/[id]/edit/page.tsx
View File

@@ -269,7 +269,7 @@ export default function EditAuthorPage() {
</label>
<ImageUpload
onImageSelect={setAvatarImage}
accept="image/jpeg,image/png,image/webp"
accept="image/jpeg,image/png"
maxSizeMB={5}
aspectRatio="1:1"
placeholder="Drop an avatar image here or click to select"

35
frontend/src/app/library/page.tsx
View File

@@ -11,16 +11,17 @@ import TagFilter from '../../components/stories/TagFilter';
import LoadingSpinner from '../../components/ui/LoadingSpinner';
type ViewMode = 'grid' | 'list';
type SortOption = 'createdAt' | 'title' | 'authorName' | 'rating' | 'wordCount';
type SortOption = 'createdAt' | 'title' | 'authorName' | 'rating' | 'wordCount' | 'lastRead';
export default function LibraryPage() {
const [stories, setStories] = useState<Story[]>([]);
const [tags, setTags] = useState<Tag[]>([]);
const [loading, setLoading] = useState(false);
const [searchLoading, setSearchLoading] = useState(false);
const [searchQuery, setSearchQuery] = useState('');
const [selectedTags, setSelectedTags] = useState<string[]>([]);
const [viewMode, setViewMode] = useState<ViewMode>('list');
const [sortOption, setSortOption] = useState<SortOption>('createdAt');
const [sortOption, setSortOption] = useState<SortOption>('lastRead');
const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>('desc');
const [page, setPage] = useState(0);
const [totalPages, setTotalPages] = useState(1);
@@ -47,7 +48,13 @@ export default function LibraryPage() {
const debounceTimer = setTimeout(() => {
const performSearch = async () => {
try {
setLoading(true);
// Use searchLoading for background search, loading only for initial load
const isInitialLoad = stories.length === 0 && !searchQuery && selectedTags.length === 0;
if (isInitialLoad) {
setLoading(true);
} else {
setSearchLoading(true);
}
// Always use search API for consistency - use '*' for match-all when no query
const result = await searchApi.search({
@@ -73,11 +80,12 @@ export default function LibraryPage() {
setStories([]);
} finally {
setLoading(false);
setSearchLoading(false);
}
};
performSearch();
}, searchQuery ? 300 : 0); // Debounce search, but not other changes
}, searchQuery ? 500 : 0); // 500ms debounce for search, immediate for other changes
return () => clearTimeout(debounceTimer);
}, [searchQuery, selectedTags, page, sortOption, sortDirection, refreshTrigger]);
@@ -154,16 +162,21 @@ export default function LibraryPage() {
</p>
</div>
<Button href="/add-story">
Add New Story
</Button>
<div className="flex gap-2">
<Button href="/add-story">
Add New Story
</Button>
<Button href="/stories/import/epub" variant="secondary">
📖 Import EPUB
</Button>
</div>
</div>
{/* Search and Filters */}
<div className="space-y-4">
{/* Search Bar */}
<div className="flex flex-col sm:flex-row gap-4">
<div className="flex-1">
<div className="flex-1 relative">
<Input
type="search"
placeholder="Search by title, author, or tags..."
@@ -171,6 +184,11 @@ export default function LibraryPage() {
onChange={handleSearchChange}
className="w-full"
/>
{searchLoading && (
<div className="absolute right-3 top-1/2 transform -translate-y-1/2">
<div className="animate-spin h-4 w-4 border-2 border-theme-accent border-t-transparent rounded-full"></div>
</div>
)}
</div>
{/* View Mode Toggle */}
@@ -215,6 +233,7 @@ export default function LibraryPage() {
<option value="authorName">Author</option>
<option value="rating">Rating</option>
<option value="wordCount">Word Count</option>
<option value="lastRead">Last Read</option>
</select>
{/* Sort Direction Toggle */}

93
frontend/src/app/scrape/bulk/progress/route.ts Normal file
View File

@@ -0,0 +1,93 @@
import { NextRequest } from 'next/server';
// Configure route timeout for long-running progress streams
export const maxDuration = 900; // 15 minutes (900 seconds)
interface ProgressUpdate {
type: 'progress' | 'completed' | 'error';
current: number;
total: number;
message: string;
url?: string;
title?: string;
author?: string;
wordCount?: number;
totalWordCount?: number;
error?: string;
combinedStory?: any;
results?: any[];
summary?: any;
}
// Global progress storage (in production, use Redis or database)
const progressStore = new Map<string, ProgressUpdate[]>();
export async function GET(request: NextRequest) {
const searchParams = request.nextUrl.searchParams;
const sessionId = searchParams.get('sessionId');
if (!sessionId) {
return new Response('Session ID required', { status: 400 });
}
// Set up Server-Sent Events
const stream = new ReadableStream({
start(controller) {
const encoder = new TextEncoder();
// Send initial connection message
const data = `data: ${JSON.stringify({ type: 'connected', sessionId })}\n\n`;
controller.enqueue(encoder.encode(data));
// Check for progress updates every 500ms
const interval = setInterval(() => {
const updates = progressStore.get(sessionId);
if (updates && updates.length > 0) {
// Send all pending updates
updates.forEach(update => {
const data = `data: ${JSON.stringify(update)}\n\n`;
controller.enqueue(encoder.encode(data));
});
// Clear sent updates
progressStore.delete(sessionId);
// If this was a completion or error, close the stream
const lastUpdate = updates[updates.length - 1];
if (lastUpdate.type === 'completed' || lastUpdate.type === 'error') {
clearInterval(interval);
controller.close();
}
}
}, 500);
// Cleanup after timeout
setTimeout(() => {
clearInterval(interval);
progressStore.delete(sessionId);
controller.close();
}, 900000); // 15 minutes
}
});
return new Response(stream, {
headers: {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Headers': 'Cache-Control',
},
});
}
// Helper function for other routes to send progress updates
export function sendProgressUpdate(sessionId: string, update: ProgressUpdate) {
if (!progressStore.has(sessionId)) {
progressStore.set(sessionId, []);
}
progressStore.get(sessionId)!.push(update);
}
// Export the helper for other modules to use
export { progressStore };

730
frontend/src/app/scrape/bulk/route.ts
View File

@@ -1,7 +1,23 @@
import { NextRequest, NextResponse } from 'next/server';
// Configure route timeout for long-running scraping operations
export const maxDuration = 900; // 15 minutes (900 seconds)
// Import progress tracking helper
async function sendProgressUpdate(sessionId: string, update: any) {
try {
// Dynamic import to avoid circular dependency
const { sendProgressUpdate: sendUpdate } = await import('./progress/route');
sendUpdate(sessionId, update);
} catch (error) {
console.warn('Failed to send progress update:', error);
}
}
interface BulkImportRequest {
urls: string[];
combineIntoOne?: boolean;
sessionId?: string; // For progress tracking
}
interface ImportResult {
@@ -22,52 +38,430 @@ interface BulkImportResponse {
skipped: number;
errors: number;
};
combinedStory?: {
title: string;
author: string;
content: string;
summary?: string;
sourceUrl: string;
tags?: string[];
};
}
export async function POST(request: NextRequest) {
// Background processing function for combined mode
async function processCombinedMode(
urls: string[],
sessionId: string,
authorization: string,
scraper: any
) {
const results: ImportResult[] = [];
let importedCount = 0;
let errorCount = 0;
const combinedContent: string[] = [];
let baseTitle = '';
let baseAuthor = '';
let baseSummary = '';
let baseSourceUrl = '';
const combinedTags = new Set<string>();
let totalWordCount = 0;
// Send initial progress update
await sendProgressUpdate(sessionId, {
type: 'progress',
current: 0,
total: urls.length,
message: `Starting to scrape ${urls.length} URLs for combining...`,
totalWordCount: 0
});
for (let i = 0; i < urls.length; i++) {
const url = urls[i];
console.log(`Scraping URL ${i + 1}/${urls.length} for combine: ${url}`);
// Send progress update
await sendProgressUpdate(sessionId, {
type: 'progress',
current: i,
total: urls.length,
message: `Scraping URL ${i + 1} of ${urls.length}...`,
url: url,
totalWordCount
});
try {
const trimmedUrl = url.trim();
if (!trimmedUrl) {
results.push({
url: url || 'Empty URL',
status: 'error',
error: 'Empty URL in combined mode'
});
errorCount++;
continue;
}
const scrapedStory = await scraper.scrapeStory(trimmedUrl);
// Check if we got content - this is required for combined mode
if (!scrapedStory.content || scrapedStory.content.trim() === '') {
results.push({
url: trimmedUrl,
status: 'error',
error: 'No content found - required for combined mode'
});
errorCount++;
continue;
}
// Use first URL for base metadata (title can be empty for combined mode)
if (i === 0) {
baseTitle = scrapedStory.title || 'Combined Story';
baseAuthor = scrapedStory.author || 'Unknown Author';
baseSummary = scrapedStory.summary || '';
baseSourceUrl = trimmedUrl;
}
// Add content with URL separator
combinedContent.push(`<!-- Content from: ${trimmedUrl} -->`);
if (scrapedStory.title && i > 0) {
combinedContent.push(`<h2>${scrapedStory.title}</h2>`);
}
combinedContent.push(scrapedStory.content);
combinedContent.push('<hr/>'); // Visual separator between parts
// Calculate word count for this story
const textContent = scrapedStory.content.replace(/<[^>]*>/g, ''); // Strip HTML
const wordCount = textContent.split(/\s+/).filter((word: string) => word.length > 0).length;
totalWordCount += wordCount;
// Collect tags from all stories
if (scrapedStory.tags) {
scrapedStory.tags.forEach((tag: string) => combinedTags.add(tag));
}
results.push({
url: trimmedUrl,
status: 'imported',
title: scrapedStory.title,
author: scrapedStory.author
});
importedCount++;
// Send progress update with word count
await sendProgressUpdate(sessionId, {
type: 'progress',
current: i + 1,
total: urls.length,
message: `Scraped "${scrapedStory.title}" (${wordCount.toLocaleString()} words)`,
url: trimmedUrl,
title: scrapedStory.title,
author: scrapedStory.author,
wordCount: wordCount,
totalWordCount: totalWordCount
});
} catch (error) {
console.error(`Error processing URL ${url} in combined mode:`, error);
results.push({
url: url,
status: 'error',
error: error instanceof Error ? error.message : 'Unknown error'
});
errorCount++;
}
}
// If we have any errors, fail the entire combined operation
if (errorCount > 0) {
await sendProgressUpdate(sessionId, {
type: 'error',
current: urls.length,
total: urls.length,
message: 'Combined mode failed: some URLs could not be processed',
error: `${errorCount} URLs failed to process`
});
return;
}
// Check content size to prevent response size issues
const combinedContentString = combinedContent.join('\n');
const contentSizeInMB = new Blob([combinedContentString]).size / (1024 * 1024);
console.log(`Combined content size: ${contentSizeInMB.toFixed(2)} MB`);
console.log(`Combined content character length: ${combinedContentString.length}`);
console.log(`Combined content parts count: ${combinedContent.length}`);
// Return the combined story data via progress update
const combinedStory = {
title: baseTitle,
author: baseAuthor,
content: contentSizeInMB > 10 ?
combinedContentString.substring(0, Math.floor(combinedContentString.length * (10 / contentSizeInMB))) + '\n\n<!-- Content truncated due to size limit -->' :
combinedContentString,
summary: contentSizeInMB > 10 ? baseSummary + ' (Content truncated due to size limit)' : baseSummary,
sourceUrl: baseSourceUrl,
tags: Array.from(combinedTags)
};
// Send completion notification for combine mode
await sendProgressUpdate(sessionId, {
type: 'completed',
current: urls.length,
total: urls.length,
message: `Combined scraping completed: ${totalWordCount.toLocaleString()} words from ${importedCount} stories`,
totalWordCount: totalWordCount,
combinedStory: combinedStory
});
console.log(`Combined scraping completed: ${importedCount} URLs combined into one story`);
}
// Background processing function for individual mode
async function processIndividualMode(
urls: string[],
sessionId: string,
authorization: string,
scraper: any
) {
const results: ImportResult[] = [];
let importedCount = 0;
let skippedCount = 0;
let errorCount = 0;
await sendProgressUpdate(sessionId, {
type: 'progress',
current: 0,
total: urls.length,
message: `Starting to import ${urls.length} URLs individually...`
});
for (let i = 0; i < urls.length; i++) {
const url = urls[i];
console.log(`Processing URL ${i + 1}/${urls.length}: ${url}`);
await sendProgressUpdate(sessionId, {
type: 'progress',
current: i,
total: urls.length,
message: `Processing URL ${i + 1} of ${urls.length}...`,
url: url
});
try {
// Validate URL format
if (!url || typeof url !== 'string' || url.trim() === '') {
results.push({
url: url || 'Empty URL',
status: 'error',
error: 'Invalid URL format'
});
errorCount++;
continue;
}
const trimmedUrl = url.trim();
// Scrape the story
const scrapedStory = await scraper.scrapeStory(trimmedUrl);
// Validate required fields
if (!scrapedStory.title || !scrapedStory.author || !scrapedStory.content) {
const missingFields = [];
if (!scrapedStory.title) missingFields.push('title');
if (!scrapedStory.author) missingFields.push('author');
if (!scrapedStory.content) missingFields.push('content');
results.push({
url: trimmedUrl,
status: 'skipped',
reason: `Missing required fields: ${missingFields.join(', ')}`,
title: scrapedStory.title,
author: scrapedStory.author
});
skippedCount++;
continue;
}
// Check for duplicates using query parameters
try {
const duplicateCheckUrl = `http://backend:8080/api/stories/check-duplicate`;
const params = new URLSearchParams({
title: scrapedStory.title,
authorName: scrapedStory.author
});
const duplicateCheckResponse = await fetch(`${duplicateCheckUrl}?${params.toString()}`, {
method: 'GET',
headers: {
'Authorization': authorization,
'Content-Type': 'application/json',
},
});
if (duplicateCheckResponse.ok) {
const duplicateResult = await duplicateCheckResponse.json();
if (duplicateResult.hasDuplicates) {
results.push({
url: trimmedUrl,
status: 'skipped',
reason: `Duplicate story found (${duplicateResult.count} existing)`,
title: scrapedStory.title,
author: scrapedStory.author
});
skippedCount++;
continue;
}
}
} catch (error) {
console.warn('Duplicate check failed:', error);
// Continue with import if duplicate check fails
}
// Create the story
try {
const storyData = {
title: scrapedStory.title,
summary: scrapedStory.summary || undefined,
contentHtml: scrapedStory.content,
sourceUrl: scrapedStory.sourceUrl || trimmedUrl,
authorName: scrapedStory.author,
tagNames: scrapedStory.tags && scrapedStory.tags.length > 0 ? scrapedStory.tags : undefined,
};
const createUrl = `http://backend:8080/api/stories`;
const createResponse = await fetch(createUrl, {
method: 'POST',
headers: {
'Authorization': authorization,
'Content-Type': 'application/json',
},
body: JSON.stringify(storyData),
});
if (!createResponse.ok) {
const errorData = await createResponse.json();
throw new Error(errorData.message || 'Failed to create story');
}
const createdStory = await createResponse.json();
results.push({
url: trimmedUrl,
status: 'imported',
title: scrapedStory.title,
author: scrapedStory.author,
storyId: createdStory.id
});
importedCount++;
console.log(`Successfully imported: ${scrapedStory.title} by ${scrapedStory.author} (ID: ${createdStory.id})`);
// Send progress update for successful import
await sendProgressUpdate(sessionId, {
type: 'progress',
current: i + 1,
total: urls.length,
message: `Imported "${scrapedStory.title}" by ${scrapedStory.author}`,
url: trimmedUrl,
title: scrapedStory.title,
author: scrapedStory.author
});
} catch (error) {
console.error(`Failed to create story for ${trimmedUrl}:`, error);
let errorMessage = 'Failed to create story';
if (error instanceof Error) {
errorMessage = error.message;
}
results.push({
url: trimmedUrl,
status: 'error',
error: errorMessage,
title: scrapedStory.title,
author: scrapedStory.author
});
errorCount++;
}
} catch (error) {
console.error(`Error processing URL ${url}:`, error);
let errorMessage = 'Unknown error';
if (error instanceof Error) {
errorMessage = error.message;
}
results.push({
url: url,
status: 'error',
error: errorMessage
});
errorCount++;
}
}
// Send completion notification
await sendProgressUpdate(sessionId, {
type: 'completed',
current: urls.length,
total: urls.length,
message: `Bulk import completed: ${importedCount} imported, ${skippedCount} skipped, ${errorCount} errors`,
results: results,
summary: {
total: urls.length,
imported: importedCount,
skipped: skippedCount,
errors: errorCount
}
});
console.log(`Bulk import completed: ${importedCount} imported, ${skippedCount} skipped, ${errorCount} errors`);
// Trigger Typesense reindex if any stories were imported
if (importedCount > 0) {
try {
console.log('Triggering Typesense reindex after bulk import...');
const reindexUrl = `http://backend:8080/api/stories/reindex-typesense`;
const reindexResponse = await fetch(reindexUrl, {
method: 'POST',
headers: {
'Authorization': authorization,
'Content-Type': 'application/json',
},
});
if (reindexResponse.ok) {
const reindexResult = await reindexResponse.json();
console.log('Typesense reindex completed:', reindexResult);
} else {
console.warn('Typesense reindex failed:', reindexResponse.status);
}
} catch (error) {
console.warn('Failed to trigger Typesense reindex:', error);
// Don't fail the whole request if reindex fails
}
}
}
// Background processing function
async function processBulkImport(
urls: string[],
combineIntoOne: boolean,
sessionId: string,
authorization: string
) {
try {
// Check for authentication
const authorization = request.headers.get('authorization');
if (!authorization) {
return NextResponse.json(
{ error: 'Authentication required for bulk import' },
{ status: 401 }
);
}
const body = await request.json();
const { urls } = body as BulkImportRequest;
if (!urls || !Array.isArray(urls) || urls.length === 0) {
return NextResponse.json(
{ error: 'URLs array is required and must not be empty' },
{ status: 400 }
);
}
if (urls.length > 50) {
return NextResponse.json(
{ error: 'Maximum 50 URLs allowed per bulk import' },
{ status: 400 }
);
}
// Dynamic imports to prevent client-side bundling
const { StoryScraper } = await import('@/lib/scraper/scraper');
const scraper = new StoryScraper();
const results: ImportResult[] = [];
let importedCount = 0;
let skippedCount = 0;
let errorCount = 0;
console.log(`Starting bulk scraping for ${urls.length} URLs`);
console.log(`Environment NEXT_PUBLIC_API_URL: ${process.env.NEXT_PUBLIC_API_URL}`);
// For server-side API calls in Docker, use direct backend container URL
// Client-side calls use NEXT_PUBLIC_API_URL through nginx, but server-side needs direct container access
const serverSideApiBaseUrl = 'http://backend:8080/api';
console.log(`DEBUG: serverSideApiBaseUrl variable is: ${serverSideApiBaseUrl}`);
console.log(`Starting bulk scraping for ${urls.length} URLs${combineIntoOne ? ' (combine mode)' : ''}`);
console.log(`Session ID: ${sessionId}`);
// Quick test to verify backend connectivity
try {
@@ -84,208 +478,86 @@ export async function POST(request: NextRequest) {
console.error(`Backend connectivity test failed:`, error);
}
for (const url of urls) {
console.log(`Processing URL: ${url}`);
try {
// Validate URL format
if (!url || typeof url !== 'string' || url.trim() === '') {
results.push({
url: url || 'Empty URL',
status: 'error',
error: 'Invalid URL format'
});
errorCount++;
continue;
}
const trimmedUrl = url.trim();
// Scrape the story
const scrapedStory = await scraper.scrapeStory(trimmedUrl);
// Validate required fields
if (!scrapedStory.title || !scrapedStory.author || !scrapedStory.content) {
const missingFields = [];
if (!scrapedStory.title) missingFields.push('title');
if (!scrapedStory.author) missingFields.push('author');
if (!scrapedStory.content) missingFields.push('content');
results.push({
url: trimmedUrl,
status: 'skipped',
reason: `Missing required fields: ${missingFields.join(', ')}`,
title: scrapedStory.title,
author: scrapedStory.author
});
skippedCount++;
continue;
}
// Check for duplicates using query parameters
try {
// Use hardcoded backend URL for container-to-container communication
const duplicateCheckUrl = `http://backend:8080/api/stories/check-duplicate`;
console.log(`Duplicate check URL: ${duplicateCheckUrl}`);
const params = new URLSearchParams({
title: scrapedStory.title,
authorName: scrapedStory.author
});
const duplicateCheckResponse = await fetch(`${duplicateCheckUrl}?${params.toString()}`, {
method: 'GET',
headers: {
'Authorization': authorization,
'Content-Type': 'application/json',
},
});
if (duplicateCheckResponse.ok) {
const duplicateResult = await duplicateCheckResponse.json();
if (duplicateResult.hasDuplicates) {
results.push({
url: trimmedUrl,
status: 'skipped',
reason: `Duplicate story found (${duplicateResult.count} existing)`,
title: scrapedStory.title,
author: scrapedStory.author
});
skippedCount++;
continue;
}
}
} catch (error) {
console.warn('Duplicate check failed:', error);
// Continue with import if duplicate check fails
}
// Create the story
try {
const storyData = {
title: scrapedStory.title,
summary: scrapedStory.summary || undefined,
contentHtml: scrapedStory.content,
sourceUrl: scrapedStory.sourceUrl || trimmedUrl,
authorName: scrapedStory.author,
tagNames: scrapedStory.tags && scrapedStory.tags.length > 0 ? scrapedStory.tags : undefined,
};
// Use hardcoded backend URL for container-to-container communication
const createUrl = `http://backend:8080/api/stories`;
console.log(`Create story URL: ${createUrl}`);
const createResponse = await fetch(createUrl, {
method: 'POST',
headers: {
'Authorization': authorization,
'Content-Type': 'application/json',
},
body: JSON.stringify(storyData),
});
if (!createResponse.ok) {
const errorData = await createResponse.json();
throw new Error(errorData.message || 'Failed to create story');
}
const createdStory = await createResponse.json();
results.push({
url: trimmedUrl,
status: 'imported',
title: scrapedStory.title,
author: scrapedStory.author,
storyId: createdStory.id
});
importedCount++;
console.log(`Successfully imported: ${scrapedStory.title} by ${scrapedStory.author} (ID: ${createdStory.id})`);
} catch (error) {
console.error(`Failed to create story for ${trimmedUrl}:`, error);
let errorMessage = 'Failed to create story';
if (error instanceof Error) {
errorMessage = error.message;
}
results.push({
url: trimmedUrl,
status: 'error',
error: errorMessage,
title: scrapedStory.title,
author: scrapedStory.author
});
errorCount++;
}
} catch (error) {
console.error(`Error processing URL ${url}:`, error);
let errorMessage = 'Unknown error';
if (error instanceof Error) {
errorMessage = error.message;
}
results.push({
url: url,
status: 'error',
error: errorMessage
});
errorCount++;
}
// Handle combined mode
if (combineIntoOne) {
await processCombinedMode(urls, sessionId, authorization, scraper);
} else {
// Normal individual processing mode
await processIndividualMode(urls, sessionId, authorization, scraper);
}
const response: BulkImportResponse = {
results,
summary: {
total: urls.length,
imported: importedCount,
skipped: skippedCount,
errors: errorCount
}
};
console.log(`Bulk import completed:`, response.summary);
// Trigger Typesense reindex if any stories were imported
if (importedCount > 0) {
try {
console.log('Triggering Typesense reindex after bulk import...');
const reindexUrl = `http://backend:8080/api/stories/reindex-typesense`;
const reindexResponse = await fetch(reindexUrl, {
method: 'POST',
headers: {
'Authorization': authorization,
'Content-Type': 'application/json',
},
});
if (reindexResponse.ok) {
const reindexResult = await reindexResponse.json();
console.log('Typesense reindex completed:', reindexResult);
} else {
console.warn('Typesense reindex failed:', reindexResponse.status);
}
} catch (error) {
console.warn('Failed to trigger Typesense reindex:', error);
// Don't fail the whole request if reindex fails
}
}
return NextResponse.json(response);
} catch (error) {
console.error('Bulk import error:', error);
console.error('Background bulk import error:', error);
await sendProgressUpdate(sessionId, {
type: 'error',
current: 0,
total: urls.length,
message: 'Bulk import failed due to an error',
error: error instanceof Error ? error.message : 'Unknown error'
});
}
}
export async function POST(request: NextRequest) {
try {
// Check for authentication
const authorization = request.headers.get('authorization');
if (!authorization) {
return NextResponse.json(
{ error: 'Authentication required for bulk import' },
{ status: 401 }
);
}
const body = await request.json();
const { urls, combineIntoOne = false, sessionId } = body as BulkImportRequest;
if (!urls || !Array.isArray(urls) || urls.length === 0) {
return NextResponse.json(
{ error: 'URLs array is required and must not be empty' },
{ status: 400 }
);
}
if (urls.length > 200) {
return NextResponse.json(
{ error: 'Maximum 200 URLs allowed per bulk import' },
{ status: 400 }
);
}
if (!sessionId) {
return NextResponse.json(
{ error: 'Session ID is required for progress tracking' },
{ status: 400 }
);
}
// Start the background processing
processBulkImport(urls, combineIntoOne, sessionId, authorization).catch(error => {
console.error('Failed to start background processing:', error);
});
// Return immediately with session info
return NextResponse.json({
message: 'Bulk import started',
sessionId: sessionId,
totalUrls: urls.length,
combineMode: combineIntoOne
});
} catch (error) {
console.error('Bulk import initialization error:', error);
if (error instanceof Error) {
return NextResponse.json(
{ error: `Bulk import failed: ${error.message}` },
{ error: `Bulk import failed to start: ${error.message}` },
{ status: 500 }
);
}
return NextResponse.json(
{ error: 'Bulk import failed due to an unknown error' },
{ error: 'Bulk import failed to start due to an unknown error' },
{ status: 500 }
);
}

56
frontend/src/app/stories/[id]/detail/page.tsx
View File

@@ -21,6 +21,7 @@ export default function StoryDetailPage() {
const [collections, setCollections] = useState<Collection[]>([]);
const [loading, setLoading] = useState(true);
const [updating, setUpdating] = useState(false);
const [isExporting, setIsExporting] = useState(false);
useEffect(() => {
const loadStoryData = async () => {
@@ -65,6 +66,53 @@ export default function StoryDetailPage() {
}
};
const handleEPUBExport = async () => {
if (!story) return;
setIsExporting(true);
try {
const token = localStorage.getItem('auth-token');
const response = await fetch(`/api/stories/${story.id}/epub`, {
method: 'GET',
headers: {
'Authorization': token ? `Bearer ${token}` : '',
},
});
if (response.ok) {
const blob = await response.blob();
const url = window.URL.createObjectURL(blob);
const link = document.createElement('a');
link.href = url;
// Get filename from Content-Disposition header or create default
const contentDisposition = response.headers.get('Content-Disposition');
let filename = `${story.title}.epub`;
if (contentDisposition) {
const match = contentDisposition.match(/filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/);
if (match && match[1]) {
filename = match[1].replace(/['"]/g, '');
}
}
link.download = filename;
document.body.appendChild(link);
link.click();
window.URL.revokeObjectURL(url);
document.body.removeChild(link);
} else if (response.status === 401 || response.status === 403) {
alert('Authentication required. Please log in.');
} else {
throw new Error('Failed to export EPUB');
}
} catch (error) {
console.error('Error exporting EPUB:', error);
alert('Failed to export EPUB. Please try again.');
} finally {
setIsExporting(false);
}
};
const formatDate = (dateString: string) => {
return new Date(dateString).toLocaleDateString('en-US', {
year: 'numeric',
@@ -358,6 +406,14 @@ export default function StoryDetailPage() {
>
📚 Start Reading
</Button>
<Button
onClick={handleEPUBExport}
variant="ghost"
size="lg"
disabled={isExporting}
>
{isExporting ? 'Exporting...' : '📖 Export EPUB'}
</Button>
<Button
href={`/stories/${story.id}/edit`}
variant="ghost"

2
frontend/src/app/stories/[id]/edit/page.tsx
View File

@@ -252,7 +252,7 @@ export default function EditStoryPage() {
</label>
<ImageUpload
onImageSelect={setCoverImage}
accept="image/jpeg,image/png,image/webp"
accept="image/jpeg,image/png"
maxSizeMB={5}
aspectRatio="3:4"
placeholder="Drop a new cover image here or click to select"

1
frontend/src/app/stories/[id]/page.tsx
View File

@@ -201,6 +201,7 @@ export default function StoryReadingPage() {
}
};
const findNextStory = (): Story | null => {
if (!story?.seriesId || seriesStories.length <= 1) return null;

116
frontend/src/app/stories/import/bulk/page.tsx
View File

@@ -4,6 +4,7 @@ import { useState } from 'react';
import { useRouter } from 'next/navigation';
import Link from 'next/link';
import { ArrowLeftIcon } from '@heroicons/react/24/outline';
import BulkImportProgress from '@/components/BulkImportProgress';
interface ImportResult {
url: string;
@@ -23,14 +24,25 @@ interface BulkImportResponse {
skipped: number;
errors: number;
};
combinedStory?: {
title: string;
author: string;
content: string;
summary?: string;
sourceUrl: string;
tags?: string[];
};
}
export default function BulkImportPage() {
const router = useRouter();
const [urls, setUrls] = useState('');
const [combineIntoOne, setCombineIntoOne] = useState(false);
const [isLoading, setIsLoading] = useState(false);
const [results, setResults] = useState<BulkImportResponse | null>(null);
const [error, setError] = useState<string | null>(null);
const [sessionId, setSessionId] = useState<string | null>(null);
const [showProgress, setShowProgress] = useState(false);
const handleSubmit = async (e: React.FormEvent) => {
e.preventDefault();
@@ -57,12 +69,17 @@ export default function BulkImportPage() {
return;
}
if (urlList.length > 50) {
setError('Maximum 50 URLs allowed per bulk import');
if (urlList.length > 200) {
setError('Maximum 200 URLs allowed per bulk import');
setIsLoading(false);
return;
}
// Generate session ID for progress tracking
const newSessionId = `bulk-import-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
setSessionId(newSessionId);
setShowProgress(true);
// Get auth token for server-side API calls
const token = localStorage.getItem('auth-token');
@@ -72,16 +89,18 @@ export default function BulkImportPage() {
'Content-Type': 'application/json',
'Authorization': token ? `Bearer ${token}` : '',
},
body: JSON.stringify({ urls: urlList }),
body: JSON.stringify({ urls: urlList, combineIntoOne, sessionId: newSessionId }),
});
if (!response.ok) {
const errorData = await response.json();
throw new Error(errorData.error || 'Bulk import failed');
throw new Error(errorData.error || 'Failed to start bulk import');
}
const data: BulkImportResponse = await response.json();
setResults(data);
const startData = await response.json();
console.log('Bulk import started:', startData);
// The progress component will handle the rest via SSE
} catch (err) {
console.error('Bulk import error:', err);
@@ -93,8 +112,43 @@ export default function BulkImportPage() {
const handleReset = () => {
setUrls('');
setCombineIntoOne(false);
setResults(null);
setError(null);
setSessionId(null);
setShowProgress(false);
};
const handleProgressComplete = (data?: any) => {
// Progress component will handle this when the operation completes
setShowProgress(false);
setIsLoading(false);
// Handle completion data
if (data) {
if (data.combinedStory && combineIntoOne) {
// For combine mode, redirect to add story page with the combined content
localStorage.setItem('pendingStory', JSON.stringify(data.combinedStory));
router.push('/add-story?from=bulk-combine');
return;
} else if (data.results && data.summary) {
// For individual mode, show the results
setResults({
results: data.results,
summary: data.summary
});
return;
}
}
// Fallback: just hide progress and let user know it completed
console.log('Import completed successfully');
};
const handleProgressError = (errorMessage: string) => {
setError(errorMessage);
setIsLoading(false);
setShowProgress(false);
};
const getStatusColor = (status: string) => {
@@ -145,7 +199,7 @@ export default function BulkImportPage() {
Story URLs
</label>
<p className="text-sm text-gray-500 mb-3">
Enter one URL per line. Maximum 50 URLs per import.
Enter one URL per line. Maximum 200 URLs per import.
</p>
<textarea
id="urls"
@@ -160,6 +214,37 @@ export default function BulkImportPage() {
</p>
</div>
<div className="flex items-center">
<input
id="combine-into-one"
type="checkbox"
checked={combineIntoOne}
onChange={(e) => setCombineIntoOne(e.target.checked)}
className="h-4 w-4 text-blue-600 focus:ring-blue-500 border-gray-300 rounded"
disabled={isLoading}
/>
<label htmlFor="combine-into-one" className="ml-2 block text-sm text-gray-700">
Combine all URL content into a single story
</label>
</div>
{combineIntoOne && (
<div className="bg-blue-50 border border-blue-200 rounded-md p-4">
<div className="text-sm text-blue-800">
<p className="font-medium mb-2">Combined Story Mode:</p>
<ul className="list-disc list-inside space-y-1 text-blue-700">
<li>All URLs will be scraped and their content combined into one story</li>
<li>Story title and author will be taken from the first URL</li>
<li>Import will fail if any URL has no content (title/author can be empty)</li>
<li>You'll be redirected to the story creation page to review and edit</li>
{urls.split('\n').filter(url => url.trim().length > 0).length > 50 && (
<li className="text-yellow-700 font-medium">⚠️ Large imports (50+ URLs) may take several minutes and could be truncated if too large</li>
)}
</ul>
</div>
</div>
)}
{error && (
<div className="bg-red-50 border border-red-200 rounded-md p-4">
<div className="flex">
@@ -192,14 +277,25 @@ export default function BulkImportPage() {
</button>
</div>
{isLoading && (
{/* Progress Component */}
{showProgress && sessionId && (
<BulkImportProgress
sessionId={sessionId}
onComplete={handleProgressComplete}
onError={handleProgressError}
combineMode={combineIntoOne}
/>
)}
{/* Fallback loading indicator if progress isn't shown yet */}
{isLoading && !showProgress && (
<div className="bg-blue-50 border border-blue-200 rounded-md p-4">
<div className="flex items-center">
<div className="animate-spin rounded-full h-5 w-5 border-b-2 border-blue-600 mr-3"></div>
<div>
<p className="text-sm font-medium text-blue-800">Processing URLs...</p>
<p className="text-sm font-medium text-blue-800">Starting import...</p>
<p className="text-sm text-blue-600">
This may take a few minutes depending on the number of URLs and response times of the source websites.
Preparing to process {urls.split('\n').filter(url => url.trim().length > 0).length} URLs.
</p>
</div>
</div>

432
frontend/src/app/stories/import/epub/page.tsx Normal file
View File

@@ -0,0 +1,432 @@
'use client';
import { useState } from 'react';
import { useRouter } from 'next/navigation';
import Link from 'next/link';
import { ArrowLeftIcon, DocumentArrowUpIcon } from '@heroicons/react/24/outline';
import Button from '@/components/ui/Button';
import { Input } from '@/components/ui/Input';
interface EPUBImportResponse {
success: boolean;
message: string;
storyId?: string;
storyTitle?: string;
totalChapters?: number;
wordCount?: number;
readingPosition?: any;
warnings?: string[];
errors?: string[];
}
export default function EPUBImportPage() {
const router = useRouter();
const [selectedFile, setSelectedFile] = useState<File | null>(null);
const [isLoading, setIsLoading] = useState(false);
const [isValidating, setIsValidating] = useState(false);
const [validationResult, setValidationResult] = useState<any>(null);
const [importResult, setImportResult] = useState<EPUBImportResponse | null>(null);
const [error, setError] = useState<string | null>(null);
// Import options
const [authorName, setAuthorName] = useState<string>('');
const [seriesName, setSeriesName] = useState<string>('');
const [seriesVolume, setSeriesVolume] = useState<string>('');
const [tags, setTags] = useState<string>('');
const [preserveReadingPosition, setPreserveReadingPosition] = useState(true);
const [overwriteExisting, setOverwriteExisting] = useState(false);
const [createMissingAuthor, setCreateMissingAuthor] = useState(true);
const [createMissingSeries, setCreateMissingSeries] = useState(true);
const handleFileChange = async (e: React.ChangeEvent<HTMLInputElement>) => {
const file = e.target.files?.[0];
if (file) {
setSelectedFile(file);
setValidationResult(null);
setImportResult(null);
setError(null);
if (file.name.toLowerCase().endsWith('.epub')) {
await validateFile(file);
} else {
setError('Please select a valid EPUB file (.epub extension)');
}
}
};
const validateFile = async (file: File) => {
setIsValidating(true);
try {
const token = localStorage.getItem('auth-token');
const formData = new FormData();
formData.append('file', file);
const response = await fetch('/api/stories/epub/validate', {
method: 'POST',
headers: {
'Authorization': token ? `Bearer ${token}` : '',
},
body: formData,
});
if (response.ok) {
const result = await response.json();
setValidationResult(result);
if (!result.valid) {
setError('EPUB file validation failed: ' + result.errors.join(', '));
}
} else if (response.status === 401 || response.status === 403) {
setError('Authentication required. Please log in.');
} else {
setError('Failed to validate EPUB file');
}
} catch (err) {
setError('Error validating EPUB file: ' + (err as Error).message);
} finally {
setIsValidating(false);
}
};
const handleSubmit = async (e: React.FormEvent) => {
e.preventDefault();
if (!selectedFile) {
setError('Please select an EPUB file');
return;
}
if (validationResult && !validationResult.valid) {
setError('Cannot import invalid EPUB file');
return;
}
setIsLoading(true);
setError(null);
try {
const token = localStorage.getItem('auth-token');
const formData = new FormData();
formData.append('file', selectedFile);
if (authorName) formData.append('authorName', authorName);
if (seriesName) formData.append('seriesName', seriesName);
if (seriesVolume) formData.append('seriesVolume', seriesVolume);
if (tags) {
const tagList = tags.split(',').map(t => t.trim()).filter(t => t.length > 0);
tagList.forEach(tag => formData.append('tags', tag));
}
formData.append('preserveReadingPosition', preserveReadingPosition.toString());
formData.append('overwriteExisting', overwriteExisting.toString());
formData.append('createMissingAuthor', createMissingAuthor.toString());
formData.append('createMissingSeries', createMissingSeries.toString());
const response = await fetch('/api/stories/epub/import', {
method: 'POST',
headers: {
'Authorization': token ? `Bearer ${token}` : '',
},
body: formData,
});
const result = await response.json();
if (response.ok && result.success) {
setImportResult(result);
} else if (response.status === 401 || response.status === 403) {
setError('Authentication required. Please log in.');
} else {
setError(result.message || 'Failed to import EPUB');
}
} catch (err) {
setError('Error importing EPUB: ' + (err as Error).message);
} finally {
setIsLoading(false);
}
};
const resetForm = () => {
setSelectedFile(null);
setValidationResult(null);
setImportResult(null);
setError(null);
setAuthorName('');
setSeriesName('');
setSeriesVolume('');
setTags('');
};
if (importResult?.success) {
return (
<div className="container mx-auto px-4 py-8">
<div className="mb-6">
<Link
href="/stories"
className="inline-flex items-center text-blue-600 hover:text-blue-800 mb-4"
>
<ArrowLeftIcon className="h-4 w-4 mr-2" />
Back to Stories
</Link>
<h1 className="text-3xl font-bold text-gray-900 dark:text-white">
EPUB Import Successful
</h1>
</div>
<div className="bg-white dark:bg-gray-800 border border-gray-200 dark:border-gray-700 rounded-lg p-6">
<div className="mb-6">
<h2 className="text-xl font-semibold text-green-600 mb-2">Import Completed</h2>
<p className="text-gray-600 dark:text-gray-300">
Your EPUB has been successfully imported into StoryCove.
</p>
</div>
<div>
<div className="space-y-4">
<div>
<span className="font-semibold text-gray-700 dark:text-gray-300">Story Title:</span>
<p className="text-gray-900 dark:text-white">{importResult.storyTitle}</p>
</div>
{importResult.wordCount && (
<div>
<span className="font-semibold text-gray-700 dark:text-gray-300">Word Count:</span>
<p className="text-gray-900 dark:text-white">{importResult.wordCount.toLocaleString()} words</p>
</div>
)}
{importResult.totalChapters && (
<div>
<span className="font-semibold text-gray-700 dark:text-gray-300">Chapters:</span>
<p className="text-gray-900 dark:text-white">{importResult.totalChapters}</p>
</div>
)}
{importResult.warnings && importResult.warnings.length > 0 && (
<div className="bg-yellow-50 border border-yellow-200 rounded-md p-4">
<strong className="text-yellow-800">Warnings:</strong>
<ul className="list-disc list-inside mt-2 text-yellow-700">
{importResult.warnings.map((warning, index) => (
<li key={index}>{warning}</li>
))}
</ul>
</div>
)}
<div className="flex gap-4 mt-6">
<Button
onClick={() => router.push(`/stories/${importResult.storyId}`)}
className="bg-blue-600 hover:bg-blue-700 text-white"
>
View Story
</Button>
<Button
onClick={resetForm}
variant="secondary"
>
Import Another EPUB
</Button>
</div>
</div>
</div>
</div>
</div>
);
}
return (
<div className="container mx-auto px-4 py-8">
<div className="mb-6">
<Link
href="/stories"
className="inline-flex items-center text-blue-600 hover:text-blue-800 mb-4"
>
<ArrowLeftIcon className="h-4 w-4 mr-2" />
Back to Stories
</Link>
<h1 className="text-3xl font-bold text-gray-900 dark:text-white">
Import EPUB
</h1>
<p className="text-gray-600 dark:text-gray-300 mt-2">
Upload an EPUB file to import it as a story into your library.
</p>
</div>
{error && (
<div className="bg-red-50 border border-red-200 rounded-md p-4 mb-6">
<p className="text-red-700">{error}</p>
</div>
)}
<form onSubmit={handleSubmit} className="space-y-6">
{/* File Upload */}
<div className="bg-white dark:bg-gray-800 border border-gray-200 dark:border-gray-700 rounded-lg p-6">
<div className="mb-4">
<h3 className="text-lg font-semibold mb-2">Select EPUB File</h3>
<p className="text-gray-600 dark:text-gray-300">
Choose an EPUB file from your device to import.
</p>
</div>
<div className="space-y-4">
<div>
<label htmlFor="epub-file" className="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">EPUB File</label>
<Input
id="epub-file"
type="file"
accept=".epub,application/epub+zip"
onChange={handleFileChange}
disabled={isLoading || isValidating}
/>
</div>
{selectedFile && (
<div className="flex items-center gap-2">
<DocumentArrowUpIcon className="h-5 w-5" />
<span className="text-sm text-gray-600">
{selectedFile.name} ({(selectedFile.size / 1024 / 1024).toFixed(2)} MB)
</span>
</div>
)}
{isValidating && (
<div className="text-sm text-blue-600">
Validating EPUB file...
</div>
)}
{validationResult && (
<div className="text-sm">
{validationResult.valid ? (
<span className="inline-flex items-center px-2 py-1 rounded text-xs font-medium bg-green-100 text-green-800">
Valid EPUB
</span>
) : (
<span className="inline-flex items-center px-2 py-1 rounded text-xs font-medium bg-red-100 text-red-800">
Invalid EPUB
</span>
)}
</div>
)}
</div>
</div>
{/* Import Options */}
<div className="bg-white dark:bg-gray-800 border border-gray-200 dark:border-gray-700 rounded-lg p-6">
<div className="mb-4">
<h3 className="text-lg font-semibold mb-2">Import Options</h3>
<p className="text-gray-600 dark:text-gray-300">
Configure how the EPUB should be imported.
</p>
</div>
<div className="space-y-4">
<div>
<label htmlFor="author-name" className="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Author Name (Override)</label>
<Input
id="author-name"
value={authorName}
onChange={(e) => setAuthorName(e.target.value)}
placeholder="Leave empty to use EPUB metadata"
/>
</div>
<div>
<label htmlFor="series-name" className="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Series Name</label>
<Input
id="series-name"
value={seriesName}
onChange={(e) => setSeriesName(e.target.value)}
placeholder="Optional: Add to a series"
/>
</div>
{seriesName && (
<div>
<label htmlFor="series-volume" className="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Series Volume</label>
<Input
id="series-volume"
type="number"
value={seriesVolume}
onChange={(e) => setSeriesVolume(e.target.value)}
placeholder="Volume number in series"
/>
</div>
)}
<div>
<label htmlFor="tags" className="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Tags</label>
<Input
id="tags"
value={tags}
onChange={(e) => setTags(e.target.value)}
placeholder="Comma-separated tags (e.g., fantasy, adventure, romance)"
/>
</div>
<div className="space-y-3">
<div className="flex items-center">
<input
type="checkbox"
id="preserve-reading-position"
checked={preserveReadingPosition}
onChange={(e) => setPreserveReadingPosition(e.target.checked)}
className="mr-2"
/>
<label htmlFor="preserve-reading-position" className="text-sm text-gray-700 dark:text-gray-300">
Preserve reading position from EPUB metadata
</label>
</div>
<div className="flex items-center">
<input
type="checkbox"
id="create-missing-author"
checked={createMissingAuthor}
onChange={(e) => setCreateMissingAuthor(e.target.checked)}
className="mr-2"
/>
<label htmlFor="create-missing-author" className="text-sm text-gray-700 dark:text-gray-300">
Create author if not found
</label>
</div>
<div className="flex items-center">
<input
type="checkbox"
id="create-missing-series"
checked={createMissingSeries}
onChange={(e) => setCreateMissingSeries(e.target.checked)}
className="mr-2"
/>
<label htmlFor="create-missing-series" className="text-sm text-gray-700 dark:text-gray-300">
Create series if not found
</label>
</div>
<div className="flex items-center">
<input
type="checkbox"
id="overwrite-existing"
checked={overwriteExisting}
onChange={(e) => setOverwriteExisting(e.target.checked)}
className="mr-2"
/>
<label htmlFor="overwrite-existing" className="text-sm text-gray-700 dark:text-gray-300">
Overwrite existing story with same title and author
</label>
</div>
</div>
</div>
</div>
{/* Submit Button */}
<div className="flex justify-end">
<Button
type="submit"
disabled={!selectedFile || isLoading || isValidating || (validationResult && !validationResult.valid)}
className="bg-blue-600 hover:bg-blue-700 text-white"
>
{isLoading ? 'Importing...' : 'Import EPUB'}
</Button>
</div>
</form>
</div>
);
}

207
frontend/src/components/BulkImportProgress.tsx Normal file
View File

@@ -0,0 +1,207 @@
'use client';
import { useEffect, useState } from 'react';
interface ProgressUpdate {
type: 'progress' | 'completed' | 'error' | 'connected';
current: number;
total: number;
message: string;
url?: string;
title?: string;
author?: string;
wordCount?: number;
totalWordCount?: number;
error?: string;
sessionId?: string;
}
interface BulkImportProgressProps {
sessionId: string;
onComplete?: (data?: any) => void;
onError?: (error: string) => void;
combineMode?: boolean;
}
export default function BulkImportProgress({
sessionId,
onComplete,
onError,
combineMode = false
}: BulkImportProgressProps) {
const [progress, setProgress] = useState<ProgressUpdate>({
type: 'progress',
current: 0,
total: 1,
message: 'Connecting...'
});
const [isConnected, setIsConnected] = useState(false);
const [recentActivities, setRecentActivities] = useState<string[]>([]);
useEffect(() => {
const eventSource = new EventSource(`/scrape/bulk/progress?sessionId=${sessionId}`);
eventSource.onmessage = (event) => {
try {
const data: ProgressUpdate = JSON.parse(event.data);
if (data.type === 'connected') {
setIsConnected(true);
return;
}
setProgress(data);
// Add to recent activities (keep last 5)
if (data.message) {
setRecentActivities(prev => [
data.message,
...prev.slice(0, 4)
]);
}
if (data.type === 'completed') {
setTimeout(() => {
onComplete?.(data);
eventSource.close();
}, 2000); // Show completion message for 2 seconds
} else if (data.type === 'error') {
onError?.(data.error || 'Unknown error occurred');
eventSource.close();
}
} catch (error) {
console.error('Failed to parse progress update:', error);
}
};
eventSource.onerror = (error) => {
console.error('EventSource error:', error);
setIsConnected(false);
onError?.('Connection to progress stream failed');
eventSource.close();
};
return () => {
eventSource.close();
};
}, [sessionId, onComplete, onError]);
const progressPercentage = progress.total > 0
? Math.round((progress.current / progress.total) * 100)
: 0;
const getStatusColor = () => {
switch (progress.type) {
case 'completed': return 'bg-green-600';
case 'error': return 'bg-red-600';
default: return 'bg-blue-600';
}
};
const getStatusIcon = () => {
switch (progress.type) {
case 'completed': return '✓';
case 'error': return '✗';
default: return null;
}
};
return (
<div className="bg-white border border-gray-200 rounded-lg p-6">
<div className="mb-4">
<div className="flex items-center justify-between mb-2">
<h3 className="text-lg font-medium text-gray-900">
{combineMode ? 'Combining Stories' : 'Bulk Import Progress'}
</h3>
<div className="flex items-center gap-2">
{!isConnected && (
<div className="h-2 w-2 bg-yellow-400 rounded-full animate-pulse"></div>
)}
<span className="text-sm text-gray-600">
{progress.current} of {progress.total}
</span>
</div>
</div>
{/* Progress Bar */}
<div className="w-full bg-gray-200 rounded-full h-3 mb-3">
<div
className={`h-3 rounded-full transition-all duration-500 ${getStatusColor()}`}
style={{ width: `${progressPercentage}%` }}
></div>
</div>
{/* Progress Percentage */}
<div className="flex items-center justify-between">
<span className="text-sm font-medium text-gray-900">
{progressPercentage}%
</span>
{progress.type === 'completed' && (
<span className="text-green-600 font-medium">
{getStatusIcon()} Complete
</span>
)}
{progress.type === 'error' && (
<span className="text-red-600 font-medium">
{getStatusIcon()} Error
</span>
)}
</div>
</div>
{/* Current Status Message */}
<div className="mb-4">
<div className="flex items-center gap-2">
{progress.type === 'progress' && (
<div className="animate-spin rounded-full h-4 w-4 border-b-2 border-blue-600"></div>
)}
<p className="text-sm text-gray-700">{progress.message}</p>
</div>
{/* Word Count for Combine Mode */}
{combineMode && progress.totalWordCount !== undefined && (
<p className="text-sm text-gray-500 mt-1">
Total words collected: {progress.totalWordCount.toLocaleString()}
</p>
)}
</div>
{/* Current URL being processed */}
{progress.url && (
<div className="mb-4 p-3 bg-gray-50 rounded-md">
<p className="text-sm text-gray-600 mb-1">Currently processing:</p>
<p className="text-sm font-mono text-gray-800 truncate">{progress.url}</p>
{progress.title && progress.author && (
<p className="text-sm text-gray-600 mt-1">
"{progress.title}" by {progress.author}
{progress.wordCount && (
<span className="ml-2 text-gray-500">
({progress.wordCount.toLocaleString()} words)
</span>
)}
</p>
)}
</div>
)}
{/* Recent Activities */}
{recentActivities.length > 0 && (
<div>
<h4 className="text-sm font-medium text-gray-900 mb-2">Recent Activity</h4>
<div className="space-y-1 max-h-32 overflow-y-auto">
{recentActivities.map((activity, index) => (
<p
key={index}
className={`text-xs text-gray-600 ${
index === 0 ? 'font-medium text-gray-800' : ''
}`}
>
{activity}
</p>
))}
</div>
</div>
)}
</div>
);
}

2
frontend/src/components/collections/CollectionForm.tsx
View File

@@ -227,7 +227,7 @@ export default function CollectionForm({
<input
id="coverImage"
type="file"
accept="image/jpeg,image/png,image/webp"
accept="image/jpeg,image/png"
onChange={handleCoverImageChange}
className="w-full px-3 py-2 border theme-border rounded-lg theme-card theme-text focus:outline-none focus:ring-2 focus:ring-theme-accent"
/>

12
frontend/src/components/layout/Header.tsx
View File

@@ -26,6 +26,11 @@ export default function Header() {
label: 'Import from URL',
description: 'Import a single story from a website'
},
{
href: '/stories/import/epub',
label: 'Import EPUB',
description: 'Import a story from an EPUB file'
},
{
href: '/stories/import/bulk',
label: 'Bulk Import',
@@ -165,6 +170,13 @@ export default function Header() {
>
Import from URL
</Link>
<Link
href="/stories/import/epub"
className="block theme-text hover:theme-accent transition-colors text-sm py-1"
onClick={() => setIsMenuOpen(false)}
>
Import EPUB
</Link>
<Link
href="/stories/import/bulk"
className="block theme-text hover:theme-accent transition-colors text-sm py-1"

250
frontend/src/components/stories/RichTextEditor.tsx
View File

@@ -20,9 +20,12 @@ export default function RichTextEditor({
}: RichTextEditorProps) {
const [viewMode, setViewMode] = useState<'visual' | 'html'>('visual');
const [htmlValue, setHtmlValue] = useState(value);
const [isMaximized, setIsMaximized] = useState(false);
const [containerHeight, setContainerHeight] = useState(300); // Default height in pixels
const previewRef = useRef<HTMLDivElement>(null);
const visualTextareaRef = useRef<HTMLTextAreaElement>(null);
const visualDivRef = useRef<HTMLDivElement>(null);
const containerRef = useRef<HTMLDivElement>(null);
const [isUserTyping, setIsUserTyping] = useState(false);
// Utility functions for cursor position preservation
@@ -60,6 +63,62 @@ export default function RichTextEditor({
}
};
// Maximize/minimize functionality
const toggleMaximize = () => {
if (!isMaximized) {
// Store current height before maximizing
if (containerRef.current) {
setContainerHeight(containerRef.current.scrollHeight || containerHeight);
}
}
setIsMaximized(!isMaximized);
};
// Handle manual resize when dragging resize handle
const handleMouseDown = (e: React.MouseEvent) => {
if (isMaximized) return; // Don't allow resize when maximized
e.preventDefault();
const startY = e.clientY;
const startHeight = containerHeight;
const handleMouseMove = (e: MouseEvent) => {
const deltaY = e.clientY - startY;
const newHeight = Math.max(200, Math.min(800, startHeight + deltaY)); // Min 200px, Max 800px
setContainerHeight(newHeight);
};
const handleMouseUp = () => {
document.removeEventListener('mousemove', handleMouseMove);
document.removeEventListener('mouseup', handleMouseUp);
};
document.addEventListener('mousemove', handleMouseMove);
document.addEventListener('mouseup', handleMouseUp);
};
// Escape key handler for maximized mode
useEffect(() => {
const handleEscapeKey = (e: KeyboardEvent) => {
if (e.key === 'Escape' && isMaximized) {
setIsMaximized(false);
}
};
if (isMaximized) {
document.addEventListener('keydown', handleEscapeKey);
// Prevent body from scrolling when maximized
document.body.style.overflow = 'hidden';
} else {
document.body.style.overflow = '';
}
return () => {
document.removeEventListener('keydown', handleEscapeKey);
document.body.style.overflow = '';
};
}, [isMaximized]);
// Set initial content when component mounts
useEffect(() => {
const div = visualDivRef.current;
@@ -439,6 +498,17 @@ export default function RichTextEditor({
</div>
<div className="flex items-center gap-1">
<Button
type="button"
size="sm"
variant="ghost"
onClick={toggleMaximize}
title={isMaximized ? "Minimize editor" : "Maximize editor"}
className="font-mono"
>
{isMaximized ? "⊡" : "⊞"}
</Button>
<div className="w-px h-4 bg-gray-300 mx-1" />
<Button
type="button"
size="sm"
@@ -504,40 +574,160 @@ export default function RichTextEditor({
</div>
{/* Editor */}
<div className="border theme-border rounded-b-lg overflow-hidden">
{viewMode === 'visual' ? (
<div className="relative">
<div
ref={visualDivRef}
contentEditable
onInput={handleVisualContentChange}
onPaste={handlePaste}
className="p-3 min-h-[300px] focus:outline-none focus:ring-0 whitespace-pre-wrap"
style={{ minHeight: '300px' }}
suppressContentEditableWarning={true}
/>
{!value && (
<div
className="absolute top-3 left-3 text-gray-500 dark:text-gray-400 pointer-events-none select-none"
style={{ minHeight: '300px' }}
>
{placeholder}
<div
className={`relative border theme-border rounded-b-lg ${
isMaximized ? 'fixed inset-4 z-50 bg-white dark:bg-gray-900 shadow-2xl' : ''
}`}
style={isMaximized ? {} : { height: containerHeight }}
>
<div
ref={containerRef}
className="h-full flex flex-col overflow-hidden"
>
{/* Maximized toolbar (shown when maximized) */}
{isMaximized && (
<div className="flex items-center justify-between p-2 theme-card border-b theme-border">
<div className="flex items-center gap-2">
<Button
type="button"
size="sm"
variant="ghost"
onClick={() => setViewMode('visual')}
className={viewMode === 'visual' ? 'theme-accent-bg text-white' : ''}
>
Visual
</Button>
<Button
type="button"
size="sm"
variant="ghost"
onClick={() => setViewMode('html')}
className={viewMode === 'html' ? 'theme-accent-bg text-white' : ''}
>
HTML
</Button>
</div>
<div className="flex items-center gap-1">
<Button
type="button"
size="sm"
variant="ghost"
onClick={toggleMaximize}
title="Minimize editor"
className="font-mono"
>
⊡
</Button>
<div className="w-px h-4 bg-gray-300 mx-1" />
<Button
type="button"
size="sm"
variant="ghost"
onClick={() => formatText('strong')}
title="Bold"
className="font-bold"
>
B
</Button>
<Button
type="button"
size="sm"
variant="ghost"
onClick={() => formatText('em')}
title="Italic"
className="italic"
>
I
</Button>
<div className="w-px h-4 bg-gray-300 mx-1" />
<Button
type="button"
size="sm"
variant="ghost"
onClick={() => formatText('h1')}
title="Heading 1"
className="text-lg font-bold"
>
H1
</Button>
<Button
type="button"
size="sm"
variant="ghost"
onClick={() => formatText('h2')}
title="Heading 2"
className="text-base font-bold"
>
H2
</Button>
<Button
type="button"
size="sm"
variant="ghost"
onClick={() => formatText('h3')}
title="Heading 3"
className="text-sm font-bold"
>
H3
</Button>
<div className="w-px h-4 bg-gray-300 mx-1" />
<Button
type="button"
size="sm"
variant="ghost"
onClick={() => formatText('p')}
title="Paragraph"
>
P
</Button>
</div>
</div>
)}
{/* Editor content */}
<div className="flex-1 overflow-hidden">
{viewMode === 'visual' ? (
<div className="relative h-full">
<div
ref={visualDivRef}
contentEditable
onInput={handleVisualContentChange}
onPaste={handlePaste}
className="p-3 h-full overflow-y-auto focus:outline-none focus:ring-0 whitespace-pre-wrap resize-none"
suppressContentEditableWarning={true}
/>
{!value && (
<div className="absolute top-3 left-3 text-gray-500 dark:text-gray-400 pointer-events-none select-none">
{placeholder}
</div>
)}
</div>
) : (
<Textarea
value={htmlValue}
onChange={handleHtmlChange}
placeholder="<p>Write your HTML content here...</p>"
className="border-0 rounded-none focus:ring-0 font-mono text-sm h-full resize-none"
/>
)}
</div>
) : (
<Textarea
value={htmlValue}
onChange={handleHtmlChange}
placeholder="<p>Write your HTML content here...</p>"
rows={12}
className="border-0 rounded-none focus:ring-0 font-mono text-sm"
/>
</div>
{/* Resize handle (only show when not maximized) */}
{!isMaximized && (
<div
onMouseDown={handleMouseDown}
className="absolute bottom-0 left-0 right-0 h-2 cursor-ns-resize bg-gray-200 dark:bg-gray-700 hover:bg-gray-300 dark:hover:bg-gray-600 transition-colors flex items-center justify-center"
title="Drag to resize"
>
<div className="w-8 h-0.5 bg-gray-400 dark:bg-gray-500 rounded-full"></div>
</div>
)}
</div>
{/* Preview for HTML mode */}
{viewMode === 'html' && value && (
{/* Preview for HTML mode (only show when not maximized) */}
{viewMode === 'html' && value && !isMaximized && (
<div className="space-y-2">
<h4 className="text-sm font-medium theme-header">Preview:</h4>
<div
@@ -561,6 +751,10 @@ export default function RichTextEditor({
<strong>HTML mode:</strong> Edit HTML source directly for advanced formatting.
Allowed tags: p, br, div, span, strong, em, b, i, u, s, h1-h6, ul, ol, li, blockquote, and more.
</p>
<p>
<strong>Tips:</strong> Use the ⊞ button to maximize the editor for larger stories.
Drag the resize handle at the bottom to adjust height. Press Escape to exit maximized mode.
</p>
</div>
</div>
);

26
frontend/src/components/ui/ImageUpload.tsx
View File

@@ -32,7 +32,8 @@ export default function ImageUpload({
if (rejection.errors?.[0]?.code === 'file-too-large') {
setError(`File is too large. Maximum size is ${maxSizeMB}MB.`);
} else if (rejection.errors?.[0]?.code === 'file-invalid-type') {
setError('Invalid file type. Please select an image file.');
const allowedTypes = accept.split(',').map(type => type.trim()).join(', ');
setError(`Invalid file type. Supported formats: ${allowedTypes.replace(/image\//g, '').toUpperCase()}.`);
} else {
setError('File rejected. Please try another file.');
}
@@ -41,18 +42,31 @@ export default function ImageUpload({
const file = acceptedFiles[0];
if (file) {
// Additional client-side validation for file type
const allowedTypes = accept.split(',').map(type => type.trim());
if (!allowedTypes.includes(file.type)) {
const supportedFormats = allowedTypes.map(type => type.replace('image/', '').toUpperCase()).join(', ');
setError(`Invalid file type. Your file is ${file.type}. Supported formats: ${supportedFormats}.`);
return;
}
// Create preview
const previewUrl = URL.createObjectURL(file);
setPreview(previewUrl);
onImageSelect(file);
}
}, [onImageSelect, maxSizeMB]);
}, [onImageSelect, maxSizeMB, accept]);
// Build proper accept object for dropzone based on specific MIME types
const acceptTypes = accept.split(',').map(type => type.trim());
const dropzoneAccept = acceptTypes.reduce((acc, type) => {
acc[type] = []; // Empty array means accept files with this MIME type
return acc;
}, {} as Record<string, string[]>);
const { getRootProps, getInputProps, isDragActive } = useDropzone({
onDrop,
accept: {
'image/*': accept.split(',').map(type => type.trim()),
},
accept: dropzoneAccept,
maxFiles: 1,
maxSize: maxSizeMB * 1024 * 1024, // Convert MB to bytes
});
@@ -123,7 +137,7 @@ export default function ImageUpload({
)}
</div>
<p className="text-sm text-gray-500">
Supports JPEG, PNG, WebP up to {maxSizeMB}MB
Supports {acceptTypes.map(type => type.replace('image/', '').toUpperCase()).join(', ')} up to {maxSizeMB}MB
</p>
</div>
)}

14
frontend/src/lib/scraper/config/sites.json
View File

@@ -235,6 +235,16 @@
"requiresJavaScript": true
}
}
},
"wanderinginn.com": {
"story": {
"title": "h1.entry-title",
"author": "pirateaba",
"content": ".entry-content",
"summary": "meta[property='og:description']",
"summaryAttribute": "content"
}
}
},
@@ -329,6 +339,10 @@
"fanfiction.net": {
"note": "Older site with simpler HTML structure",
"warning": "Known to block IPs for aggressive scraping"
},
"wanderinginn.com": {
"note": "WordPress-based site with consistent structure",
"author": "All stories by pirateaba - uses text pattern matching for content extraction"
}
}
}

3
frontend/src/lib/scraper/types.ts
View File

@@ -1,6 +1,6 @@
export interface SiteConfig {
story: StorySelectors;
authorPage: AuthorPageSelectors;
authorPage?: AuthorPageSelectors;
}
export interface StorySelectors {
@@ -13,6 +13,7 @@ export interface StorySelectors {
multiPage?: MultiPageConfig;
titleFallback?: string;
titleFallbackAttribute?: string;
contentFallback?: string;
titleTransform?: string;
summaryAttribute?: string;
coverImageAttribute?: string;