shardegger/storycove
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

removing typesense

Browse Source
This commit is contained in:
Stefan Hardegger
2025-09-20 14:39:51 +02:00
parent f1773873d4
commit aae8f8926b
34 changed files with 4664 additions and 5094 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
frontend/src/app/authors/page.tsx
View File

@@ -22,7 +22,7 @@ export default function AuthorsPage() {
const [currentPage, setCurrentPage] = useState(0);
const [totalHits, setTotalHits] = useState(0);
const [hasMore, setHasMore] = useState(false);
const ITEMS_PER_PAGE = 50; // Safe limit under Typesense's 250 limit
const ITEMS_PER_PAGE = 50;
useEffect(() => {
const debounceTimer = setTimeout(() => {
@@ -35,41 +35,30 @@ export default function AuthorsPage() {
} else {
setSearchLoading(true);
}
const searchResults = await authorApi.searchAuthorsTypesense({
q: searchQuery || '*',
page: currentPage,
const searchResults = await authorApi.getAuthors({
page: currentPage,
size: ITEMS_PER_PAGE,
sortBy: sortBy,
sortOrder: sortOrder
sortDir: sortOrder
});
if (currentPage === 0) {
// First page - replace all results
setAuthors(searchResults.results || []);
setFilteredAuthors(searchResults.results || []);
setAuthors(searchResults.content || []);
setFilteredAuthors(searchResults.content || []);
} else {
// Subsequent pages - append results
setAuthors(prev => [...prev, ...(searchResults.results || [])]);
setFilteredAuthors(prev => [...prev, ...(searchResults.results || [])]);
setAuthors(prev => [...prev, ...(searchResults.content || [])]);
setFilteredAuthors(prev => [...prev, ...(searchResults.content || [])]);
}
setTotalHits(searchResults.totalHits);
setHasMore(searchResults.results.length === ITEMS_PER_PAGE && (currentPage + 1) * ITEMS_PER_PAGE < searchResults.totalHits);
setTotalHits(searchResults.totalElements || 0);
setHasMore(searchResults.content.length === ITEMS_PER_PAGE && (currentPage + 1) * ITEMS_PER_PAGE < (searchResults.totalElements || 0));
} catch (error) {
console.error('Failed to load authors:', error);
// Fallback to regular API if Typesense fails (only for first page)
if (currentPage === 0) {
try {
const authorsResult = await authorApi.getAuthors({ page: 0, size: ITEMS_PER_PAGE });
setAuthors(authorsResult.content || []);
setFilteredAuthors(authorsResult.content || []);
setTotalHits(authorsResult.totalElements || 0);
setHasMore(authorsResult.content.length === ITEMS_PER_PAGE);
} catch (fallbackError) {
console.error('Fallback also failed:', fallbackError);
}
}
// Error handling for API failures
console.error('Failed to load authors:', error);
} finally {
setLoading(false);
setSearchLoading(false);
@@ -95,7 +84,17 @@ export default function AuthorsPage() {
}
};
// Client-side filtering no longer needed since we use Typesense
// Client-side filtering for search query when using regular API
useEffect(() => {
if (searchQuery) {
const filtered = authors.filter(author =>
author.name.toLowerCase().includes(searchQuery.toLowerCase())
);
setFilteredAuthors(filtered);
} else {
setFilteredAuthors(authors);
}
}, [authors, searchQuery]);
// Note: We no longer have individual story ratings in the author list
// Average rating would need to be calculated on backend if needed
@@ -118,9 +117,9 @@ export default function AuthorsPage() {
<div>
<h1 className="text-3xl font-bold theme-header">Authors</h1>
<p className="theme-text mt-1">
{filteredAuthors.length} of {totalHits} {totalHits === 1 ? 'author' : 'authors'}
{searchQuery ? `${filteredAuthors.length} of ${authors.length}` : filteredAuthors.length} {(searchQuery ? authors.length : filteredAuthors.length) === 1 ? 'author' : 'authors'}
{searchQuery ? ` found` : ` in your library`}
{hasMore && ` (showing first ${filteredAuthors.length})`}
{!searchQuery && hasMore && ` (showing first ${filteredAuthors.length})`}
</p>
</div>
@@ -218,7 +217,7 @@ export default function AuthorsPage() {
)}
{/* Load More Button */}
{hasMore && (
{hasMore && !searchQuery && (
<div className="flex justify-center pt-8">
<Button
onClick={loadMore}
@@ -227,7 +226,7 @@ export default function AuthorsPage() {
className="px-8 py-3"
loading={loading}
>
{loading ? 'Loading...' : `Load More Authors (${totalHits - filteredAuthors.length} remaining)`}
{loading ? 'Loading...' : `Load More Authors (${totalHits - authors.length} remaining)`}
</Button>
</div>
)}

14
frontend/src/app/scrape/bulk/route.ts
View File

@@ -501,11 +501,11 @@ async function processIndividualMode(
console.log(`Bulk import completed: ${importedCount} imported, ${skippedCount} skipped, ${errorCount} errors`);
// Trigger Typesense reindex if any stories were imported
// Trigger OpenSearch reindex if any stories were imported
if (importedCount > 0) {
try {
console.log('Triggering Typesense reindex after bulk import...');
const reindexUrl = `http://backend:8080/api/stories/reindex-typesense`;
console.log('Triggering OpenSearch reindex after bulk import...');
const reindexUrl = `http://backend:8080/api/admin/search/opensearch/reindex`;
const reindexResponse = await fetch(reindexUrl, {
method: 'POST',
headers: {
@@ -513,15 +513,15 @@ async function processIndividualMode(
'Content-Type': 'application/json',
},
});
if (reindexResponse.ok) {
const reindexResult = await reindexResponse.json();
console.log('Typesense reindex completed:', reindexResult);
console.log('OpenSearch reindex completed:', reindexResult);
} else {
console.warn('Typesense reindex failed:', reindexResponse.status);
console.warn('OpenSearch reindex failed:', reindexResponse.status);
}
} catch (error) {
console.warn('Failed to trigger Typesense reindex:', error);
console.warn('Failed to trigger OpenSearch reindex:', error);
// Don't fail the whole request if reindex fails
}
}

356
frontend/src/components/settings/SystemSettings.tsx
View File

@@ -2,7 +2,7 @@
import React, { useState, useEffect } from 'react';
import Button from '../ui/Button';
import { storyApi, authorApi, databaseApi, configApi, searchAdminApi } from '../../lib/api';
import { databaseApi, configApi, searchAdminApi } from '../../lib/api';
interface SystemSettingsProps {
// No props needed - this component manages its own state
@@ -11,16 +11,12 @@ interface SystemSettingsProps {
export default function SystemSettings({}: SystemSettingsProps) {
const [searchEngineStatus, setSearchEngineStatus] = useState<{
currentEngine: string;
dualWrite: boolean;
typesenseAvailable: boolean;
openSearchAvailable: boolean;
loading: boolean;
message: string;
success?: boolean;
}>({
currentEngine: 'typesense',
dualWrite: false,
typesenseAvailable: false,
currentEngine: 'opensearch',
openSearchAvailable: false,
loading: false,
message: ''
@@ -34,13 +30,6 @@ export default function SystemSettings({}: SystemSettingsProps) {
recreate: { loading: false, message: '' }
});
const [typesenseStatus, setTypesenseStatus] = useState<{
reindex: { loading: boolean; message: string; success?: boolean };
recreate: { loading: boolean; message: string; success?: boolean };
}>({
reindex: { loading: false, message: '' },
recreate: { loading: false, message: '' }
});
const [databaseStatus, setDatabaseStatus] = useState<{
completeBackup: { loading: boolean; message: string; success?: boolean };
completeRestore: { loading: boolean; message: string; success?: boolean };
@@ -58,135 +47,7 @@ export default function SystemSettings({}: SystemSettingsProps) {
execute: { loading: false, message: '' }
});
const handleFullReindex = async () => {
setTypesenseStatus(prev => ({
...prev,
reindex: { loading: true, message: 'Reindexing all collections...', success: undefined }
}));
try {
// Run both story and author reindex in parallel
const [storiesResult, authorsResult] = await Promise.all([
storyApi.reindexTypesense(),
authorApi.reindexTypesense()
]);
const allSuccessful = storiesResult.success && authorsResult.success;
const messages: string[] = [];
if (storiesResult.success) {
messages.push(`Stories: ${storiesResult.message}`);
} else {
messages.push(`Stories failed: ${storiesResult.error || 'Unknown error'}`);
}
if (authorsResult.success) {
messages.push(`Authors: ${authorsResult.message}`);
} else {
messages.push(`Authors failed: ${authorsResult.error || 'Unknown error'}`);
}
setTypesenseStatus(prev => ({
...prev,
reindex: {
loading: false,
message: allSuccessful
? `Full reindex completed successfully. ${messages.join(', ')}`
: `Reindex completed with errors. ${messages.join(', ')}`,
success: allSuccessful
}
}));
// Clear message after 8 seconds (longer for combined operation)
setTimeout(() => {
setTypesenseStatus(prev => ({
...prev,
reindex: { loading: false, message: '', success: undefined }
}));
}, 8000);
} catch (error) {
setTypesenseStatus(prev => ({
...prev,
reindex: {
loading: false,
message: 'Network error occurred during reindex',
success: false
}
}));
setTimeout(() => {
setTypesenseStatus(prev => ({
...prev,
reindex: { loading: false, message: '', success: undefined }
}));
}, 8000);
}
};
const handleRecreateAllCollections = async () => {
setTypesenseStatus(prev => ({
...prev,
recreate: { loading: true, message: 'Recreating all collections...', success: undefined }
}));
try {
// Run both story and author recreation in parallel
const [storiesResult, authorsResult] = await Promise.all([
storyApi.recreateTypesenseCollection(),
authorApi.recreateTypesenseCollection()
]);
const allSuccessful = storiesResult.success && authorsResult.success;
const messages: string[] = [];
if (storiesResult.success) {
messages.push(`Stories: ${storiesResult.message}`);
} else {
messages.push(`Stories failed: ${storiesResult.error || 'Unknown error'}`);
}
if (authorsResult.success) {
messages.push(`Authors: ${authorsResult.message}`);
} else {
messages.push(`Authors failed: ${authorsResult.error || 'Unknown error'}`);
}
setTypesenseStatus(prev => ({
...prev,
recreate: {
loading: false,
message: allSuccessful
? `All collections recreated successfully. ${messages.join(', ')}`
: `Recreation completed with errors. ${messages.join(', ')}`,
success: allSuccessful
}
}));
// Clear message after 8 seconds (longer for combined operation)
setTimeout(() => {
setTypesenseStatus(prev => ({
...prev,
recreate: { loading: false, message: '', success: undefined }
}));
}, 8000);
} catch (error) {
setTypesenseStatus(prev => ({
...prev,
recreate: {
loading: false,
message: 'Network error occurred during recreation',
success: false
}
}));
setTimeout(() => {
setTypesenseStatus(prev => ({
...prev,
recreate: { loading: false, message: '', success: undefined }
}));
}, 8000);
}
};
const handleCompleteBackup = async () => {
setDatabaseStatus(prev => ({
@@ -451,8 +312,6 @@ export default function SystemSettings({}: SystemSettingsProps) {
setSearchEngineStatus(prev => ({
...prev,
currentEngine: status.primaryEngine,
dualWrite: status.dualWrite,
typesenseAvailable: status.typesenseAvailable,
openSearchAvailable: status.openSearchAvailable,
}));
} catch (error: any) {
@@ -460,76 +319,7 @@ export default function SystemSettings({}: SystemSettingsProps) {
}
};
const handleSwitchEngine = async (engine: string) => {
setSearchEngineStatus(prev => ({ ...prev, loading: true, message: `Switching to ${engine}...` }));
try {
const result = engine === 'opensearch'
? await searchAdminApi.switchToOpenSearch()
: await searchAdminApi.switchToTypesense();
setSearchEngineStatus(prev => ({
...prev,
loading: false,
message: result.message,
success: true,
currentEngine: engine
}));
setTimeout(() => {
setSearchEngineStatus(prev => ({ ...prev, message: '', success: undefined }));
}, 5000);
} catch (error: any) {
setSearchEngineStatus(prev => ({
...prev,
loading: false,
message: error.message || 'Failed to switch engine',
success: false
}));
setTimeout(() => {
setSearchEngineStatus(prev => ({ ...prev, message: '', success: undefined }));
}, 5000);
}
};
const handleToggleDualWrite = async () => {
const newDualWrite = !searchEngineStatus.dualWrite;
setSearchEngineStatus(prev => ({
...prev,
loading: true,
message: `${newDualWrite ? 'Enabling' : 'Disabling'} dual-write...`
}));
try {
const result = newDualWrite
? await searchAdminApi.enableDualWrite()
: await searchAdminApi.disableDualWrite();
setSearchEngineStatus(prev => ({
...prev,
loading: false,
message: result.message,
success: true,
dualWrite: newDualWrite
}));
setTimeout(() => {
setSearchEngineStatus(prev => ({ ...prev, message: '', success: undefined }));
}, 5000);
} catch (error: any) {
setSearchEngineStatus(prev => ({
...prev,
loading: false,
message: error.message || 'Failed to toggle dual-write',
success: false
}));
setTimeout(() => {
setSearchEngineStatus(prev => ({ ...prev, message: '', success: undefined }));
}, 5000);
}
};
const handleOpenSearchReindex = async () => {
setOpenSearchStatus(prev => ({
@@ -624,36 +414,18 @@ export default function SystemSettings({}: SystemSettingsProps) {
return (
<div className="space-y-6">
{/* Search Engine Management */}
{/* Search Management */}
<div className="theme-card theme-shadow rounded-lg p-6">
<h2 className="text-xl font-semibold theme-header mb-4">Search Engine Migration</h2>
<h2 className="text-xl font-semibold theme-header mb-4">Search Management</h2>
<p className="theme-text mb-6">
Manage the transition from Typesense to OpenSearch. Switch between engines, enable dual-write mode, and perform maintenance operations.
Manage OpenSearch indices for stories and authors. Use these tools if search isn't returning expected results.
</p>
<div className="space-y-6">
{/* Current Status */}
<div className="border theme-border rounded-lg p-4">
<h3 className="text-lg font-semibold theme-header mb-3">Current Configuration</h3>
<h3 className="text-lg font-semibold theme-header mb-3">Search Status</h3>
<div className="grid grid-cols-1 sm:grid-cols-2 gap-3 text-sm">
<div className="flex justify-between">
<span>Primary Engine:</span>
<span className={`font-medium ${searchEngineStatus.currentEngine === 'opensearch' ? 'text-blue-600 dark:text-blue-400' : 'text-green-600 dark:text-green-400'}`}>
{searchEngineStatus.currentEngine.charAt(0).toUpperCase() + searchEngineStatus.currentEngine.slice(1)}
</span>
</div>
<div className="flex justify-between">
<span>Dual-Write:</span>
<span className={`font-medium ${searchEngineStatus.dualWrite ? 'text-orange-600 dark:text-orange-400' : 'text-gray-600 dark:text-gray-400'}`}>
{searchEngineStatus.dualWrite ? 'Enabled' : 'Disabled'}
</span>
</div>
<div className="flex justify-between">
<span>Typesense:</span>
<span className={`font-medium ${searchEngineStatus.typesenseAvailable ? 'text-green-600 dark:text-green-400' : 'text-red-600 dark:text-red-400'}`}>
{searchEngineStatus.typesenseAvailable ? 'Available' : 'Unavailable'}
</span>
</div>
<div className="flex justify-between">
<span>OpenSearch:</span>
<span className={`font-medium ${searchEngineStatus.openSearchAvailable ? 'text-green-600 dark:text-green-400' : 'text-red-600 dark:text-red-400'}`}>
@@ -663,52 +435,11 @@ export default function SystemSettings({}: SystemSettingsProps) {
</div>
</div>
{/* Engine Switching */}
{/* Search Operations */}
<div className="border theme-border rounded-lg p-4">
<h3 className="text-lg font-semibold theme-header mb-3">Engine Controls</h3>
<div className="flex flex-col sm:flex-row gap-3 mb-4">
<Button
onClick={() => handleSwitchEngine('typesense')}
disabled={searchEngineStatus.loading || !searchEngineStatus.typesenseAvailable || searchEngineStatus.currentEngine === 'typesense'}
variant={searchEngineStatus.currentEngine === 'typesense' ? 'primary' : 'ghost'}
className="flex-1"
>
{searchEngineStatus.currentEngine === 'typesense' ? '✓ Typesense (Active)' : 'Switch to Typesense'}
</Button>
<Button
onClick={() => handleSwitchEngine('opensearch')}
disabled={searchEngineStatus.loading || !searchEngineStatus.openSearchAvailable || searchEngineStatus.currentEngine === 'opensearch'}
variant={searchEngineStatus.currentEngine === 'opensearch' ? 'primary' : 'ghost'}
className="flex-1"
>
{searchEngineStatus.currentEngine === 'opensearch' ? '✓ OpenSearch (Active)' : 'Switch to OpenSearch'}
</Button>
<Button
onClick={handleToggleDualWrite}
disabled={searchEngineStatus.loading}
variant={searchEngineStatus.dualWrite ? 'secondary' : 'ghost'}
className="flex-1"
>
{searchEngineStatus.dualWrite ? 'Disable Dual-Write' : 'Enable Dual-Write'}
</Button>
</div>
{searchEngineStatus.message && (
<div className={`text-sm p-3 rounded mb-3 ${
searchEngineStatus.success
? 'bg-green-50 dark:bg-green-900/20 text-green-800 dark:text-green-200'
: 'bg-red-50 dark:bg-red-900/20 text-red-800 dark:text-red-200'
}`}>
{searchEngineStatus.message}
</div>
)}
</div>
{/* OpenSearch Operations */}
<div className="border theme-border rounded-lg p-4">
<h3 className="text-lg font-semibold theme-header mb-3">OpenSearch Operations</h3>
<h3 className="text-lg font-semibold theme-header mb-3">Search Operations</h3>
<p className="text-sm theme-text mb-4">
Perform maintenance operations on OpenSearch indices. Use these if OpenSearch isn't returning expected results.
Perform maintenance operations on search indices. Use these if search isn't returning expected results.
</p>
<div className="flex flex-col sm:flex-row gap-3 mb-4">
@@ -719,7 +450,7 @@ export default function SystemSettings({}: SystemSettingsProps) {
variant="ghost"
className="flex-1"
>
{openSearchStatus.reindex.loading ? 'Reindexing...' : '🔄 Reindex OpenSearch'}
{openSearchStatus.reindex.loading ? 'Reindexing...' : '🔄 Reindex All'}
</Button>
<Button
onClick={handleOpenSearchRecreate}
@@ -732,7 +463,7 @@ export default function SystemSettings({}: SystemSettingsProps) {
</Button>
</div>
{/* OpenSearch Status Messages */}
{/* Status Messages */}
{openSearchStatus.reindex.message && (
<div className={`text-sm p-3 rounded mb-3 ${
openSearchStatus.reindex.success
@@ -753,73 +484,12 @@ export default function SystemSettings({}: SystemSettingsProps) {
</div>
)}
</div>
</div>
</div>
{/* Legacy Typesense Management */}
<div className="theme-card theme-shadow rounded-lg p-6">
<h2 className="text-xl font-semibold theme-header mb-4">Legacy Typesense Management</h2>
<p className="theme-text mb-6">
Manage Typesense search indexes (for backwards compatibility and during migration). These tools will be removed once migration is complete.
</p>
<div className="space-y-6">
{/* Simplified Operations */}
<div className="border theme-border rounded-lg p-4">
<h3 className="text-lg font-semibold theme-header mb-3">Search Operations</h3>
<p className="text-sm theme-text mb-4">
Perform maintenance operations on all search indexes (stories, authors, collections, etc.).
</p>
<div className="flex flex-col sm:flex-row gap-3 mb-4">
<Button
onClick={handleFullReindex}
disabled={typesenseStatus.reindex.loading || typesenseStatus.recreate.loading}
loading={typesenseStatus.reindex.loading}
variant="ghost"
className="flex-1"
>
{typesenseStatus.reindex.loading ? 'Reindexing All...' : '🔄 Full Reindex'}
</Button>
<Button
onClick={handleRecreateAllCollections}
disabled={typesenseStatus.reindex.loading || typesenseStatus.recreate.loading}
loading={typesenseStatus.recreate.loading}
variant="secondary"
className="flex-1"
>
{typesenseStatus.recreate.loading ? 'Recreating All...' : '🏗️ Recreate All Collections'}
</Button>
</div>
{/* Status Messages */}
{typesenseStatus.reindex.message && (
<div className={`text-sm p-3 rounded mb-3 ${
typesenseStatus.reindex.success
? 'bg-green-50 dark:bg-green-900/20 text-green-800 dark:text-green-200'
: 'bg-red-50 dark:bg-red-900/20 text-red-800 dark:text-red-200'
}`}>
{typesenseStatus.reindex.message}
</div>
)}
{typesenseStatus.recreate.message && (
<div className={`text-sm p-3 rounded mb-3 ${
typesenseStatus.recreate.success
? 'bg-green-50 dark:bg-green-900/20 text-green-800 dark:text-green-200'
: 'bg-red-50 dark:bg-red-900/20 text-red-800 dark:text-red-200'
}`}>
{typesenseStatus.recreate.message}
</div>
)}
</div>
<div className="text-sm theme-text bg-blue-50 dark:bg-blue-900/20 p-3 rounded-lg">
<p className="font-medium mb-1">When to use these tools:</p>
<ul className="text-xs space-y-1 ml-4">
<li>• <strong>Full Reindex:</strong> Refresh all search data while keeping existing schemas (fixes data sync issues)</li>
<li>• <strong>Recreate All Collections:</strong> Delete and rebuild all search indexes from scratch (fixes schema and structure issues)</li>
<li>• <strong>Operations run in parallel</strong> across all index types for better performance</li>
<li>• <strong>Reindex All:</strong> Refresh all search data while keeping existing schemas (fixes data sync issues)</li>
<li>• <strong>Recreate Indices:</strong> Delete and rebuild all search indexes from scratch (fixes schema and structure issues)</li>
</ul>
</div>
</div>

46
frontend/src/lib/api.ts
View File

@@ -179,15 +179,6 @@ export const storyApi = {
return response.data;
},
reindexTypesense: async (): Promise<{ success: boolean; message: string; count?: number; error?: string }> => {
const response = await api.post('/stories/reindex-typesense');
return response.data;
},
recreateTypesenseCollection: async (): Promise<{ success: boolean; message: string; count?: number; error?: string }> => {
const response = await api.post('/stories/recreate-typesense-collection');
return response.data;
},
checkDuplicate: async (title: string, authorName: string): Promise<{
hasDuplicates: boolean;
@@ -305,38 +296,6 @@ export const authorApi = {
await api.delete(`/authors/${id}/avatar`);
},
searchAuthorsTypesense: async (params?: {
q?: string;
page?: number;
size?: number;
sortBy?: string;
sortOrder?: string;
}): Promise<{
results: Author[];
totalHits: number;
page: number;
perPage: number;
query: string;
searchTimeMs: number;
}> => {
const response = await api.get('/authors/search-typesense', { params });
return response.data;
},
reindexTypesense: async (): Promise<{ success: boolean; message: string; count?: number; error?: string }> => {
const response = await api.post('/authors/reindex-typesense');
return response.data;
},
recreateTypesenseCollection: async (): Promise<{ success: boolean; message: string; count?: number; error?: string }> => {
const response = await api.post('/authors/recreate-typesense-collection');
return response.data;
},
getTypesenseSchema: async (): Promise<{ success: boolean; schema?: any; error?: string }> => {
const response = await api.get('/authors/typesense-schema');
return response.data;
},
};
// Tag endpoints
@@ -617,7 +576,6 @@ export const searchAdminApi = {
getStatus: async (): Promise<{
primaryEngine: string;
dualWrite: boolean;
typesenseAvailable: boolean;
openSearchAvailable: boolean;
}> => {
const response = await api.get('/admin/search/status');
@@ -647,10 +605,6 @@ export const searchAdminApi = {
return response.data;
},
switchToTypesense: async (): Promise<{ message: string }> => {
const response = await api.post('/admin/search/switch/typesense');
return response.data;
},
// Emergency rollback
emergencyRollback: async (): Promise<{ message: string }> => {