3 Commits

Author SHA1 Message Date
Stefan Hardegger
590e2590d6 DB Backup and Restore 2025-07-31 07:12:12 +02:00
Stefan Hardegger
57859d7a84 Reading Progress 2025-07-29 14:53:44 +02:00
Stefan Hardegger
5746001c4a Bugfixes 2025-07-29 11:02:46 +02:00
19 changed files with 952 additions and 24 deletions

View File

@@ -131,9 +131,12 @@ cd backend
### 🎨 **User Experience**
- **Dark/Light Mode**: Automatic theme switching with system preference detection
- **Responsive Design**: Optimized for desktop, tablet, and mobile
- **Reading Mode**: Distraction-free reading interface
- **Reading Mode**: Distraction-free reading interface with real-time progress tracking
- **Reading Position Memory**: Character-based position tracking with smooth auto-scroll restoration
- **Smart Tag Filtering**: Dynamic tag filters with live story counts in library view
- **Keyboard Navigation**: Full keyboard accessibility
- **Rich Text Editor**: Visual and source editing modes for story content
- **Progress Indicators**: Visual reading progress bars and completion tracking
### 🔒 **Security & Administration**
- **JWT Authentication**: Secure token-based authentication
@@ -170,9 +173,9 @@ StoryCove uses a PostgreSQL database with the following core entities:
### **Stories**
- **Primary Key**: UUID
- **Fields**: title, summary, description, content_html, content_plain, source_url, word_count, rating, volume, cover_path
- **Fields**: title, summary, description, content_html, content_plain, source_url, word_count, rating, volume, cover_path, reading_position, last_read_at
- **Relationships**: Many-to-One with Author, Many-to-One with Series, Many-to-Many with Tags
- **Features**: Automatic word count calculation, HTML sanitization, plain text extraction
- **Features**: Automatic word count calculation, HTML sanitization, plain text extraction, reading progress tracking
### **Authors**
- **Primary Key**: UUID
@@ -214,7 +217,8 @@ StoryCove uses a PostgreSQL database with the following core entities:
- `POST /{id}/rating` - Set story rating
- `POST /{id}/tags/{tagId}` - Add tag to story
- `DELETE /{id}/tags/{tagId}` - Remove tag from story
- `GET /search` - Search stories (Typesense)
- `POST /{id}/reading-progress` - Update reading position
- `GET /search` - Search stories (Typesense with faceting)
- `GET /search/suggestions` - Get search suggestions
- `GET /author/{authorId}` - Stories by author
- `GET /series/{seriesId}` - Stories in series
@@ -295,6 +299,7 @@ All API endpoints use JSON format with proper HTTP status codes:
- **Backend**: Spring Boot 3, Java 21, PostgreSQL, Typesense
- **Infrastructure**: Docker, Docker Compose, Nginx
- **Security**: JWT authentication, HTML sanitization, CORS
- **Search**: Typesense with faceting and full-text search capabilities
### **Local Development Setup**

View File

@@ -56,7 +56,10 @@ public class SecurityConfig {
@Bean
public CorsConfigurationSource corsConfigurationSource() {
CorsConfiguration configuration = new CorsConfiguration();
configuration.setAllowedOriginPatterns(Arrays.asList(allowedOrigins.split(",")));
List<String> origins = Arrays.stream(allowedOrigins.split(","))
.map(String::trim)
.toList();
configuration.setAllowedOriginPatterns(origins);
configuration.setAllowedMethods(Arrays.asList("GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"));
configuration.setAllowedHeaders(List.of("*"));
configuration.setAllowCredentials(true);

View File

@@ -0,0 +1,84 @@
package com.storycove.controller;
import com.storycove.service.DatabaseManagementService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Map;
@RestController
@RequestMapping("/api/database")
public class DatabaseController {
@Autowired
private DatabaseManagementService databaseManagementService;
@PostMapping("/backup")
public ResponseEntity<Resource> backupDatabase() {
try {
Resource backup = databaseManagementService.createBackup();
String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss"));
String filename = "storycove_backup_" + timestamp + ".sql";
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\"")
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.body(backup);
} catch (Exception e) {
throw new RuntimeException("Failed to create database backup: " + e.getMessage(), e);
}
}
@PostMapping("/restore")
public ResponseEntity<Map<String, Object>> restoreDatabase(@RequestParam("file") MultipartFile file) {
try {
if (file.isEmpty()) {
return ResponseEntity.badRequest()
.body(Map.of("success", false, "message", "No file uploaded"));
}
if (!file.getOriginalFilename().endsWith(".sql")) {
return ResponseEntity.badRequest()
.body(Map.of("success", false, "message", "Invalid file type. Please upload a .sql file"));
}
databaseManagementService.restoreFromBackup(file.getInputStream());
return ResponseEntity.ok(Map.of(
"success", true,
"message", "Database restored successfully from " + file.getOriginalFilename()
));
} catch (IOException e) {
return ResponseEntity.internalServerError()
.body(Map.of("success", false, "message", "Failed to read backup file: " + e.getMessage()));
} catch (Exception e) {
return ResponseEntity.internalServerError()
.body(Map.of("success", false, "message", "Failed to restore database: " + e.getMessage()));
}
}
@PostMapping("/clear")
public ResponseEntity<Map<String, Object>> clearDatabase() {
try {
int deletedRecords = databaseManagementService.clearAllData();
return ResponseEntity.ok(Map.of(
"success", true,
"message", "Database cleared successfully",
"deletedRecords", deletedRecords
));
} catch (Exception e) {
return ResponseEntity.internalServerError()
.body(Map.of("success", false, "message", "Failed to clear database: " + e.getMessage()));
}
}
}

View File

@@ -4,6 +4,7 @@ import com.storycove.entity.Author;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
@@ -52,4 +53,5 @@ public interface AuthorRepository extends JpaRepository<Author, UUID> {
@Query(value = "SELECT author_rating FROM authors WHERE id = :id", nativeQuery = true)
Integer findAuthorRatingById(@Param("id") UUID id);
}

View File

@@ -2,6 +2,7 @@ package com.storycove.repository;
import com.storycove.entity.Collection;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
@@ -51,4 +52,5 @@ public interface CollectionRepository extends JpaRepository<Collection, UUID> {
*/
@Query("SELECT c FROM Collection c LEFT JOIN FETCH c.tags ORDER BY c.updatedAt DESC")
List<Collection> findAllWithTags();
}

View File

@@ -7,6 +7,7 @@ import com.storycove.entity.Tag;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
@@ -117,4 +118,5 @@ public interface StoryRepository extends JpaRepository<Story, UUID> {
@Query("SELECT s FROM Story s WHERE UPPER(s.title) = UPPER(:title) AND UPPER(s.author.name) = UPPER(:authorName)")
List<Story> findByTitleAndAuthorNameIgnoreCase(@Param("title") String title, @Param("authorName") String authorName);
}

View File

@@ -0,0 +1,196 @@
package com.storycove.service;
import com.storycove.entity.*;
import com.storycove.repository.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.sql.DataSource;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.sql.*;
import java.util.Arrays;
import java.util.List;
@Service
public class DatabaseManagementService {
@Autowired
private DataSource dataSource;
@Autowired
private StoryRepository storyRepository;
@Autowired
private AuthorRepository authorRepository;
@Autowired
private SeriesRepository seriesRepository;
@Autowired
private TagRepository tagRepository;
@Autowired
private CollectionRepository collectionRepository;
@Autowired
private TypesenseService typesenseService;
public Resource createBackup() throws SQLException, IOException {
StringBuilder sqlDump = new StringBuilder();
try (Connection connection = dataSource.getConnection()) {
// Add header
sqlDump.append("-- StoryCove Database Backup\n");
sqlDump.append("-- Generated at: ").append(new java.util.Date()).append("\n\n");
// Disable foreign key checks during restore (PostgreSQL syntax)
sqlDump.append("SET session_replication_role = replica;\n\n");
// List of tables in dependency order (children first for deletion, parents first for insertion)
List<String> tables = Arrays.asList(
"story_tags", "author_urls", "collection_stories",
"stories", "authors", "series", "tags", "collections"
);
// Generate TRUNCATE statements for each table (assuming tables already exist)
for (String tableName : tables) {
sqlDump.append("-- Table: ").append(tableName).append("\n");
sqlDump.append("TRUNCATE TABLE \"").append(tableName).append("\" CASCADE;\n");
// Get table data
try (PreparedStatement stmt = connection.prepareStatement("SELECT * FROM \"" + tableName + "\"");
ResultSet rs = stmt.executeQuery()) {
ResultSetMetaData metaData = rs.getMetaData();
int columnCount = metaData.getColumnCount();
// Build column names for INSERT statement
StringBuilder columnNames = new StringBuilder();
for (int i = 1; i <= columnCount; i++) {
if (i > 1) columnNames.append(", ");
columnNames.append("\"").append(metaData.getColumnName(i)).append("\"");
}
while (rs.next()) {
sqlDump.append("INSERT INTO \"").append(tableName).append("\" (")
.append(columnNames).append(") VALUES (");
for (int i = 1; i <= columnCount; i++) {
if (i > 1) sqlDump.append(", ");
Object value = rs.getObject(i);
if (value == null) {
sqlDump.append("NULL");
} else if (value instanceof String || value instanceof Timestamp ||
value instanceof java.util.UUID) {
// Escape single quotes and wrap in quotes
String escapedValue = value.toString().replace("'", "''");
sqlDump.append("'").append(escapedValue).append("'");
} else if (value instanceof Boolean) {
sqlDump.append(((Boolean) value) ? "true" : "false");
} else {
sqlDump.append(value.toString());
}
}
sqlDump.append(");\n");
}
}
sqlDump.append("\n");
}
// Re-enable foreign key checks (PostgreSQL syntax)
sqlDump.append("SET session_replication_role = DEFAULT;\n");
}
byte[] backupData = sqlDump.toString().getBytes(StandardCharsets.UTF_8);
return new ByteArrayResource(backupData);
}
@Transactional
public void restoreFromBackup(InputStream backupStream) throws IOException, SQLException {
// Read the SQL file
StringBuilder sqlContent = new StringBuilder();
try (BufferedReader reader = new BufferedReader(new InputStreamReader(backupStream, StandardCharsets.UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
// Skip comments and empty lines
if (!line.trim().startsWith("--") && !line.trim().isEmpty()) {
sqlContent.append(line).append("\n");
}
}
}
// Execute the SQL statements
try (Connection connection = dataSource.getConnection()) {
connection.setAutoCommit(false);
try {
// Split by semicolon and execute each statement
String[] statements = sqlContent.toString().split(";");
for (String statement : statements) {
String trimmedStatement = statement.trim();
if (!trimmedStatement.isEmpty()) {
try (PreparedStatement stmt = connection.prepareStatement(trimmedStatement)) {
stmt.execute();
}
}
}
connection.commit();
// Reindex search after successful restore
try {
// Note: Manual reindexing may be needed after restore
// The search indices will need to be rebuilt through the settings page
} catch (Exception e) {
// Log the error but don't fail the restore
System.err.println("Warning: Search indices may need manual reindexing after restore");
}
} catch (SQLException e) {
connection.rollback();
throw e;
} finally {
connection.setAutoCommit(true);
}
}
}
@Transactional
public int clearAllData() {
int totalDeleted = 0;
try {
// Count entities before deletion
int collectionCount = (int) collectionRepository.count();
int storyCount = (int) storyRepository.count();
int authorCount = (int) authorRepository.count();
int seriesCount = (int) seriesRepository.count();
int tagCount = (int) tagRepository.count();
// Delete main entities (cascade will handle junction tables)
collectionRepository.deleteAll();
storyRepository.deleteAll();
authorRepository.deleteAll();
seriesRepository.deleteAll();
tagRepository.deleteAll();
totalDeleted = collectionCount + storyCount + authorCount + seriesCount + tagCount;
// Note: Search indexes will need to be manually recreated after clearing
// Use the settings page to recreate Typesense collections after clearing the database
} catch (Exception e) {
throw new RuntimeException("Failed to clear database: " + e.getMessage(), e);
}
return totalDeleted;
}
}

View File

@@ -77,7 +77,7 @@ public class TypesenseService {
new Field().name("authorName").type("string").facet(true).sort(true),
new Field().name("seriesId").type("string").facet(true).optional(true),
new Field().name("seriesName").type("string").facet(true).sort(true).optional(true),
new Field().name("tagNames").type("string[]").facet(true).optional(true),
new Field().name("tagNames").type("string[]").facet(true),
new Field().name("rating").type("int32").facet(true).sort(true).optional(true),
new Field().name("wordCount").type("int32").facet(true).sort(true).optional(true),
new Field().name("volume").type("int32").facet(true).sort(true).optional(true),
@@ -232,6 +232,9 @@ public class TypesenseService {
.maxFacetValues(100)
.sortBy(buildSortParameter(normalizedQuery, sortBy, sortDir));
logger.debug("Typesense search parameters - facetBy: {}, maxFacetValues: {}",
searchParameters.getFacetBy(), searchParameters.getMaxFacetValues());
// Add filters
List<String> filterConditions = new ArrayList<>();
@@ -269,6 +272,7 @@ public class TypesenseService {
.documents()
.search(searchParameters);
logger.debug("Search result facet counts: {}", searchResult.getFacetCounts());
List<StorySearchDto> results = convertSearchResult(searchResult);
Map<String, List<FacetCountDto>> facets = processFacetCounts(searchResult);
@@ -375,7 +379,10 @@ public class TypesenseService {
.map(tag -> tag.getName())
.collect(Collectors.toList());
document.put("tagNames", tagNames);
logger.debug("Story '{}' has {} tags: {}", story.getTitle(), tagNames.size(), tagNames);
} else {
document.put("tagNames", new ArrayList<>());
logger.debug("Story '{}' has no tags, setting empty array", story.getTitle());
}
document.put("rating", story.getRating() != null ? story.getRating() : 0);
@@ -406,15 +413,34 @@ public class TypesenseService {
List<FacetCountDto> facetValues = new ArrayList<>();
if (facetCounts.getCounts() != null) {
for (Object countObj : facetCounts.getCounts()) {
if (countObj instanceof Map) {
Map<String, Object> countMap = (Map<String, Object>) countObj;
String value = (String) countMap.get("value");
Integer count = (Integer) countMap.get("count");
if (countObj instanceof org.typesense.model.FacetCountsCounts) {
org.typesense.model.FacetCountsCounts facetCount = (org.typesense.model.FacetCountsCounts) countObj;
String value = facetCount.getValue();
Integer count = facetCount.getCount();
if (value != null && count != null && count > 0) {
facetValues.add(new FacetCountDto(value, count));
}
} else if (countObj instanceof Map) {
// Fallback for Map-based responses
Map<String, Object> countMap = (Map<String, Object>) countObj;
String value = (String) countMap.get("value");
Object countValue = countMap.get("count");
if (value != null && countValue != null) {
Integer count = null;
if (countValue instanceof Integer) {
count = (Integer) countValue;
} else if (countValue instanceof Number) {
count = ((Number) countValue).intValue();
}
if (count != null && count > 0) {
facetValues.add(new FacetCountDto(value, count));
}
}
}
}
}
@@ -432,6 +458,12 @@ public class TypesenseService {
}
}
// DEBUG: Log final facet processing results
logger.info("FACET DEBUG: Final facetMap contents: {}", facetMap);
if (facetMap.isEmpty()) {
logger.info("FACET DEBUG: No facets were processed - investigating why");
}
return facetMap;
}

View File

@@ -57,6 +57,7 @@ export default function LibraryPage() {
tags: selectedTags.length > 0 ? selectedTags : undefined,
sortBy: sortOption,
sortDir: sortDirection,
facetBy: ['tagNames'], // Request tag facets for the filter UI
});
const currentStories = result?.results || [];

View File

@@ -4,7 +4,7 @@ import { useState, useEffect } from 'react';
import AppLayout from '../../components/layout/AppLayout';
import { useTheme } from '../../lib/theme';
import Button from '../../components/ui/Button';
import { storyApi, authorApi } from '../../lib/api';
import { storyApi, authorApi, databaseApi } from '../../lib/api';
type FontFamily = 'serif' | 'sans' | 'mono';
type FontSize = 'small' | 'medium' | 'large' | 'extra-large';
@@ -39,6 +39,15 @@ export default function SettingsPage() {
});
const [authorsSchema, setAuthorsSchema] = useState<any>(null);
const [showSchema, setShowSchema] = useState(false);
const [databaseStatus, setDatabaseStatus] = useState<{
backup: { loading: boolean; message: string; success?: boolean };
restore: { loading: boolean; message: string; success?: boolean };
clear: { loading: boolean; message: string; success?: boolean };
}>({
backup: { loading: false, message: '' },
restore: { loading: false, message: '' },
clear: { loading: false, message: '' }
});
// Load settings from localStorage on mount
useEffect(() => {
@@ -157,6 +166,146 @@ export default function SettingsPage() {
}
};
const handleDatabaseBackup = async () => {
setDatabaseStatus(prev => ({
...prev,
backup: { loading: true, message: 'Creating backup...', success: undefined }
}));
try {
const backupBlob = await databaseApi.backup();
// Create download link
const url = window.URL.createObjectURL(backupBlob);
const link = document.createElement('a');
link.href = url;
const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
link.download = `storycove_backup_${timestamp}.sql`;
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
window.URL.revokeObjectURL(url);
setDatabaseStatus(prev => ({
...prev,
backup: { loading: false, message: 'Backup downloaded successfully', success: true }
}));
} catch (error: any) {
setDatabaseStatus(prev => ({
...prev,
backup: { loading: false, message: error.message || 'Backup failed', success: false }
}));
}
// Clear message after 5 seconds
setTimeout(() => {
setDatabaseStatus(prev => ({
...prev,
backup: { loading: false, message: '', success: undefined }
}));
}, 5000);
};
const handleDatabaseRestore = async (event: React.ChangeEvent<HTMLInputElement>) => {
const file = event.target.files?.[0];
if (!file) return;
// Reset the input so the same file can be selected again
event.target.value = '';
if (!file.name.endsWith('.sql')) {
setDatabaseStatus(prev => ({
...prev,
restore: { loading: false, message: 'Please select a .sql file', success: false }
}));
return;
}
const confirmed = window.confirm(
'Are you sure you want to restore the database? This will PERMANENTLY DELETE all current data and replace it with the backup data. This action cannot be undone!'
);
if (!confirmed) return;
setDatabaseStatus(prev => ({
...prev,
restore: { loading: true, message: 'Restoring database...', success: undefined }
}));
try {
const result = await databaseApi.restore(file);
setDatabaseStatus(prev => ({
...prev,
restore: {
loading: false,
message: result.success ? result.message : result.message,
success: result.success
}
}));
} catch (error: any) {
setDatabaseStatus(prev => ({
...prev,
restore: { loading: false, message: error.message || 'Restore failed', success: false }
}));
}
// Clear message after 10 seconds for restore (longer because it's important)
setTimeout(() => {
setDatabaseStatus(prev => ({
...prev,
restore: { loading: false, message: '', success: undefined }
}));
}, 10000);
};
const handleDatabaseClear = async () => {
const confirmed = window.confirm(
'Are you ABSOLUTELY SURE you want to clear the entire database? This will PERMANENTLY DELETE ALL stories, authors, series, tags, and collections. This action cannot be undone!'
);
if (!confirmed) return;
const doubleConfirmed = window.confirm(
'This is your final warning! Clicking OK will DELETE EVERYTHING in your StoryCove database. Are you completely certain you want to proceed?'
);
if (!doubleConfirmed) return;
setDatabaseStatus(prev => ({
...prev,
clear: { loading: true, message: 'Clearing database...', success: undefined }
}));
try {
const result = await databaseApi.clear();
setDatabaseStatus(prev => ({
...prev,
clear: {
loading: false,
message: result.success
? `Database cleared successfully. Deleted ${result.deletedRecords} records.`
: result.message,
success: result.success
}
}));
} catch (error: any) {
setDatabaseStatus(prev => ({
...prev,
clear: { loading: false, message: error.message || 'Clear operation failed', success: false }
}));
}
// Clear message after 10 seconds for clear (longer because it's important)
setTimeout(() => {
setDatabaseStatus(prev => ({
...prev,
clear: { loading: false, message: '', success: undefined }
}));
}, 10000);
};
return (
<AppLayout>
<div className="max-w-2xl mx-auto space-y-8">
@@ -463,6 +612,110 @@ export default function SettingsPage() {
</div>
</div>
{/* Database Management */}
<div className="theme-card theme-shadow rounded-lg p-6">
<h2 className="text-xl font-semibold theme-header mb-4">Database Management</h2>
<p className="theme-text mb-6">
Backup, restore, or clear your StoryCove database. These are powerful tools - use with caution!
</p>
<div className="space-y-6">
{/* Backup Section */}
<div className="border theme-border rounded-lg p-4">
<h3 className="text-lg font-semibold theme-header mb-3">💾 Backup Database</h3>
<p className="text-sm theme-text mb-3">
Download a complete backup of your database as an SQL file. This includes all stories, authors, tags, series, and collections.
</p>
<Button
onClick={handleDatabaseBackup}
disabled={databaseStatus.backup.loading}
loading={databaseStatus.backup.loading}
variant="primary"
className="w-full sm:w-auto"
>
{databaseStatus.backup.loading ? 'Creating Backup...' : 'Download Backup'}
</Button>
{databaseStatus.backup.message && (
<div className={`text-sm p-2 rounded mt-3 ${
databaseStatus.backup.success
? 'bg-green-50 dark:bg-green-900/20 text-green-800 dark:text-green-200'
: 'bg-red-50 dark:bg-red-900/20 text-red-800 dark:text-red-200'
}`}>
{databaseStatus.backup.message}
</div>
)}
</div>
{/* Restore Section */}
<div className="border theme-border rounded-lg p-4 border-orange-200 dark:border-orange-800">
<h3 className="text-lg font-semibold theme-header mb-3">📥 Restore Database</h3>
<p className="text-sm theme-text mb-3">
<strong className="text-orange-600 dark:text-orange-400">⚠️ Warning:</strong> This will completely replace your current database with the backup file. All existing data will be permanently deleted.
</p>
<div className="flex items-center gap-3">
<input
type="file"
accept=".sql"
onChange={handleDatabaseRestore}
disabled={databaseStatus.restore.loading}
className="flex-1 text-sm theme-text file:mr-4 file:py-2 file:px-4 file:rounded-lg file:border-0 file:text-sm file:font-medium file:theme-accent-bg file:text-white hover:file:bg-opacity-90 file:cursor-pointer"
/>
</div>
{databaseStatus.restore.message && (
<div className={`text-sm p-2 rounded mt-3 ${
databaseStatus.restore.success
? 'bg-green-50 dark:bg-green-900/20 text-green-800 dark:text-green-200'
: 'bg-red-50 dark:bg-red-900/20 text-red-800 dark:text-red-200'
}`}>
{databaseStatus.restore.message}
</div>
)}
{databaseStatus.restore.loading && (
<div className="text-sm theme-text mt-3 flex items-center gap-2">
<div className="animate-spin w-4 h-4 border-2 border-blue-500 border-t-transparent rounded-full"></div>
Restoring database...
</div>
)}
</div>
{/* Clear Section */}
<div className="border theme-border rounded-lg p-4 border-red-200 dark:border-red-800">
<h3 className="text-lg font-semibold theme-header mb-3">🗑️ Clear Database</h3>
<p className="text-sm theme-text mb-3">
<strong className="text-red-600 dark:text-red-400">⚠️ Danger Zone:</strong> This will permanently delete ALL data from your database. Stories, authors, tags, series, and collections will be completely removed. This action cannot be undone!
</p>
<Button
onClick={handleDatabaseClear}
disabled={databaseStatus.clear.loading}
loading={databaseStatus.clear.loading}
variant="secondary"
className="w-full sm:w-auto bg-red-600 hover:bg-red-700 text-white border-red-600"
>
{databaseStatus.clear.loading ? 'Clearing Database...' : 'Clear All Data'}
</Button>
{databaseStatus.clear.message && (
<div className={`text-sm p-2 rounded mt-3 ${
databaseStatus.clear.success
? 'bg-green-50 dark:bg-green-900/20 text-green-800 dark:text-green-200'
: 'bg-red-50 dark:bg-red-900/20 text-red-800 dark:text-red-200'
}`}>
{databaseStatus.clear.message}
</div>
)}
</div>
<div className="text-sm theme-text bg-blue-50 dark:bg-blue-900/20 p-3 rounded-lg">
<p className="font-medium mb-1">💡 Best Practices:</p>
<ul className="text-xs space-y-1 ml-4">
<li>• <strong>Always backup</strong> before performing restore or clear operations</li>
<li>• <strong>Test restores</strong> in a development environment when possible</li>
<li>• <strong>Store backups safely</strong> in multiple locations for important data</li>
<li>• <strong>Verify backup files</strong> are complete before relying on them</li>
</ul>
</div>
</div>
</div>
{/* Actions */}
<div className="flex justify-end gap-4">
<Button

View File

@@ -1,6 +1,6 @@
'use client';
import { useState, useEffect } from 'react';
import { useState, useEffect, useRef, useCallback } from 'react';
import { useParams, useRouter } from 'next/navigation';
import Link from 'next/link';
import { storyApi, seriesApi } from '../../../lib/api';
@@ -19,9 +19,85 @@ export default function StoryReadingPage() {
const [error, setError] = useState<string | null>(null);
const [readingProgress, setReadingProgress] = useState(0);
const [sanitizedContent, setSanitizedContent] = useState<string>('');
const [hasScrolledToPosition, setHasScrolledToPosition] = useState(false);
const contentRef = useRef<HTMLDivElement>(null);
const saveTimeoutRef = useRef<NodeJS.Timeout | null>(null);
const storyId = params.id as string;
// Convert scroll position to approximate character position in the content
const getCharacterPositionFromScroll = useCallback((): number => {
if (!contentRef.current || !story) return 0;
const content = contentRef.current;
const scrolled = window.scrollY;
const contentTop = content.offsetTop;
const contentHeight = content.scrollHeight;
const windowHeight = window.innerHeight;
// Calculate how far through the content we are (0-1)
const scrollRatio = Math.min(1, Math.max(0,
(scrolled - contentTop + windowHeight * 0.3) / contentHeight
));
// Convert to character position in the plain text content
const textLength = story.contentPlain?.length || story.contentHtml.length;
return Math.floor(scrollRatio * textLength);
}, [story]);
// Convert character position back to scroll position for auto-scroll
const scrollToCharacterPosition = useCallback((position: number) => {
if (!contentRef.current || !story || hasScrolledToPosition) return;
const textLength = story.contentPlain?.length || story.contentHtml.length;
if (textLength === 0 || position === 0) return;
const ratio = position / textLength;
const content = contentRef.current;
const contentTop = content.offsetTop;
const contentHeight = content.scrollHeight;
const windowHeight = window.innerHeight;
// Calculate target scroll position
const targetScroll = contentTop + (ratio * contentHeight) - (windowHeight * 0.3);
// Smooth scroll to position
window.scrollTo({
top: Math.max(0, targetScroll),
behavior: 'smooth'
});
setHasScrolledToPosition(true);
}, [story, hasScrolledToPosition]);
// Debounced function to save reading position
const saveReadingPosition = useCallback(async (position: number) => {
if (!story || position === story.readingPosition) {
console.log('Skipping save - no story or position unchanged:', { story: !!story, position, current: story?.readingPosition });
return;
}
console.log('Saving reading position:', position, 'for story:', story.id);
try {
const updatedStory = await storyApi.updateReadingProgress(story.id, position);
console.log('Reading position saved successfully, updated story:', updatedStory.readingPosition);
setStory(prev => prev ? { ...prev, readingPosition: position, lastReadAt: updatedStory.lastReadAt } : null);
} catch (error) {
console.error('Failed to save reading position:', error);
}
}, [story]);
// Debounced version of saveReadingPosition
const debouncedSavePosition = useCallback((position: number) => {
if (saveTimeoutRef.current) {
clearTimeout(saveTimeoutRef.current);
}
saveTimeoutRef.current = setTimeout(() => {
saveReadingPosition(position);
}, 2000); // Save after 2 seconds of no scrolling
}, [saveReadingPosition]);
useEffect(() => {
const loadStory = async () => {
try {
@@ -57,7 +133,27 @@ export default function StoryReadingPage() {
}
}, [storyId]);
// Track reading progress
// Auto-scroll to saved reading position when story content is loaded
useEffect(() => {
if (story && sanitizedContent && !hasScrolledToPosition) {
// Use a small delay to ensure content is rendered
const timeout = setTimeout(() => {
console.log('Initializing reading position tracking, saved position:', story.readingPosition);
if (story.readingPosition && story.readingPosition > 0) {
console.log('Auto-scrolling to saved position:', story.readingPosition);
scrollToCharacterPosition(story.readingPosition);
} else {
// Even if there's no saved position, mark as ready for tracking
console.log('No saved position, starting fresh tracking');
setHasScrolledToPosition(true);
}
}, 500);
return () => clearTimeout(timeout);
}
}, [story, sanitizedContent, scrollToCharacterPosition, hasScrolledToPosition]);
// Track reading progress and save position
useEffect(() => {
const handleScroll = () => {
const article = document.querySelector('[data-reading-content]') as HTMLElement;
@@ -72,12 +168,27 @@ export default function StoryReadingPage() {
));
setReadingProgress(progress);
// Save reading position (debounced)
if (hasScrolledToPosition) { // Only save after initial auto-scroll
const characterPosition = getCharacterPositionFromScroll();
console.log('Scroll detected, character position:', characterPosition);
debouncedSavePosition(characterPosition);
} else {
console.log('Scroll detected but not ready for tracking yet');
}
}
};
window.addEventListener('scroll', handleScroll);
return () => window.removeEventListener('scroll', handleScroll);
}, [story]);
return () => {
window.removeEventListener('scroll', handleScroll);
// Clean up timeout on unmount
if (saveTimeoutRef.current) {
clearTimeout(saveTimeoutRef.current);
}
};
}, [story, hasScrolledToPosition, getCharacterPositionFromScroll, debouncedSavePosition]);
const handleRatingUpdate = async (newRating: number) => {
if (!story) return;
@@ -229,6 +340,7 @@ export default function StoryReadingPage() {
{/* Story Content */}
<div
ref={contentRef}
className="reading-content"
dangerouslySetInnerHTML={{ __html: sanitizedContent }}
/>

View File

@@ -1,6 +1,8 @@
'use client';
import { useState, useEffect, useRef, useCallback } from 'react';
import { StoryWithCollectionContext } from '../../types/api';
import { storyApi } from '../../lib/api';
import Button from '../ui/Button';
import Link from 'next/link';
@@ -16,6 +18,120 @@ export default function CollectionReadingView({
onBackToCollection
}: CollectionReadingViewProps) {
const { story, collection } = data;
const [hasScrolledToPosition, setHasScrolledToPosition] = useState(false);
const contentRef = useRef<HTMLDivElement>(null);
const saveTimeoutRef = useRef<NodeJS.Timeout | null>(null);
// Convert scroll position to approximate character position in the content
const getCharacterPositionFromScroll = useCallback((): number => {
if (!contentRef.current || !story) return 0;
const content = contentRef.current;
const scrolled = window.scrollY;
const contentTop = content.offsetTop;
const contentHeight = content.scrollHeight;
const windowHeight = window.innerHeight;
// Calculate how far through the content we are (0-1)
const scrollRatio = Math.min(1, Math.max(0,
(scrolled - contentTop + windowHeight * 0.3) / contentHeight
));
// Convert to character position in the plain text content
const textLength = story.contentPlain?.length || story.contentHtml.length;
return Math.floor(scrollRatio * textLength);
}, [story]);
// Convert character position back to scroll position for auto-scroll
const scrollToCharacterPosition = useCallback((position: number) => {
if (!contentRef.current || !story || hasScrolledToPosition) return;
const textLength = story.contentPlain?.length || story.contentHtml.length;
if (textLength === 0 || position === 0) return;
const ratio = position / textLength;
const content = contentRef.current;
const contentTop = content.offsetTop;
const contentHeight = content.scrollHeight;
const windowHeight = window.innerHeight;
// Calculate target scroll position
const targetScroll = contentTop + (ratio * contentHeight) - (windowHeight * 0.3);
// Smooth scroll to position
window.scrollTo({
top: Math.max(0, targetScroll),
behavior: 'smooth'
});
setHasScrolledToPosition(true);
}, [story, hasScrolledToPosition]);
// Debounced function to save reading position
const saveReadingPosition = useCallback(async (position: number) => {
if (!story || position === story.readingPosition) {
console.log('Collection view - skipping save - no story or position unchanged:', { story: !!story, position, current: story?.readingPosition });
return;
}
console.log('Collection view - saving reading position:', position, 'for story:', story.id);
try {
await storyApi.updateReadingProgress(story.id, position);
console.log('Collection view - reading position saved successfully');
} catch (error) {
console.error('Collection view - failed to save reading position:', error);
}
}, [story]);
// Debounced version of saveReadingPosition
const debouncedSavePosition = useCallback((position: number) => {
if (saveTimeoutRef.current) {
clearTimeout(saveTimeoutRef.current);
}
saveTimeoutRef.current = setTimeout(() => {
saveReadingPosition(position);
}, 2000);
}, [saveReadingPosition]);
// Auto-scroll to saved reading position when story content is loaded
useEffect(() => {
if (story && !hasScrolledToPosition) {
const timeout = setTimeout(() => {
console.log('Collection view - initializing reading position tracking, saved position:', story.readingPosition);
if (story.readingPosition && story.readingPosition > 0) {
console.log('Collection view - auto-scrolling to saved position:', story.readingPosition);
scrollToCharacterPosition(story.readingPosition);
} else {
console.log('Collection view - no saved position, starting fresh tracking');
setHasScrolledToPosition(true);
}
}, 500);
return () => clearTimeout(timeout);
}
}, [story, scrollToCharacterPosition, hasScrolledToPosition]);
// Track reading progress and save position
useEffect(() => {
const handleScroll = () => {
if (hasScrolledToPosition) {
const characterPosition = getCharacterPositionFromScroll();
console.log('Collection view - scroll detected, character position:', characterPosition);
debouncedSavePosition(characterPosition);
} else {
console.log('Collection view - scroll detected but not ready for tracking yet');
}
};
window.addEventListener('scroll', handleScroll);
return () => {
window.removeEventListener('scroll', handleScroll);
if (saveTimeoutRef.current) {
clearTimeout(saveTimeoutRef.current);
}
};
}, [hasScrolledToPosition, getCharacterPositionFromScroll, debouncedSavePosition]);
const handlePrevious = () => {
if (collection.previousStoryId) {
@@ -180,6 +296,7 @@ export default function CollectionReadingView({
{/* Story Content */}
<div className="theme-card p-8">
<div
ref={contentRef}
className="prose prose-lg max-w-none theme-text"
dangerouslySetInnerHTML={{ __html: story.contentHtml }}
/>

View File

@@ -133,6 +133,11 @@ export const storyApi = {
updateRating: async (id: string, rating: number): Promise<void> => {
await api.post(`/stories/${id}/rating`, { rating });
},
updateReadingProgress: async (id: string, position: number): Promise<Story> => {
const response = await api.post(`/stories/${id}/reading-progress`, { position });
return response.data;
},
uploadCover: async (id: string, coverImage: File): Promise<{ imagePath: string }> => {
const formData = new FormData();
@@ -314,6 +319,7 @@ export const searchApi = {
maxRating?: number;
sortBy?: string;
sortDir?: string;
facetBy?: string[];
}): Promise<SearchResult> => {
// Create URLSearchParams to properly handle array parameters
const searchParams = new URLSearchParams();
@@ -334,6 +340,9 @@ export const searchApi = {
if (params.tags && params.tags.length > 0) {
params.tags.forEach(tag => searchParams.append('tags', tag));
}
if (params.facetBy && params.facetBy.length > 0) {
params.facetBy.forEach(facet => searchParams.append('facetBy', facet));
}
const response = await api.get(`/stories/search?${searchParams.toString()}`);
return response.data;
@@ -489,6 +498,30 @@ export const collectionApi = {
},
};
// Database management endpoints
export const databaseApi = {
backup: async (): Promise<Blob> => {
const response = await api.post('/database/backup', {}, {
responseType: 'blob'
});
return response.data;
},
restore: async (file: File): Promise<{ success: boolean; message: string }> => {
const formData = new FormData();
formData.append('file', file);
const response = await api.post('/database/restore', formData, {
headers: { 'Content-Type': 'multipart/form-data' },
});
return response.data;
},
clear: async (): Promise<{ success: boolean; message: string; deletedRecords?: number }> => {
const response = await api.post('/database/clear');
return response.data;
},
};
// Image utility
export const getImageUrl = (path: string): string => {
if (!path) return '';

View File

@@ -12,7 +12,7 @@
"searchBefore": "</title>"
},
"content": {
"strategy": "text-blocks",
"strategy": "deviantart-content",
"minLength": 200,
"containerHints": ["journal", "literature", "story", "text", "content"],
"excludeSelectors": ["script", "style", "nav", "header", "footer", ".dev-page-sidebar"]

View File

@@ -17,6 +17,7 @@ import { UrlParser } from './utils/urlParser';
import {
extractByTextPattern,
extractTextBlocks,
extractDeviantArtContent,
extractHtmlBetween,
extractLinkText,
extractLinkWithPath,
@@ -246,6 +247,8 @@ export class StoryScraper {
return extractLinkWithPath($, strategy as any);
case 'text-blocks':
return extractTextBlocks($, strategy as any);
case 'deviantart-content':
return extractDeviantArtContent($, strategy as any);
case 'href-pattern':
return extractHrefPattern($, strategy as any);
case 'html-between':

View File

@@ -82,6 +82,58 @@ export function extractTextBlocks(
return largestBlock ? $(largestBlock.element).html() || '' : '';
}
export function extractDeviantArtContent(
$: cheerio.CheerioAPI,
config: TextBlockStrategy
): string {
// Remove excluded elements first
if (config.excludeSelectors) {
config.excludeSelectors.forEach(selector => {
$(selector).remove();
});
}
// DeviantArt has two main content structures:
// 1. Old format: <div class="text"> containing the full story
// 2. New format: <div class="_83r8m _2CKTq"> or similar classes containing multiple <p> elements
// Try the old format first (single text div)
const textDiv = $('.text');
if (textDiv.length > 0 && textDiv.text().trim().length >= (config.minLength || 200)) {
return textDiv.html() || '';
}
// Try the new format (multiple paragraphs in specific containers)
const newFormatSelectors = [
'div[class*="_83r8m"] p', // Main story content container
'div[class*="_2CKTq"] p', // Alternate story content container
'div[class*="journal"] p' // Generic journal container
];
for (const selector of newFormatSelectors) {
const paragraphs = $(selector);
if (paragraphs.length > 0) {
let totalText = '';
paragraphs.each((_, p) => {
totalText += $(p).text().trim();
});
// Check if this container has enough content
if (totalText.length >= (config.minLength || 200)) {
// Combine all paragraphs into a single HTML string
let combinedHtml = '';
paragraphs.each((_, p) => {
combinedHtml += $(p).prop('outerHTML') || '';
});
return combinedHtml;
}
}
}
// Fallback to the original text-blocks strategy
return extractTextBlocks($, config);
}
export function extractHtmlBetween(
html: string,
config: HtmlBetweenStrategy

View File

@@ -15,6 +15,8 @@ export interface Story {
coverPath?: string;
tags: Tag[];
tagNames?: string[] | null; // Used in search results
readingPosition?: number;
lastReadAt?: string;
createdAt: string;
updatedAt: string;
}

5
package.json Normal file
View File

@@ -0,0 +1,5 @@
{
"dependencies": {
"cheerio": "^1.1.2"
}
}

View File

@@ -69,6 +69,8 @@ CREATE TABLE stories (
volume INTEGER,
rating INTEGER CHECK (rating >= 1 AND rating <= 5),
cover_image_path VARCHAR(500), -- Phase 2: Consider storing base filename without size suffix
reading_position INTEGER DEFAULT 0, -- Character position for reading progress
last_read_at TIMESTAMP, -- Last time story was accessed
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (author_id) REFERENCES authors(id),
@@ -140,7 +142,7 @@ CREATE TABLE story_tags (
{"name": "summary", "type": "string", "optional": true},
{"name": "author_name", "type": "string"},
{"name": "content", "type": "string"},
{"name": "tags", "type": "string[]"},
{"name": "tagNames", "type": "string[]", "facet": true},
{"name": "series_name", "type": "string", "optional": true},
{"name": "word_count", "type": "int32"},
{"name": "rating", "type": "int32", "optional": true},
@@ -178,6 +180,7 @@ Query parameters:
- `tags` (string[]): Filter by tags
- `authorId` (uuid): Filter by author
- `seriesId` (uuid): Filter by series
- `facetBy` (string[]): Enable faceting for specified fields (e.g., ['tagNames'])
#### POST /api/stories
```json
@@ -223,6 +226,21 @@ Request:
}
```
#### POST /api/stories/{id}/reading-progress
```json
Request:
{
"position": 1250
}
Response:
{
"id": "uuid",
"readingPosition": 1250,
"lastReadAt": "2024-01-01T12:00:00Z"
}
```
### 4.3 Author Endpoints
#### GET /api/authors
@@ -300,7 +318,7 @@ Get all stories in a series ordered by volume
#### Story List View
- Grid/List toggle
- Search bar with real-time results
- Tag cloud for filtering
- Dynamic tag filtering with live story counts (faceted search)
- Sort options: Date added, Title, Author, Rating
- Pagination
- Cover image thumbnails in grid view
@@ -321,7 +339,9 @@ Get all stories in a series ordered by volume
- Clean, distraction-free interface
- Cover image display at the top (if available)
- Responsive typography
- Progress indicator
- Real-time reading progress indicator with visual progress bar
- Character-based position tracking with automatic saving
- Automatic scroll-to-position restoration on story reopen
- Navigation: Previous/Next in series
- Quick access to rate story
- Back to library button
@@ -468,6 +488,8 @@ Get all stories in a series ordered by volume
2. On story delete: Remove from index
3. Batch reindex endpoint for maintenance
4. Search includes: title, author, content, tags
5. Faceted search support for dynamic filtering
6. Tag facets provide real-time counts for library filtering
### 6.4 Security Considerations
@@ -567,11 +589,11 @@ APP_PASSWORD=application_password_here
## 9. Phase 2 Roadmap
### 9.1 URL Content Grabbing
### 9.1 URL Content Grabbing ✅ IMPLEMENTED
- Configurable scrapers for specific sites
- Site configuration stored in database
- Content extraction rules per site
- Image download and storage
- Site configuration stored in JSON files
- Content extraction rules per site (DeviantArt support added)
- Adaptive content extraction for varying HTML structures
### 9.2 Enhanced Image Processing & Optimization
- **Multi-size generation during upload**
@@ -623,7 +645,9 @@ APP_PASSWORD=application_password_here
- Search interface
### Milestone 4: Reading Experience (Week 6)
- Reading view implementation
- Reading view implementation with progress tracking
- Character-based reading position persistence
- Automatic position restoration
- Settings management
- Rating system