diff --git a/CLEANUP_CHECKLIST.md b/CLEANUP_CHECKLIST.md new file mode 100644 index 0000000..6b7c4ad --- /dev/null +++ b/CLEANUP_CHECKLIST.md @@ -0,0 +1,137 @@ +# Search Engine Migration Cleanup Checklist + +**Use this checklist when removing Typesense and completing the migration to OpenSearch.** + +## ๐Ÿ—‘๏ธ Files to DELETE Completely + +- [ ] `SearchMigrationManager.java` - Temporary migration manager +- [ ] `AdminSearchController.java` - Temporary admin endpoints for migration +- [ ] `TypesenseService.java` - Old search service (if exists) +- [ ] `TypesenseConfig.java` - Old configuration (if exists) +- [ ] Any frontend migration UI components + +## ๐Ÿ”ง Files to MODIFY + +### SearchServiceAdapter.java +Replace delegation with direct OpenSearch calls: + +```java +@Service +public class SearchServiceAdapter { + + @Autowired + private OpenSearchService openSearchService; // Only this remains + + public void indexStory(Story story) { + openSearchService.indexStory(story); // Direct call, no delegation + } + + public SearchResultDto searchStories(...) { + return openSearchService.searchStories(...); // Direct call + } + + // Remove all migration-related methods: + // - isDualWriteEnabled() + // - getMigrationStatus() + // - etc. +} +``` + +### pom.xml +Remove Typesense dependency: +```xml + + + org.typesense + typesense-java + 1.3.0 + +``` + +### application.yml +Remove migration configuration: +```yaml +storycove: + search: + # DELETE these lines: + engine: opensearch + dual-write: false + # DELETE entire typesense section: + typesense: + api-key: xyz + host: localhost + port: 8108 + # ... etc +``` + +### docker-compose.yml +Remove Typesense service: +```yaml +# DELETE entire typesense service block +typesense: + image: typesense/typesense:0.25.1 + # ... etc + +# DELETE typesense volume +volumes: + typesense_data: +``` + +## ๐ŸŒ Environment Variables to REMOVE + +- [ ] `TYPESENSE_API_KEY` +- [ ] `TYPESENSE_HOST` +- [ ] `TYPESENSE_PORT` +- [ ] `TYPESENSE_ENABLED` +- [ ] `SEARCH_ENGINE` +- [ ] `SEARCH_DUAL_WRITE` + +## โœ… Configuration to KEEP (OpenSearch only) + +```yaml +storycove: + opensearch: + host: ${OPENSEARCH_HOST:localhost} + port: ${OPENSEARCH_PORT:9200} + # ... all OpenSearch config remains +``` + +## ๐Ÿงช Testing After Cleanup + +- [ ] Compilation successful: `mvn compile` +- [ ] All tests pass: `mvn test` +- [ ] Application starts without errors +- [ ] Search functionality works correctly +- [ ] No references to Typesense in logs +- [ ] Docker containers start without Typesense + +## ๐Ÿ“ Estimated Cleanup Time + +**Total: ~30 minutes** +- Delete files: 5 minutes +- Update SearchServiceAdapter: 10 minutes +- Remove configuration: 5 minutes +- Testing: 10 minutes + +## ๐Ÿšจ Rollback Plan (if needed) + +If issues arise during cleanup: + +1. **Immediate:** Restore from git: `git checkout HEAD~1` +2. **Verify:** Ensure application works with previous state +3. **Investigate:** Fix issues in a separate branch +4. **Retry:** Complete cleanup when issues resolved + +## โœจ Post-Cleanup Benefits + +- **Simpler codebase:** No dual-engine complexity +- **Reduced dependencies:** Smaller build artifacts +- **Better performance:** No dual-write overhead +- **Easier maintenance:** Single search engine to manage +- **Cleaner configuration:** Fewer environment variables + +--- + +**Created:** 2025-09-18 +**Purpose:** Temporary migration assistance +**Delete this file:** After cleanup is complete \ No newline at end of file diff --git a/DUAL_WRITE_USAGE.md b/DUAL_WRITE_USAGE.md new file mode 100644 index 0000000..f2c3250 --- /dev/null +++ b/DUAL_WRITE_USAGE.md @@ -0,0 +1,188 @@ +# Dual-Write Search Engine Usage Guide + +This guide explains how to use the dual-write functionality during the search engine migration. + +## ๐ŸŽ›๏ธ Configuration Options + +### Environment Variables + +```bash +# Search engine selection +SEARCH_ENGINE=typesense # or 'opensearch' +SEARCH_DUAL_WRITE=false # or 'true' + +# OpenSearch connection (required when using OpenSearch) +OPENSEARCH_PASSWORD=your_password +``` + +### Migration Flow + +#### Phase 1: Initial Setup +```bash +SEARCH_ENGINE=typesense # Keep using Typesense +SEARCH_DUAL_WRITE=false # Single-write to Typesense only +``` + +#### Phase 2: Enable Dual-Write +```bash +SEARCH_ENGINE=typesense # Still reading from Typesense +SEARCH_DUAL_WRITE=true # Now writing to BOTH engines +``` + +#### Phase 3: Switch to OpenSearch +```bash +SEARCH_ENGINE=opensearch # Now reading from OpenSearch +SEARCH_DUAL_WRITE=true # Still writing to both (safety) +``` + +#### Phase 4: Complete Migration +```bash +SEARCH_ENGINE=opensearch # Reading from OpenSearch +SEARCH_DUAL_WRITE=false # Only writing to OpenSearch +``` + +## ๐Ÿ”ง Admin API Endpoints + +### Check Current Status +```bash +GET /api/admin/search/status + +Response: +{ + "primaryEngine": "typesense", + "dualWrite": false, + "typesenseAvailable": true, + "openSearchAvailable": true +} +``` + +### Switch Configuration +```bash +POST /api/admin/search/configure +Content-Type: application/json + +{ + "engine": "opensearch", + "dualWrite": true +} +``` + +### Quick Actions +```bash +# Enable dual-write +POST /api/admin/search/dual-write/enable + +# Disable dual-write +POST /api/admin/search/dual-write/disable + +# Switch to OpenSearch +POST /api/admin/search/switch/opensearch + +# Switch back to Typesense +POST /api/admin/search/switch/typesense + +# Emergency rollback (Typesense only, no dual-write) +POST /api/admin/search/emergency-rollback +``` + +## ๐Ÿš€ Migration Process Example + +### Step 1: Verify OpenSearch is Ready +```bash +# Check if OpenSearch is available +curl http://localhost:8080/api/admin/search/status + +# Should show openSearchAvailable: true +``` + +### Step 2: Enable Dual-Write +```bash +# Enable writing to both engines +curl -X POST http://localhost:8080/api/admin/search/dual-write/enable + +# Or via configuration +curl -X POST http://localhost:8080/api/admin/search/configure \ + -H "Content-Type: application/json" \ + -d '{"engine": "typesense", "dualWrite": true}' +``` + +### Step 3: Populate OpenSearch +At this point, any new/updated stories will be written to both engines. +For existing data, you may want to trigger a reindex. + +### Step 4: Switch to OpenSearch +```bash +# Switch primary engine to OpenSearch +curl -X POST http://localhost:8080/api/admin/search/switch/opensearch + +# Check it worked +curl http://localhost:8080/api/admin/search/status +# Should show: primaryEngine: "opensearch", dualWrite: true +``` + +### Step 5: Test & Validate +- Test search functionality +- Verify results match expectations +- Monitor for any errors + +### Step 6: Complete Migration +```bash +# Disable dual-write (OpenSearch only) +curl -X POST http://localhost:8080/api/admin/search/dual-write/disable +``` + +## ๐Ÿšจ Emergency Procedures + +### Immediate Rollback +If OpenSearch has issues: +```bash +curl -X POST http://localhost:8080/api/admin/search/emergency-rollback +``` +This immediately switches to Typesense-only mode. + +### Partial Rollback +Switch back to Typesense but keep dual-write: +```bash +curl -X POST http://localhost:8080/api/admin/search/switch/typesense +``` + +## ๐Ÿ“Š Monitoring + +### Check Current Configuration +```bash +curl http://localhost:8080/api/admin/search/status +``` + +### Application Logs +Watch for dual-write success/failure messages: +```bash +# Successful operations +2025-09-18 08:00:00 DEBUG SearchMigrationManager - Successfully indexed story 123 in OpenSearch +2025-09-18 08:00:00 DEBUG SearchMigrationManager - Successfully indexed story 123 in Typesense + +# Failed operations (non-critical in dual-write mode) +2025-09-18 08:00:00 ERROR SearchMigrationManager - Failed to index story 123 in OpenSearch +``` + +## โš ๏ธ Important Notes + +1. **Dual-write errors are non-critical** - if one engine fails, the other continues +2. **Read operations only use primary engine** - no dual-read +3. **Configuration updates take effect immediately** - no restart required +4. **Emergency rollback is always available** - safe to experiment +5. **Both engines must be available** for dual-write to work optimally + +## ๐Ÿ”„ Typical Migration Timeline + +| Step | Duration | Configuration | Purpose | +|------|----------|---------------|---------| +| 1 | - | typesense + dual:false | Current state | +| 2 | 1 day | typesense + dual:true | Populate OpenSearch | +| 3 | 1 day | opensearch + dual:true | Test OpenSearch | +| 4 | - | opensearch + dual:false | Complete migration | + +**Total migration time: ~2 days of gradual transition** + +--- + +**Note:** This dual-write mechanism is temporary and will be removed once the migration is complete. See `CLEANUP_CHECKLIST.md` for removal instructions. \ No newline at end of file diff --git a/backend/pom.xml b/backend/pom.xml index 845e02c..bab98a5 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -136,6 +136,13 @@ org.springframework.boot spring-boot-maven-plugin + + org.apache.maven.plugins + maven-compiler-plugin + + true + + \ No newline at end of file diff --git a/backend/src/main/java/com/storycove/config/OpenSearchConfig.java b/backend/src/main/java/com/storycove/config/OpenSearchConfig.java index 5f5e40f..476d7a5 100644 --- a/backend/src/main/java/com/storycove/config/OpenSearchConfig.java +++ b/backend/src/main/java/com/storycove/config/OpenSearchConfig.java @@ -1,5 +1,7 @@ package com.storycove.config; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import org.apache.hc.client5.http.auth.AuthScope; import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; @@ -8,13 +10,13 @@ import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBu import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; import org.apache.hc.core5.http.HttpHost; import org.apache.hc.core5.util.Timeout; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; import org.opensearch.client.opensearch.OpenSearchClient; import org.opensearch.client.transport.OpenSearchTransport; import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -26,19 +28,17 @@ import java.security.KeyStore; import java.security.cert.X509Certificate; @Configuration -@EnableConfigurationProperties(OpenSearchProperties.class) public class OpenSearchConfig { private static final Logger logger = LoggerFactory.getLogger(OpenSearchConfig.class); private final OpenSearchProperties properties; - public OpenSearchConfig(OpenSearchProperties properties) { + public OpenSearchConfig(@Qualifier("openSearchProperties") OpenSearchProperties properties) { this.properties = properties; } @Bean - @ConditionalOnProperty(name = "storycove.search.engine", havingValue = "opensearch") public OpenSearchClient openSearchClient() throws Exception { logger.info("Initializing OpenSearch client for profile: {}", properties.getProfile()); @@ -51,13 +51,23 @@ public class OpenSearchConfig { // Create connection manager with pooling PoolingAsyncClientConnectionManager connectionManager = createConnectionManager(sslContext); - // Create the transport with all configurations + // Create custom ObjectMapper for proper date serialization + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerModule(new JavaTimeModule()); + objectMapper.disable(com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + + // Create the transport with all configurations and custom Jackson mapper OpenSearchTransport transport = ApacheHttpClient5TransportBuilder .builder(new HttpHost(properties.getScheme(), properties.getHost(), properties.getPort())) + .setMapper(new JacksonJsonpMapper(objectMapper)) .setHttpClientConfigCallback(httpClientBuilder -> { - httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider) - .setConnectionManager(connectionManager); + // Only set credentials provider if authentication is configured + if (properties.getUsername() != null && !properties.getUsername().isEmpty() && + properties.getPassword() != null && !properties.getPassword().isEmpty()) { + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + } + + httpClientBuilder.setConnectionManager(connectionManager); // Set timeouts httpClientBuilder.setDefaultRequestConfig( @@ -81,13 +91,22 @@ public class OpenSearchConfig { private BasicCredentialsProvider createCredentialsProvider() { BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials( - new AuthScope(properties.getHost(), properties.getPort()), - new UsernamePasswordCredentials( - properties.getUsername(), - properties.getPassword() != null ? properties.getPassword().toCharArray() : new char[0] - ) - ); + + // Only set credentials if username and password are provided + if (properties.getUsername() != null && !properties.getUsername().isEmpty() && + properties.getPassword() != null && !properties.getPassword().isEmpty()) { + credentialsProvider.setCredentials( + new AuthScope(properties.getHost(), properties.getPort()), + new UsernamePasswordCredentials( + properties.getUsername(), + properties.getPassword().toCharArray() + ) + ); + logger.info("OpenSearch credentials configured for user: {}", properties.getUsername()); + } else { + logger.info("OpenSearch running without authentication (no credentials configured)"); + } + return credentialsProvider; } @@ -184,8 +203,9 @@ public class OpenSearchConfig { response.version().number(), response.clusterName()); } catch (Exception e) { - logger.error("Failed to connect to OpenSearch cluster", e); - throw new RuntimeException("OpenSearch connection failed", e); + logger.warn("OpenSearch connection test failed during initialization: {}", e.getMessage()); + logger.debug("OpenSearch connection test full error", e); + // Don't throw exception here - let the client be created and handle failures in service methods } } } \ No newline at end of file diff --git a/backend/src/main/java/com/storycove/controller/AdminSearchController.java b/backend/src/main/java/com/storycove/controller/AdminSearchController.java new file mode 100644 index 0000000..81a125d --- /dev/null +++ b/backend/src/main/java/com/storycove/controller/AdminSearchController.java @@ -0,0 +1,296 @@ +package com.storycove.controller; + +import com.storycove.entity.Author; +import com.storycove.entity.Story; +import com.storycove.service.AuthorService; +import com.storycove.service.OpenSearchService; +import com.storycove.service.SearchMigrationManager; +import com.storycove.service.StoryService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import java.util.List; +import java.util.Map; + +/** + * TEMPORARY ADMIN CONTROLLER - DELETE THIS ENTIRE CLASS WHEN TYPESENSE IS REMOVED + * + * This controller provides admin endpoints for managing the search engine migration. + * It allows real-time switching between engines and enabling/disabling dual-write. + * + * CLEANUP INSTRUCTIONS: + * 1. Delete this entire file: AdminSearchController.java + * 2. Remove any frontend components that call these endpoints + */ +@RestController +@RequestMapping("/api/admin/search") +public class AdminSearchController { + + private static final Logger logger = LoggerFactory.getLogger(AdminSearchController.class); + + @Autowired + private SearchMigrationManager migrationManager; + + @Autowired(required = false) + private OpenSearchService openSearchService; + + @Autowired + private StoryService storyService; + + @Autowired + private AuthorService authorService; + + /** + * Get current search engine configuration status + */ + @GetMapping("/status") + public ResponseEntity getStatus() { + try { + SearchMigrationManager.SearchMigrationStatus status = migrationManager.getStatus(); + return ResponseEntity.ok(status); + } catch (Exception e) { + logger.error("Error getting search migration status", e); + return ResponseEntity.internalServerError().build(); + } + } + + /** + * Update search engine configuration + */ + @PostMapping("/configure") + public ResponseEntity configureSearchEngine(@RequestBody SearchEngineConfigRequest request) { + try { + logger.info("Updating search engine configuration: engine={}, dualWrite={}", + request.getEngine(), request.isDualWrite()); + + // Validate engine + if (!"typesense".equalsIgnoreCase(request.getEngine()) && + !"opensearch".equalsIgnoreCase(request.getEngine())) { + return ResponseEntity.badRequest().body("Invalid engine. Must be 'typesense' or 'opensearch'"); + } + + // Update configuration + migrationManager.updateConfiguration(request.getEngine(), request.isDualWrite()); + + return ResponseEntity.ok("Search engine configuration updated successfully"); + + } catch (Exception e) { + logger.error("Error updating search engine configuration", e); + return ResponseEntity.internalServerError().body("Failed to update configuration: " + e.getMessage()); + } + } + + /** + * Enable dual-write mode (writes to both engines) + */ + @PostMapping("/dual-write/enable") + public ResponseEntity enableDualWrite() { + try { + String currentEngine = migrationManager.getCurrentSearchEngine(); + migrationManager.updateConfiguration(currentEngine, true); + logger.info("Dual-write enabled for engine: {}", currentEngine); + return ResponseEntity.ok("Dual-write enabled"); + } catch (Exception e) { + logger.error("Error enabling dual-write", e); + return ResponseEntity.internalServerError().body("Failed to enable dual-write: " + e.getMessage()); + } + } + + /** + * Disable dual-write mode + */ + @PostMapping("/dual-write/disable") + public ResponseEntity disableDualWrite() { + try { + String currentEngine = migrationManager.getCurrentSearchEngine(); + migrationManager.updateConfiguration(currentEngine, false); + logger.info("Dual-write disabled for engine: {}", currentEngine); + return ResponseEntity.ok("Dual-write disabled"); + } catch (Exception e) { + logger.error("Error disabling dual-write", e); + return ResponseEntity.internalServerError().body("Failed to disable dual-write: " + e.getMessage()); + } + } + + /** + * Switch to OpenSearch engine + */ + @PostMapping("/switch/opensearch") + public ResponseEntity switchToOpenSearch() { + try { + if (!migrationManager.canSwitchToOpenSearch()) { + return ResponseEntity.badRequest().body("OpenSearch is not available or healthy"); + } + + boolean currentDualWrite = migrationManager.isDualWriteEnabled(); + migrationManager.updateConfiguration("opensearch", currentDualWrite); + logger.info("Switched to OpenSearch with dual-write: {}", currentDualWrite); + return ResponseEntity.ok("Switched to OpenSearch"); + } catch (Exception e) { + logger.error("Error switching to OpenSearch", e); + return ResponseEntity.internalServerError().body("Failed to switch to OpenSearch: " + e.getMessage()); + } + } + + /** + * Switch to Typesense engine (rollback) + */ + @PostMapping("/switch/typesense") + public ResponseEntity switchToTypesense() { + try { + if (!migrationManager.canSwitchToTypesense()) { + return ResponseEntity.badRequest().body("Typesense is not available"); + } + + boolean currentDualWrite = migrationManager.isDualWriteEnabled(); + migrationManager.updateConfiguration("typesense", currentDualWrite); + logger.info("Switched to Typesense with dual-write: {}", currentDualWrite); + return ResponseEntity.ok("Switched to Typesense"); + } catch (Exception e) { + logger.error("Error switching to Typesense", e); + return ResponseEntity.internalServerError().body("Failed to switch to Typesense: " + e.getMessage()); + } + } + + /** + * Emergency rollback to Typesense with dual-write disabled + */ + @PostMapping("/emergency-rollback") + public ResponseEntity emergencyRollback() { + try { + migrationManager.updateConfiguration("typesense", false); + logger.warn("Emergency rollback to Typesense executed"); + return ResponseEntity.ok("Emergency rollback completed - switched to Typesense only"); + } catch (Exception e) { + logger.error("Error during emergency rollback", e); + return ResponseEntity.internalServerError().body("Emergency rollback failed: " + e.getMessage()); + } + } + + /** + * Reindex all data in OpenSearch (equivalent to Typesense reindex) + */ + @PostMapping("/opensearch/reindex") + public ResponseEntity> reindexOpenSearch() { + try { + logger.info("Starting OpenSearch full reindex"); + + if (!migrationManager.canSwitchToOpenSearch()) { + return ResponseEntity.badRequest().body(Map.of( + "success", false, + "error", "OpenSearch is not available or healthy" + )); + } + + // Get all data from services (similar to Typesense reindex) + List allStories = storyService.findAllWithAssociations(); + List allAuthors = authorService.findAllWithStories(); + + // Bulk index directly in OpenSearch + if (openSearchService != null) { + openSearchService.bulkIndexStories(allStories); + openSearchService.bulkIndexAuthors(allAuthors); + } else { + return ResponseEntity.badRequest().body(Map.of( + "success", false, + "error", "OpenSearch service not available" + )); + } + + int totalIndexed = allStories.size() + allAuthors.size(); + + return ResponseEntity.ok(Map.of( + "success", true, + "message", String.format("Reindexed %d stories and %d authors in OpenSearch", + allStories.size(), allAuthors.size()), + "storiesCount", allStories.size(), + "authorsCount", allAuthors.size(), + "totalCount", totalIndexed + )); + + } catch (Exception e) { + logger.error("Error during OpenSearch reindex", e); + return ResponseEntity.internalServerError().body(Map.of( + "success", false, + "error", "OpenSearch reindex failed: " + e.getMessage() + )); + } + } + + /** + * Recreate OpenSearch indices (equivalent to Typesense collection recreation) + */ + @PostMapping("/opensearch/recreate") + public ResponseEntity> recreateOpenSearchIndices() { + try { + logger.info("Starting OpenSearch indices recreation"); + + if (!migrationManager.canSwitchToOpenSearch()) { + return ResponseEntity.badRequest().body(Map.of( + "success", false, + "error", "OpenSearch is not available or healthy" + )); + } + + // Recreate OpenSearch indices directly + if (openSearchService != null) { + openSearchService.recreateIndices(); + } else { + logger.error("OpenSearchService not available for index recreation"); + return ResponseEntity.badRequest().body(Map.of( + "success", false, + "error", "OpenSearchService not available" + )); + } + + // Now populate the freshly created indices directly in OpenSearch + List allStories = storyService.findAllWithAssociations(); + List allAuthors = authorService.findAllWithStories(); + + openSearchService.bulkIndexStories(allStories); + openSearchService.bulkIndexAuthors(allAuthors); + + int totalIndexed = allStories.size() + allAuthors.size(); + + return ResponseEntity.ok(Map.of( + "success", true, + "message", String.format("Recreated OpenSearch indices and indexed %d stories and %d authors", + allStories.size(), allAuthors.size()), + "storiesCount", allStories.size(), + "authorsCount", allAuthors.size(), + "totalCount", totalIndexed + )); + + } catch (Exception e) { + logger.error("Error during OpenSearch indices recreation", e); + return ResponseEntity.internalServerError().body(Map.of( + "success", false, + "error", "OpenSearch indices recreation failed: " + e.getMessage() + )); + } + } + + /** + * DTO for search engine configuration requests + */ + public static class SearchEngineConfigRequest { + private String engine; + private boolean dualWrite; + + public SearchEngineConfigRequest() {} + + public SearchEngineConfigRequest(String engine, boolean dualWrite) { + this.engine = engine; + this.dualWrite = dualWrite; + } + + public String getEngine() { return engine; } + public void setEngine(String engine) { this.engine = engine; } + + public boolean isDualWrite() { return dualWrite; } + public void setDualWrite(boolean dualWrite) { this.dualWrite = dualWrite; } + } +} \ No newline at end of file diff --git a/backend/src/main/java/com/storycove/controller/AuthorController.java b/backend/src/main/java/com/storycove/controller/AuthorController.java index f1f8dd9..69afcf0 100644 --- a/backend/src/main/java/com/storycove/controller/AuthorController.java +++ b/backend/src/main/java/com/storycove/controller/AuthorController.java @@ -4,6 +4,7 @@ import com.storycove.dto.*; import com.storycove.entity.Author; import com.storycove.service.AuthorService; import com.storycove.service.ImageService; +import com.storycove.service.SearchServiceAdapter; import com.storycove.service.TypesenseService; import jakarta.servlet.http.HttpServletRequest; import jakarta.validation.Valid; @@ -33,11 +34,13 @@ public class AuthorController { private final AuthorService authorService; private final ImageService imageService; private final TypesenseService typesenseService; + private final SearchServiceAdapter searchServiceAdapter; - public AuthorController(AuthorService authorService, ImageService imageService, TypesenseService typesenseService) { + public AuthorController(AuthorService authorService, ImageService imageService, TypesenseService typesenseService, SearchServiceAdapter searchServiceAdapter) { this.authorService = authorService; this.imageService = imageService; this.typesenseService = typesenseService; + this.searchServiceAdapter = searchServiceAdapter; } @GetMapping @@ -258,7 +261,17 @@ public class AuthorController { @RequestParam(defaultValue = "name") String sortBy, @RequestParam(defaultValue = "asc") String sortOrder) { - SearchResultDto searchResults = typesenseService.searchAuthors(q, page, size, sortBy, sortOrder); + // Use SearchServiceAdapter to handle routing between search engines + List authorSearchResults = searchServiceAdapter.searchAuthors(q, size); + + // Create SearchResultDto to match expected return format + SearchResultDto searchResults = new SearchResultDto<>(); + searchResults.setResults(authorSearchResults); + searchResults.setQuery(q); + searchResults.setPage(page); + searchResults.setPerPage(size); + searchResults.setTotalHits(authorSearchResults.size()); + searchResults.setSearchTimeMs(0); // SearchServiceAdapter doesn't provide timing // Convert AuthorSearchDto results to AuthorDto SearchResultDto results = new SearchResultDto<>(); diff --git a/backend/src/main/java/com/storycove/controller/StoryController.java b/backend/src/main/java/com/storycove/controller/StoryController.java index fa2d9fa..e8d6393 100644 --- a/backend/src/main/java/com/storycove/controller/StoryController.java +++ b/backend/src/main/java/com/storycove/controller/StoryController.java @@ -42,6 +42,7 @@ public class StoryController { private final HtmlSanitizationService sanitizationService; private final ImageService imageService; private final TypesenseService typesenseService; + private final SearchServiceAdapter searchServiceAdapter; private final CollectionService collectionService; private final ReadingTimeService readingTimeService; private final EPUBImportService epubImportService; @@ -54,6 +55,7 @@ public class StoryController { ImageService imageService, CollectionService collectionService, @Autowired(required = false) TypesenseService typesenseService, + SearchServiceAdapter searchServiceAdapter, ReadingTimeService readingTimeService, EPUBImportService epubImportService, EPUBExportService epubExportService) { @@ -64,6 +66,7 @@ public class StoryController { this.imageService = imageService; this.collectionService = collectionService; this.typesenseService = typesenseService; + this.searchServiceAdapter = searchServiceAdapter; this.readingTimeService = readingTimeService; this.epubImportService = epubImportService; this.epubExportService = epubExportService; @@ -326,7 +329,7 @@ public class StoryController { @RequestParam(required = false) Integer maxRating, @RequestParam(required = false) String sortBy, @RequestParam(required = false) String sortDir, - @RequestParam(required = false) String facetBy, + @RequestParam(required = false) List facetBy, // Advanced filters @RequestParam(required = false) Integer minWordCount, @RequestParam(required = false) Integer maxWordCount, @@ -345,16 +348,35 @@ public class StoryController { @RequestParam(required = false) Boolean hiddenGemsOnly) { - if (typesenseService != null) { - SearchResultDto results = typesenseService.searchStories( - query, page, size, authors, tags, minRating, maxRating, sortBy, sortDir, facetBy, - minWordCount, maxWordCount, createdAfter, createdBefore, lastReadAfter, lastReadBefore, - unratedOnly, readingStatus, hasReadingProgress, hasCoverImage, sourceDomain, seriesFilter, - minTagCount, popularOnly, hiddenGemsOnly); + // Use SearchServiceAdapter to handle routing between search engines + try { + // Convert authors list to single author string (for now, use first author) + String authorFilter = (authors != null && !authors.isEmpty()) ? authors.get(0) : null; + + // DEBUG: Log all received parameters + logger.info("CONTROLLER DEBUG - Received parameters:"); + logger.info(" readingStatus: '{}'", readingStatus); + logger.info(" seriesFilter: '{}'", seriesFilter); + logger.info(" hasReadingProgress: {}", hasReadingProgress); + logger.info(" hasCoverImage: {}", hasCoverImage); + logger.info(" createdAfter: '{}'", createdAfter); + logger.info(" lastReadAfter: '{}'", lastReadAfter); + logger.info(" unratedOnly: {}", unratedOnly); + + SearchResultDto results = searchServiceAdapter.searchStories( + query, tags, authorFilter, seriesFilter, minWordCount, maxWordCount, + minRating != null ? minRating.floatValue() : null, + null, // isRead - now handled by readingStatus advanced filter + null, // isFavorite - now handled by readingStatus advanced filter + sortBy, sortDir, page, size, facetBy, + // Advanced filters + createdAfter, createdBefore, lastReadAfter, lastReadBefore, + unratedOnly, readingStatus, hasReadingProgress, hasCoverImage, + sourceDomain, seriesFilter, minTagCount, popularOnly, hiddenGemsOnly); return ResponseEntity.ok(results); - } else { - // Fallback to basic search if Typesense is not available - return ResponseEntity.badRequest().body(null); + } catch (Exception e) { + logger.error("Search failed", e); + return ResponseEntity.internalServerError().body(null); } } @@ -363,10 +385,12 @@ public class StoryController { @RequestParam String query, @RequestParam(defaultValue = "5") int limit) { - if (typesenseService != null) { - List suggestions = typesenseService.searchSuggestions(query, limit); + // Use SearchServiceAdapter to handle routing between search engines + try { + List suggestions = searchServiceAdapter.getTagSuggestions(query, limit); return ResponseEntity.ok(suggestions); - } else { + } catch (Exception e) { + logger.error("Failed to get search suggestions", e); return ResponseEntity.ok(new ArrayList<>()); } } diff --git a/backend/src/main/java/com/storycove/dto/StorySearchDto.java b/backend/src/main/java/com/storycove/dto/StorySearchDto.java index 33af6b0..3bf2e75 100644 --- a/backend/src/main/java/com/storycove/dto/StorySearchDto.java +++ b/backend/src/main/java/com/storycove/dto/StorySearchDto.java @@ -17,6 +17,7 @@ public class StorySearchDto { // Reading status private Boolean isRead; + private Integer readingPosition; private LocalDateTime lastReadAt; // Author info @@ -32,6 +33,9 @@ public class StorySearchDto { private LocalDateTime createdAt; private LocalDateTime updatedAt; + + // Alias for createdAt to match frontend expectations + private LocalDateTime dateAdded; // Search-specific fields private double searchScore; @@ -120,6 +124,14 @@ public class StorySearchDto { public void setLastReadAt(LocalDateTime lastReadAt) { this.lastReadAt = lastReadAt; } + + public Integer getReadingPosition() { + return readingPosition; + } + + public void setReadingPosition(Integer readingPosition) { + this.readingPosition = readingPosition; + } public UUID getAuthorId() { return authorId; @@ -176,6 +188,14 @@ public class StorySearchDto { public void setUpdatedAt(LocalDateTime updatedAt) { this.updatedAt = updatedAt; } + + public LocalDateTime getDateAdded() { + return dateAdded; + } + + public void setDateAdded(LocalDateTime dateAdded) { + this.dateAdded = dateAdded; + } public double getSearchScore() { return searchScore; diff --git a/backend/src/main/java/com/storycove/service/OpenSearchService.java b/backend/src/main/java/com/storycove/service/OpenSearchService.java index 23e770b..589c58b 100644 --- a/backend/src/main/java/com/storycove/service/OpenSearchService.java +++ b/backend/src/main/java/com/storycove/service/OpenSearchService.java @@ -2,15 +2,24 @@ package com.storycove.service; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Iterator; import com.storycove.config.OpenSearchProperties; import com.storycove.dto.AuthorSearchDto; +import com.storycove.dto.FacetCountDto; import com.storycove.dto.SearchResultDto; import com.storycove.dto.StorySearchDto; +import com.storycove.entity.Author; +import com.storycove.entity.Story; import org.opensearch.client.opensearch.OpenSearchClient; import org.opensearch.client.opensearch._types.mapping.TypeMapping; import org.opensearch.client.opensearch.indices.CreateIndexRequest; import org.opensearch.client.opensearch.indices.ExistsRequest; import org.opensearch.client.opensearch.indices.IndexSettings; +import org.opensearch.client.opensearch.core.*; +import org.opensearch.client.opensearch._types.query_dsl.*; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -23,258 +32,1046 @@ import jakarta.annotation.PostConstruct; import java.io.IOException; import java.io.InputStream; import java.util.*; +import java.util.stream.Collectors; @Service -@ConditionalOnProperty(name = "storycove.search.engine", havingValue = "opensearch") public class OpenSearchService { private static final Logger logger = LoggerFactory.getLogger(OpenSearchService.class); - private final OpenSearchClient openSearchClient; - private final LibraryService libraryService; - private final ReadingTimeService readingTimeService; - private final ObjectMapper objectMapper; - private final OpenSearchProperties properties; + @Autowired(required = false) + private OpenSearchClient openSearchClient; - // Services for complete reindexing (avoiding circular dependencies with @Lazy) @Autowired - @Lazy - private StoryService storyService; + private OpenSearchProperties properties; @Autowired @Lazy - private AuthorService authorService; + private ReadingTimeService readingTimeService; - @Autowired - @Lazy - private CollectionService collectionService; - - @Autowired - public OpenSearchService(OpenSearchClient openSearchClient, - LibraryService libraryService, - ReadingTimeService readingTimeService, - ObjectMapper objectMapper, - OpenSearchProperties properties) { - this.openSearchClient = openSearchClient; - this.libraryService = libraryService; - this.readingTimeService = readingTimeService; - this.objectMapper = objectMapper; - this.properties = properties; - } - - // Index names are now dynamic based on active library - private String getStoriesIndex() { - var library = libraryService.getCurrentLibrary(); - return library != null ? "stories_" + library.getId() : "stories"; - } - - private String getAuthorsIndex() { - var library = libraryService.getCurrentLibrary(); - return library != null ? "authors_" + library.getId() : "authors"; - } - - private String getCollectionsIndex() { - var library = libraryService.getCurrentLibrary(); - return library != null ? "collections_" + library.getId() : "collections"; - } + private static final String STORIES_INDEX = "stories"; + private static final String AUTHORS_INDEX = "authors"; @PostConstruct - public void initializeIndexes() { + public void initializeIndices() { + if (!isAvailable()) { + logger.info("OpenSearch client not available - skipping index initialization"); + return; + } + try { - createStoriesIndexIfNotExists(); - createAuthorsIndexIfNotExists(); - createCollectionsIndexIfNotExists(); - } catch (Exception e) { - logger.error("Failed to initialize OpenSearch indexes", e); + logger.info("Initializing OpenSearch indices..."); + createStoriesIndex(); + createAuthorsIndex(); + logger.info("OpenSearch indices initialized successfully"); + } catch (IOException e) { + logger.error("Failed to initialize OpenSearch indices", e); } } - /** - * Initialize indexes for the current active library - */ - public void initializeIndexesForCurrentLibrary() { - try { - logger.info("Initializing OpenSearch indexes for current library"); - createStoriesIndexIfNotExists(); - createAuthorsIndexIfNotExists(); - createCollectionsIndexIfNotExists(); - logger.info("Successfully initialized OpenSearch indexes for current library"); - } catch (Exception e) { - logger.error("Failed to initialize OpenSearch indexes for current library", e); - throw new RuntimeException("OpenSearch index initialization failed", e); + // =============================== + // INDEX MANAGEMENT + // =============================== + + private void createStoriesIndex() throws IOException { + String indexName = getStoriesIndex(); + if (indexExists(indexName)) { + logger.debug("Stories index already exists: {}", indexName); + return; } - } - /** - * Test OpenSearch connection - */ - public boolean testConnection() { - try { - var response = openSearchClient.info(); - logger.info("OpenSearch connection successful. Version: {}", response.version().number()); - return true; - } catch (Exception e) { - logger.error("Failed to connect to OpenSearch", e); - return false; - } - } + logger.info("Creating stories index: {}", indexName); - /** - * Load index configuration from JSON file - */ - private JsonNode loadIndexConfiguration(String mappingFile) throws IOException { - ClassPathResource resource = new ClassPathResource("opensearch/mappings/" + mappingFile); - try (InputStream inputStream = resource.getInputStream()) { - return objectMapper.readTree(inputStream); - } - } - - /** - * Create index from JSON configuration - */ - private void createIndexFromConfiguration(String indexName, String mappingFile) throws IOException { - if (!indexExists(indexName)) { - logger.info("Creating OpenSearch index: {} from {}", indexName, mappingFile); - - // For now, create indexes with programmatic configuration - // TODO: Implement full JSON parsing when OpenSearch Java client supports it better - createProgrammaticIndex(indexName, mappingFile); - } - } - - /** - * Create index using programmatic configuration (temporary solution) - */ - private void createProgrammaticIndex(String indexName, String mappingFile) throws IOException { - logger.info("Creating OpenSearch index programmatically: {}", indexName); - - CreateIndexRequest.Builder requestBuilder = new CreateIndexRequest.Builder() - .index(indexName); - - // Set basic index settings based on environment - IndexSettings.Builder settingsBuilder = new IndexSettings.Builder() + // Create index settings programmatically + IndexSettings indexSettings = IndexSettings.of(is -> is .numberOfShards(properties.getIndices().getDefaultShards()) .numberOfReplicas(properties.getIndices().getDefaultReplicas()) - .refreshInterval(t -> t.time(properties.getIndices().getRefreshInterval())); + .refreshInterval(t -> t.time(properties.getIndices().getRefreshInterval())) + .analysis(a -> a + .analyzer("story_analyzer", an -> an + .standard(st -> st.stopwords("_english_")) + ) + ) + ); - requestBuilder.settings(settingsBuilder.build()); + // Create mapping programmatically + TypeMapping mapping = TypeMapping.of(tm -> tm + .properties("id", p -> p.keyword(k -> k)) + .properties("title", p -> p.text(t -> t + .analyzer("story_analyzer") + .fields("keyword", f -> f.keyword(k -> k.ignoreAbove(256))) + )) + .properties("content", p -> p.text(t -> t.analyzer("story_analyzer"))) + .properties("summary", p -> p.text(t -> t.analyzer("story_analyzer"))) + .properties("authorNames", p -> p.text(t -> t.analyzer("story_analyzer"))) + .properties("authorName", p -> p.keyword(k -> k)) + .properties("seriesName", p -> p.keyword(k -> k)) + .properties("tagNames", p -> p.keyword(k -> k)) + .properties("wordCount", p -> p.integer(i -> i)) + .properties("rating", p -> p.integer(i -> i)) + .properties("isRead", p -> p.boolean_(b -> b)) + .properties("readingPosition", p -> p.integer(i -> i)) + .properties("isFavorite", p -> p.boolean_(b -> b)) + .properties("createdAt", p -> p.date(d -> d.format("strict_date_optional_time||epoch_millis"))) + .properties("updatedAt", p -> p.date(d -> d.format("strict_date_optional_time||epoch_millis"))) + .properties("lastReadAt", p -> p.date(d -> d.format("strict_date_optional_time||epoch_millis"))) + .properties("dateAdded", p -> p.date(d -> d.format("strict_date_optional_time||epoch_millis"))) + ); - // Create mappings based on index type - if (mappingFile.contains("stories")) { - requestBuilder.mappings(createStoryMapping()); - } else if (mappingFile.contains("authors")) { - requestBuilder.mappings(createAuthorMapping()); - } else if (mappingFile.contains("collections")) { - requestBuilder.mappings(createCollectionMapping()); + CreateIndexRequest request = CreateIndexRequest.of(cir -> cir + .index(indexName) + .settings(indexSettings) + .mappings(mapping) + ); + + openSearchClient.indices().create(request); + logger.info("Created stories index successfully: {}", indexName); + } + + private void createAuthorsIndex() throws IOException { + String indexName = getAuthorsIndex(); + if (indexExists(indexName)) { + logger.debug("Authors index already exists: {}", indexName); + return; } - openSearchClient.indices().create(requestBuilder.build()); - logger.info("Created OpenSearch index: {}", indexName); - } + logger.info("Creating authors index: {}", indexName); - private TypeMapping createStoryMapping() { - return TypeMapping.of(m -> m - .properties("id", p -> p.keyword(k -> k)) - .properties("title", p -> p.text(t -> t.analyzer("standard"))) - .properties("content", p -> p.text(t -> t.analyzer("standard"))) - .properties("summary", p -> p.text(t -> t.analyzer("standard"))) - .properties("authorNames", p -> p.text(t -> t.analyzer("standard"))) - .properties("authorIds", p -> p.keyword(k -> k)) - .properties("tagNames", p -> p.keyword(k -> k)) - .properties("seriesTitle", p -> p.text(t -> t.analyzer("standard"))) - .properties("seriesId", p -> p.keyword(k -> k)) - .properties("wordCount", p -> p.integer(i -> i)) - .properties("rating", p -> p.float_(f -> f)) - .properties("readingTime", p -> p.integer(i -> i)) - .properties("language", p -> p.keyword(k -> k)) - .properties("status", p -> p.keyword(k -> k)) - .properties("createdAt", p -> p.date(d -> d)) - .properties("updatedAt", p -> p.date(d -> d)) - .properties("publishedAt", p -> p.date(d -> d)) - .properties("isRead", p -> p.boolean_(b -> b)) - .properties("isFavorite", p -> p.boolean_(b -> b)) - .properties("readingProgress", p -> p.float_(f -> f)) - .properties("libraryId", p -> p.keyword(k -> k)) + IndexSettings indexSettings = IndexSettings.of(is -> is + .numberOfShards(properties.getIndices().getDefaultShards()) + .numberOfReplicas(properties.getIndices().getDefaultReplicas()) + .refreshInterval(t -> t.time(properties.getIndices().getRefreshInterval())) ); - } - private TypeMapping createAuthorMapping() { - return TypeMapping.of(m -> m + TypeMapping mapping = TypeMapping.of(tm -> tm .properties("id", p -> p.keyword(k -> k)) .properties("name", p -> p.text(t -> t.analyzer("standard"))) .properties("bio", p -> p.text(t -> t.analyzer("standard"))) - .properties("urls", p -> p.keyword(k -> k)) - .properties("imageUrl", p -> p.keyword(k -> k)) .properties("storyCount", p -> p.integer(i -> i)) .properties("averageRating", p -> p.float_(f -> f)) - .properties("totalWordCount", p -> p.long_(l -> l)) - .properties("totalReadingTime", p -> p.integer(i -> i)) .properties("createdAt", p -> p.date(d -> d)) - .properties("updatedAt", p -> p.date(d -> d)) - .properties("libraryId", p -> p.keyword(k -> k)) ); - } - private TypeMapping createCollectionMapping() { - return TypeMapping.of(m -> m - .properties("id", p -> p.keyword(k -> k)) - .properties("name", p -> p.text(t -> t.analyzer("standard"))) - .properties("description", p -> p.text(t -> t.analyzer("standard"))) - .properties("storyCount", p -> p.integer(i -> i)) - .properties("totalWordCount", p -> p.long_(l -> l)) - .properties("averageRating", p -> p.float_(f -> f)) - .properties("isPublic", p -> p.boolean_(b -> b)) - .properties("createdAt", p -> p.date(d -> d)) - .properties("updatedAt", p -> p.date(d -> d)) - .properties("libraryId", p -> p.keyword(k -> k)) + CreateIndexRequest request = CreateIndexRequest.of(cir -> cir + .index(indexName) + .settings(indexSettings) + .mappings(mapping) ); - } - private void createStoriesIndexIfNotExists() throws IOException { - createIndexFromConfiguration(getStoriesIndex(), "stories-mapping.json"); - } - - private void createAuthorsIndexIfNotExists() throws IOException { - createIndexFromConfiguration(getAuthorsIndex(), "authors-mapping.json"); - } - - private void createCollectionsIndexIfNotExists() throws IOException { - createIndexFromConfiguration(getCollectionsIndex(), "collections-mapping.json"); + openSearchClient.indices().create(request); + logger.info("Created authors index successfully: {}", indexName); } private boolean indexExists(String indexName) throws IOException { - ExistsRequest request = ExistsRequest.of(e -> e.index(indexName)); + if (!isAvailable()) { + return false; + } + ExistsRequest request = ExistsRequest.of(er -> er.index(indexName)); return openSearchClient.indices().exists(request).value(); } - // Placeholder methods for search functionality (to be implemented in later phases) + private String getStoriesIndex() { + return STORIES_INDEX; + } + + private String getAuthorsIndex() { + return AUTHORS_INDEX; + } + + // =============================== + // CRUD OPERATIONS + // =============================== + + public void indexStory(Story story) throws IOException { + if (!isAvailable()) { + logger.debug("OpenSearch client not available - skipping story indexing for: {}", story.getId()); + return; + } + + logger.debug("Indexing story: {}", story.getId()); + + StorySearchDto searchDto = convertToStorySearchDto(story); + + IndexRequest request = IndexRequest.of(ir -> ir + .index(getStoriesIndex()) + .id(story.getId().toString()) + .document(searchDto) + ); + + openSearchClient.index(request); + logger.debug("Successfully indexed story: {}", story.getId()); + } + + public void updateStory(Story story) throws IOException { + logger.debug("Updating story: {}", story.getId()); + + StorySearchDto searchDto = convertToStorySearchDto(story); + + UpdateRequest request = UpdateRequest.of(ur -> ur + .index(getStoriesIndex()) + .id(story.getId().toString()) + .doc(searchDto) + .upsert(searchDto) + ); + + openSearchClient.update(request, StorySearchDto.class); + logger.debug("Successfully updated story: {}", story.getId()); + } + + public void deleteStory(UUID storyId) throws IOException { + logger.debug("Deleting story: {}", storyId); + + DeleteRequest request = DeleteRequest.of(dr -> dr + .index(getStoriesIndex()) + .id(storyId.toString()) + ); + + openSearchClient.delete(request); + logger.debug("Successfully deleted story: {}", storyId); + } + + public void indexAuthor(Author author) throws IOException { + if (!isAvailable()) { + logger.debug("OpenSearch client not available - skipping author indexing for: {}", author.getId()); + return; + } + + logger.debug("Indexing author: {}", author.getId()); + + AuthorSearchDto searchDto = convertToAuthorSearchDto(author); + + IndexRequest request = IndexRequest.of(ir -> ir + .index(getAuthorsIndex()) + .id(author.getId().toString()) + .document(searchDto) + ); + + openSearchClient.index(request); + logger.debug("Successfully indexed author: {}", author.getId()); + } + + public void updateAuthor(Author author) throws IOException { + logger.debug("Updating author: {}", author.getId()); + + AuthorSearchDto searchDto = convertToAuthorSearchDto(author); + + UpdateRequest request = UpdateRequest.of(ur -> ur + .index(getAuthorsIndex()) + .id(author.getId().toString()) + .doc(searchDto) + .upsert(searchDto) + ); + + openSearchClient.update(request, AuthorSearchDto.class); + logger.debug("Successfully updated author: {}", author.getId()); + } + + public void deleteAuthor(UUID authorId) throws IOException { + logger.debug("Deleting author: {}", authorId); + + DeleteRequest request = DeleteRequest.of(dr -> dr + .index(getAuthorsIndex()) + .id(authorId.toString()) + ); + + openSearchClient.delete(request); + logger.debug("Successfully deleted author: {}", authorId); + } + + // =============================== + // BULK OPERATIONS + // =============================== + + public void bulkIndexStories(List stories) throws IOException { + if (!isAvailable()) { + logger.debug("OpenSearch client not available - skipping bulk story indexing for {} stories", stories.size()); + return; + } + + if (stories.isEmpty()) { + logger.debug("No stories to bulk index"); + return; + } + + logger.info("Bulk indexing {} stories", stories.size()); + + BulkRequest.Builder bulkBuilder = new BulkRequest.Builder(); + + for (Story story : stories) { + StorySearchDto searchDto = convertToStorySearchDto(story); + bulkBuilder.operations(op -> op + .index(idx -> idx + .index(getStoriesIndex()) + .id(story.getId().toString()) + .document(searchDto) + ) + ); + } + + BulkResponse response = openSearchClient.bulk(bulkBuilder.build()); + + if (response.errors()) { + logger.error("Bulk indexing errors occurred"); + response.items().forEach(item -> { + if (item.error() != null) { + logger.error("Error indexing story {}: {}", item.id(), item.error().reason()); + } + }); + } + + logger.info("Successfully bulk indexed {} stories", stories.size()); + } + + public void bulkIndexAuthors(List authors) throws IOException { + if (!isAvailable()) { + logger.debug("OpenSearch client not available - skipping bulk author indexing for {} authors", authors.size()); + return; + } + + if (authors.isEmpty()) { + logger.debug("No authors to bulk index"); + return; + } + + logger.info("Bulk indexing {} authors", authors.size()); + + BulkRequest.Builder bulkBuilder = new BulkRequest.Builder(); + + for (Author author : authors) { + AuthorSearchDto searchDto = convertToAuthorSearchDto(author); + bulkBuilder.operations(op -> op + .index(idx -> idx + .index(getAuthorsIndex()) + .id(author.getId().toString()) + .document(searchDto) + ) + ); + } + + BulkResponse response = openSearchClient.bulk(bulkBuilder.build()); + + if (response.errors()) { + logger.error("Bulk indexing errors occurred"); + response.items().forEach(item -> { + if (item.error() != null) { + logger.error("Error indexing author {}: {}", item.id(), item.error().reason()); + } + }); + } + + logger.info("Successfully bulk indexed {} authors", authors.size()); + } + + // =============================== + // SEARCH OPERATIONS + // =============================== + public SearchResultDto searchStories(String query, List tags, String author, String series, Integer minWordCount, Integer maxWordCount, Float minRating, Boolean isRead, Boolean isFavorite, - String sortBy, String sortOrder, int page, int size) { - // TODO: Implement OpenSearch story search - logger.warn("OpenSearch story search not yet implemented"); - return new SearchResultDto<>(new ArrayList<>(), 0, page, size, query != null ? query : "", 0); + String sortBy, String sortOrder, int page, int size, + List facetBy, + // Advanced filters + String createdAfter, String createdBefore, + String lastReadAfter, String lastReadBefore, + Boolean unratedOnly, String readingStatus, + Boolean hasReadingProgress, Boolean hasCoverImage, + String sourceDomain, String seriesFilter, + Integer minTagCount, Boolean popularOnly, + Boolean hiddenGemsOnly) { + try { + logger.info("OPENSEARCH SEARCH DEBUG:"); + logger.info(" Query: '{}'", query); + logger.info(" Tags: {}", tags); + logger.info(" Author: '{}'", author); + logger.info(" Series: '{}'", series); + logger.info(" SortBy: '{}'", sortBy); + logger.info(" SortOrder: '{}'", sortOrder); + logger.info(" Page: {}, Size: {}", page, size); + logger.info(" FacetBy: {}", facetBy); + logger.info(" Advanced filters: createdAfter='{}', createdBefore='{}', lastReadAfter='{}', lastReadBefore='{}'", + createdAfter, createdBefore, lastReadAfter, lastReadBefore); + logger.info(" Boolean filters: unratedOnly={}, readingStatus='{}', hasReadingProgress={}, hasCoverImage={}", + unratedOnly, readingStatus, hasReadingProgress, hasCoverImage); + logger.info(" Other filters: sourceDomain='{}', seriesFilter='{}', minTagCount={}, popularOnly={}, hiddenGemsOnly={}", + sourceDomain, seriesFilter, minTagCount, popularOnly, hiddenGemsOnly); + + // Check index document count + try { + var countRequest = CountRequest.of(cr -> cr.index(getStoriesIndex())); + var countResponse = openSearchClient.count(countRequest); + logger.info(" Stories index document count: {}", countResponse.count()); + + // Test a simple search without sorting to see if we get results + if (countResponse.count() > 0) { + var testSearch = SearchRequest.of(sr -> sr + .index(getStoriesIndex()) + .size(1) + .query(q -> q.matchAll(ma -> ma)) + ); + var testResponse = openSearchClient.search(testSearch, StorySearchDto.class); + logger.info(" Test search without sorting: totalHits={}, hits.size()={}", + testResponse.hits().total() != null ? testResponse.hits().total().value() : 0, + testResponse.hits().hits().size()); + } + } catch (Exception e) { + logger.warn(" Could not get document count or test search: {}", e.getMessage()); + } + + SearchRequest.Builder searchBuilder = new SearchRequest.Builder() + .index(getStoriesIndex()) + .from(page * size) + .size(size); + + // Build query + BoolQuery.Builder boolBuilder = new BoolQuery.Builder(); + + // Full-text search + if (query != null && !query.trim().isEmpty()) { + String trimmedQuery = query.trim(); + // Handle wildcard queries + if ("*".equals(trimmedQuery) || "**".equals(trimmedQuery)) { + logger.info(" Using matchAll query for wildcard: '{}'", trimmedQuery); + boolBuilder.must(m -> m.matchAll(ma -> ma)); + } else { + logger.info(" Using multiMatch query for: '{}'", trimmedQuery); + boolBuilder.must(m -> m + .multiMatch(mm -> mm + .query(trimmedQuery) + .fields("title^3", "content", "summary^2", "authorName^2") + .type(TextQueryType.BestFields) + ) + ); + } + } else { + logger.info(" Using matchAll query for empty query"); + boolBuilder.must(m -> m.matchAll(ma -> ma)); + } + + // Add filters + if (tags != null && !tags.isEmpty()) { + logger.info(" Adding tags filter: {}", tags); + boolBuilder.filter(f -> f + .terms(t -> t + .field("tagNames") + .terms(ts -> ts.value(tags.stream().map(FieldValue::of).collect(Collectors.toList()))) + ) + ); + } + + if (author != null && !author.trim().isEmpty() && !"null".equalsIgnoreCase(author.trim())) { + logger.info(" Adding author filter: '{}'", author.trim()); + boolBuilder.filter(f -> f + .term(t -> t + .field("authorName") + .value(FieldValue.of(author.trim())) + ) + ); + } + + // Series filtering is now handled by advanced seriesFilter parameter + + if (minWordCount != null || maxWordCount != null) { + logger.info(" Adding word count filter: min={}, max={}", minWordCount, maxWordCount); + boolBuilder.filter(f -> f + .range(r -> { + var rangeBuilder = r.field("wordCount"); + if (minWordCount != null) { + rangeBuilder.gte(JsonData.of(minWordCount)); + } + if (maxWordCount != null) { + rangeBuilder.lte(JsonData.of(maxWordCount)); + } + return rangeBuilder; + }) + ); + } + + if (minRating != null) { + logger.info(" Adding rating filter: min={}", minRating); + boolBuilder.filter(f -> f + .range(r -> r + .field("rating") + .gte(JsonData.of(minRating)) + ) + ); + } + + if (isRead != null) { + logger.info(" Adding isRead filter: {}", isRead); + boolBuilder.filter(f -> f + .term(t -> t + .field("isRead") + .value(FieldValue.of(isRead)) + ) + ); + } + + if (isFavorite != null) { + logger.info(" isFavorite filter requested: {} (FIELD NOT IMPLEMENTED - IGNORING)", isFavorite); + // isFavorite field is not implemented in Story entity or StorySearchDto, so ignore this filter + } + + // Advanced date filters + if (createdAfter != null && !createdAfter.trim().isEmpty() && !"null".equalsIgnoreCase(createdAfter.trim())) { + logger.info(" Adding createdAfter filter: '{}'", createdAfter.trim()); + boolBuilder.filter(f -> f + .range(r -> r + .field("createdAt") + .gte(JsonData.of(createdAfter.trim())) + ) + ); + } + + if (createdBefore != null && !createdBefore.trim().isEmpty() && !"null".equalsIgnoreCase(createdBefore.trim())) { + logger.info(" Adding createdBefore filter: '{}'", createdBefore.trim()); + boolBuilder.filter(f -> f + .range(r -> r + .field("createdAt") + .lte(JsonData.of(createdBefore.trim())) + ) + ); + } + + if (lastReadAfter != null && !lastReadAfter.trim().isEmpty() && !"null".equalsIgnoreCase(lastReadAfter.trim())) { + logger.info(" Adding lastReadAfter filter: '{}'", lastReadAfter.trim()); + boolBuilder.filter(f -> f + .range(r -> r + .field("lastReadAt") + .gte(JsonData.of(lastReadAfter.trim())) + ) + ); + } + + if (lastReadBefore != null && !lastReadBefore.trim().isEmpty() && !"null".equalsIgnoreCase(lastReadBefore.trim())) { + logger.info(" Adding lastReadBefore filter: '{}'", lastReadBefore.trim()); + boolBuilder.filter(f -> f + .range(r -> r + .field("lastReadAt") + .lte(JsonData.of(lastReadBefore.trim())) + ) + ); + } + + // Advanced boolean filters + if (unratedOnly != null && unratedOnly) { + logger.info(" Adding unratedOnly filter"); + boolBuilder.filter(f -> f + .bool(b -> b + .should(s -> s.term(t -> t.field("rating").value(FieldValue.of(0)))) + .should(s -> s.bool(bb -> bb.mustNot(mn -> mn.exists(e -> e.field("rating"))))) + ) + ); + } + + if (hasReadingProgress != null) { + logger.info(" Adding hasReadingProgress filter: {}", hasReadingProgress); + if (hasReadingProgress) { + boolBuilder.filter(f -> f + .range(r -> r + .field("readingPosition") + .gt(JsonData.of(0)) + ) + ); + } else { + boolBuilder.filter(f -> f + .bool(b -> b + .should(s -> s.term(t -> t.field("readingPosition").value(FieldValue.of(0)))) + .should(s -> s.bool(bb -> bb.mustNot(mn -> mn.exists(e -> e.field("readingPosition"))))) + ) + ); + } + } + + if (hasCoverImage != null) { + logger.info(" Adding hasCoverImage filter: {}", hasCoverImage); + if (hasCoverImage) { + boolBuilder.filter(f -> f + .exists(e -> e.field("coverPath")) + ); + } else { + boolBuilder.filter(f -> f + .bool(b -> b + .mustNot(mn -> mn.exists(e -> e.field("coverPath"))) + ) + ); + } + } + + if (sourceDomain != null && !sourceDomain.trim().isEmpty() && !"null".equalsIgnoreCase(sourceDomain.trim())) { + logger.info(" Adding sourceDomain filter: '{}'", sourceDomain.trim()); + boolBuilder.filter(f -> f + .term(t -> t + .field("sourceDomain") + .value(FieldValue.of(sourceDomain.trim())) + ) + ); + } + + // Reading status filter logic + if (readingStatus != null && !readingStatus.trim().isEmpty() && !"null".equalsIgnoreCase(readingStatus.trim()) && !"all".equalsIgnoreCase(readingStatus.trim())) { + logger.info(" Adding readingStatus filter: '{}'", readingStatus.trim()); + if ("unread".equalsIgnoreCase(readingStatus.trim())) { + // Simplified unread test: just check isRead = false + logger.info(" Applying simplified unread filter: isRead = false"); + boolBuilder.filter(f -> f + .term(t -> t.field("isRead").value(FieldValue.of(false))) + ); + } else if ("started".equalsIgnoreCase(readingStatus.trim())) { + // Started: readingPosition > 0 AND isRead = false + boolBuilder.filter(f -> f + .bool(b -> b + .must(m -> m.term(t -> t.field("isRead").value(FieldValue.of(false)))) + .must(m -> m.range(r -> r.field("readingPosition").gt(JsonData.of(0)))) + ) + ); + } else if ("completed".equalsIgnoreCase(readingStatus.trim())) { + // Completed: isRead = true + boolBuilder.filter(f -> f + .term(t -> t.field("isRead").value(FieldValue.of(true))) + ); + } + } + + // Series filter (separate from seriesFilter parameter which is handled above) + if (seriesFilter != null && !seriesFilter.trim().isEmpty() && !"null".equalsIgnoreCase(seriesFilter.trim())) { + logger.info(" Adding advanced seriesFilter: '{}'", seriesFilter.trim()); + if ("standalone".equalsIgnoreCase(seriesFilter.trim())) { + // Stories without series: seriesName field doesn't exist or is null + logger.info(" Applying standalone filter: seriesName field must not exist"); + boolBuilder.filter(f -> f + .bool(b -> b + .mustNot(mn -> mn.exists(e -> e.field("seriesName"))) + ) + ); + } else if ("series".equalsIgnoreCase(seriesFilter.trim())) { + // Stories with series: seriesName field exists and has a value + logger.info(" Applying series filter: seriesName field must exist"); + boolBuilder.filter(f -> f + .exists(e -> e.field("seriesName")) + ); + } + } + + if (minTagCount != null && minTagCount > 0) { + logger.info(" Adding minTagCount filter: {}", minTagCount); + // This is a complex aggregation-based filter, for now we'll implement a simple approach + // In a real implementation, you'd use a script query or aggregation + logger.warn(" minTagCount filter not fully implemented yet"); + } + + if (popularOnly != null && popularOnly) { + logger.info(" Adding popularOnly filter (rating >= 4)"); + boolBuilder.filter(f -> f + .range(r -> r + .field("rating") + .gte(JsonData.of(4.0)) + ) + ); + } + + if (hiddenGemsOnly != null && hiddenGemsOnly) { + logger.info(" Adding hiddenGemsOnly filter (rating >= 4 and wordCount >= 10000)"); + boolBuilder.filter(f -> f + .range(r -> r + .field("rating") + .gte(JsonData.of(4.0)) + ) + ); + boolBuilder.filter(f -> f + .range(r -> r + .field("wordCount") + .gte(JsonData.of(10000)) + ) + ); + } + + BoolQuery boolQuery = boolBuilder.build(); + logger.info(" BoolQuery - must clauses: {}, filter clauses: {}, should clauses: {}, must_not clauses: {}", + boolQuery.must() != null ? boolQuery.must().size() : 0, + boolQuery.filter() != null ? boolQuery.filter().size() : 0, + boolQuery.should() != null ? boolQuery.should().size() : 0, + boolQuery.mustNot() != null ? boolQuery.mustNot().size() : 0); + + searchBuilder.query(q -> q.bool(boolQuery)); + + // Add aggregations for facets + if (facetBy != null && facetBy.contains("tagNames")) { + logger.info(" Adding tagNames aggregation for facet"); + searchBuilder.aggregations("tagNames", a -> a + .terms(t -> t + .field("tagNames") + .size(100) // Limit to top 100 tags + ) + ); + } + + // Sorting + if (sortBy != null && !sortBy.isEmpty()) { + SortOrder order = "desc".equalsIgnoreCase(sortOrder) ? SortOrder.Desc : SortOrder.Asc; + String mappedSortField = mapSortField(sortBy); + logger.info(" Mapped sortBy '{}' to '{}'", sortBy, mappedSortField); + + // Add special handling for nullable date fields + if ("lastReadAt".equals(mappedSortField)) { + logger.info(" Adding special sorting for lastReadAt with null handling"); + searchBuilder.sort(s -> s.field(f -> f + .field(mappedSortField) + .order(order) + .missing(FieldValue.of("_last")) // Put null values last regardless of sort order + )); + } else { + searchBuilder.sort(s -> s.field(f -> f.field(mappedSortField).order(order))); + } + } + + SearchResponse response = openSearchClient.search(searchBuilder.build(), StorySearchDto.class); + + logger.info(" Search response: totalHits={}, hits.size()={}", + response.hits().total() != null ? response.hits().total().value() : 0, + response.hits().hits().size()); + + List results = response.hits().hits().stream() + .map(hit -> hit.source()) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + long totalHits = response.hits().total() != null ? response.hits().total().value() : 0; + + logger.info(" Processed results: count={}, totalHits={}", results.size(), totalHits); + + // Process aggregations if present + Map> facets = new HashMap<>(); + if (response.aggregations() != null && facetBy != null && facetBy.contains("tagNames")) { + var tagAggregation = response.aggregations().get("tagNames"); + if (tagAggregation != null) { + try { + // Try to access as terms aggregation + var termsAgg = tagAggregation._get(); + if (termsAgg instanceof org.opensearch.client.opensearch._types.aggregations.StringTermsAggregate) { + org.opensearch.client.opensearch._types.aggregations.StringTermsAggregate stringTermsAgg = + (org.opensearch.client.opensearch._types.aggregations.StringTermsAggregate) termsAgg; + List tagFacets = stringTermsAgg.buckets().array().stream() + .map(bucket -> new FacetCountDto(bucket.key(), (int) bucket.docCount())) + .collect(Collectors.toList()); + facets.put("tagNames", tagFacets); + logger.info(" Processed tag facets: count={}", tagFacets.size()); + } else { + logger.warn(" Tag aggregation is not a StringTermsAggregate: {}", termsAgg.getClass().getSimpleName()); + } + } catch (Exception e) { + logger.warn(" Failed to process tag aggregation: {}", e.getMessage()); + } + } + } + + return new SearchResultDto<>(results, totalHits, page, size, query != null ? query : "", results.size(), facets); + + } catch (IOException e) { + logger.error("Error searching stories", e); + return new SearchResultDto<>(List.of(), 0, page, size, query != null ? query : "", 0); + } } public List getRandomStories(int count, List tags, String author, String series, Integer minWordCount, Integer maxWordCount, Float minRating, Boolean isRead, Boolean isFavorite, Long seed) { - // TODO: Implement OpenSearch random story selection - logger.warn("OpenSearch random story selection not yet implemented"); - return new ArrayList<>(); + try { + logger.debug("Getting {} random stories with seed: {}", count, seed); + + SearchRequest.Builder searchBuilder = new SearchRequest.Builder() + .index(getStoriesIndex()) + .size(count); + + BoolQuery.Builder boolBuilder = new BoolQuery.Builder(); + + // Add same filters as searchStories + if (tags != null && !tags.isEmpty()) { + boolBuilder.filter(f -> f + .terms(t -> t + .field("tagNames") + .terms(ts -> ts.value(tags.stream().map(FieldValue::of).collect(Collectors.toList()))) + ) + ); + } + + // Use function_score with random_score for reliable randomization + FunctionScoreQuery functionScore = FunctionScoreQuery.of(fs -> fs + .query(q -> q.bool(boolBuilder.build())) + .functions(fn -> fn + .randomScore(rs -> rs.seed(JsonData.of(seed != null ? seed.toString() : String.valueOf(System.currentTimeMillis())))) + .weight(1.0f) + ) + ); + + searchBuilder.query(q -> q.functionScore(functionScore)); + + SearchResponse response = openSearchClient.search(searchBuilder.build(), StorySearchDto.class); + + return response.hits().hits().stream() + .map(hit -> hit.source()) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + } catch (IOException e) { + logger.error("Error getting random stories", e); + return List.of(); + } + } + + public String getRandomStoryId(Long seed) { + List randomStories = getRandomStories(1, null, null, null, null, null, null, null, null, seed); + return randomStories.isEmpty() ? null : randomStories.get(0).getId().toString(); } public List searchAuthors(String query, int limit) { - // TODO: Implement OpenSearch author search - logger.warn("OpenSearch author search not yet implemented"); - return new ArrayList<>(); + try { + logger.debug("Searching authors with query: {}, limit: {}", query, limit); + + SearchRequest.Builder searchBuilder = new SearchRequest.Builder() + .index(getAuthorsIndex()) + .size(limit); + + if (query != null && !query.trim().isEmpty()) { + searchBuilder.query(q -> q + .multiMatch(mm -> mm + .query(query.trim()) + .fields("name^2", "bio") + .type(TextQueryType.BestFields) + ) + ); + } else { + searchBuilder.query(q -> q.matchAll(ma -> ma)); + } + + searchBuilder.sort(s -> s.field(f -> f.field("name").order(SortOrder.Asc))); + + SearchResponse response = openSearchClient.search(searchBuilder.build(), AuthorSearchDto.class); + + return response.hits().hits().stream() + .map(hit -> hit.source()) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + } catch (IOException e) { + logger.error("Error searching authors", e); + return List.of(); + } } public List getTagSuggestions(String query, int limit) { - // TODO: Implement OpenSearch tag autocomplete - logger.warn("OpenSearch tag autocomplete not yet implemented"); - return new ArrayList<>(); + try { + logger.debug("Getting tag suggestions for query: {}, limit: {}", query, limit); + + SearchRequest.Builder searchBuilder = new SearchRequest.Builder() + .index(getStoriesIndex()) + .size(0); + + if (query != null && !query.trim().isEmpty()) { + searchBuilder.query(q -> q + .wildcard(w -> w + .field("tagNames") + .value(query.trim().toLowerCase() + "*") + ) + ); + } else { + searchBuilder.query(q -> q.matchAll(ma -> ma)); + } + + searchBuilder.aggregations("tags", a -> a + .terms(t -> t + .field("tagNames") + .size(limit) + ) + ); + + SearchResponse response = openSearchClient.search(searchBuilder.build(), StorySearchDto.class); + + if (response.aggregations() != null) { + var tagsAgg = response.aggregations().get("tags"); + if (tagsAgg != null && tagsAgg.isSterms()) { + return tagsAgg.sterms().buckets().array().stream() + .map(bucket -> bucket.key()) + .collect(Collectors.toList()); + } + } + + return List.of(); + + } catch (IOException e) { + logger.error("Error getting tag suggestions", e); + return List.of(); + } + } + + // =============================== + // UTILITY METHODS + // =============================== + + private boolean isAvailable() { + return openSearchClient != null; + } + + /** + * Public method to recreate indices (for admin operations) + */ + public void recreateIndices() throws IOException { + if (!isAvailable()) { + logger.warn("OpenSearch client not available - cannot recreate indices"); + return; + } + + logger.info("Recreating OpenSearch indices..."); + + // Delete existing indices if they exist + deleteIndexIfExists(getStoriesIndex()); + deleteIndexIfExists(getAuthorsIndex()); + + // Create fresh indices + createStoriesIndex(); + createAuthorsIndex(); + + logger.info("OpenSearch indices recreated successfully"); + } + + private void deleteIndexIfExists(String indexName) throws IOException { + if (indexExists(indexName)) { + logger.info("Deleting existing index: {}", indexName); + openSearchClient.indices().delete(d -> d.index(indexName)); + } + } + + public boolean testConnection() { + if (!isAvailable()) { + logger.debug("OpenSearch client not available for connection test"); + return false; + } + + try { + openSearchClient.cluster().health(); + return true; + } catch (Exception e) { + logger.error("OpenSearch connection test failed", e); + return false; + } + } + + // =============================== + // CONVERSION METHODS + // =============================== + + private StorySearchDto convertToStorySearchDto(Story story) { + StorySearchDto dto = new StorySearchDto(); + dto.setId(story.getId()); + dto.setTitle(story.getTitle()); + dto.setDescription(story.getDescription()); + dto.setCoverPath(story.getCoverPath()); + dto.setWordCount(story.getWordCount()); + dto.setRating(story.getRating()); + dto.setIsRead(story.getIsRead() != null ? story.getIsRead() : false); + dto.setReadingPosition(story.getReadingPosition() != null ? story.getReadingPosition() : 0); + dto.setLastReadAt(story.getLastReadAt()); // null for unread stories - should be handled by OpenSearch + dto.setCreatedAt(story.getCreatedAt()); + dto.setUpdatedAt(story.getUpdatedAt()); + dto.setDateAdded(story.getCreatedAt()); // Alias for frontend compatibility + + // Debug logging for date fields (only log for first few stories to avoid spam) + if (story.getId().toString().endsWith("1") || story.getId().toString().endsWith("2")) { + logger.debug("Story {} date fields: createdAt={}, lastReadAt={}, updatedAt={}", + story.getId(), story.getCreatedAt(), story.getLastReadAt(), story.getUpdatedAt()); + } + + // Handle author + if (story.getAuthor() != null) { + dto.setAuthorName(story.getAuthor().getName()); + dto.setAuthorId(story.getAuthor().getId()); + } + + // Handle series + if (story.getSeries() != null) { + dto.setSeriesName(story.getSeries().getName()); + dto.setSeriesId(story.getSeries().getId()); + dto.setVolume(story.getVolume()); + } + + // Handle tags + if (story.getTags() != null) { + dto.setTagNames(story.getTags().stream() + .map(tag -> tag.getName()) + .collect(Collectors.toList())); + } else { + dto.setTagNames(List.of()); + } + + return dto; + } + + private AuthorSearchDto convertToAuthorSearchDto(Author author) { + AuthorSearchDto dto = new AuthorSearchDto(); + dto.setId(author.getId()); + dto.setName(author.getName()); + dto.setNotes(author.getNotes()); + dto.setAuthorRating(author.getAuthorRating()); + dto.setUrls(author.getUrls()); + dto.setAvatarImagePath(author.getAvatarImagePath()); + dto.setCreatedAt(author.getCreatedAt()); + dto.setUpdatedAt(author.getUpdatedAt()); + + // Calculate story count and average rating from stories + if (author.getStories() != null) { + dto.setStoryCount(author.getStories().size()); + OptionalDouble avgRating = author.getStories().stream() + .filter(story -> story.getRating() != null) + .mapToInt(Story::getRating) + .average(); + dto.setAverageStoryRating(avgRating.isPresent() ? avgRating.getAsDouble() : null); + } else { + dto.setStoryCount(0); + dto.setAverageStoryRating(null); + } + + return dto; + } + + /** + * Map frontend sort field names to OpenSearch document field names + */ + private String mapSortField(String frontendField) { + switch (frontendField.toLowerCase()) { + case "title": + logger.debug("Mapping title sort to title.keyword field for sorting"); + return "title.keyword"; + case "author": + case "authorname": + logger.debug("Mapping author sort to authorName field"); + return "authorName"; + case "createdat": + case "created_at": + case "date": + case "dateadded": + case "date_added": + logger.debug("Mapping date sort to createdAt field"); + return "createdAt"; + case "lastread": + case "last_read": + case "lastreadat": + case "last_read_at": + logger.debug("Mapping lastRead sort to lastReadAt field"); + return "lastReadAt"; + case "updatedat": + case "updated_at": + return "updatedAt"; + case "wordcount": + case "word_count": + return "wordCount"; + case "rating": + return "rating"; + case "series": + case "seriesname": + return "seriesName"; + default: + return frontendField; + } } } \ No newline at end of file diff --git a/backend/src/main/java/com/storycove/service/SearchMigrationManager.java b/backend/src/main/java/com/storycove/service/SearchMigrationManager.java new file mode 100644 index 0000000..e2a7219 --- /dev/null +++ b/backend/src/main/java/com/storycove/service/SearchMigrationManager.java @@ -0,0 +1,473 @@ +package com.storycove.service; + +import com.storycove.dto.AuthorSearchDto; +import com.storycove.dto.SearchResultDto; +import com.storycove.dto.StorySearchDto; +import com.storycove.entity.Author; +import com.storycove.entity.Story; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.UUID; + +/** + * TEMPORARY MIGRATION MANAGER - DELETE THIS ENTIRE CLASS WHEN TYPESENSE IS REMOVED + * + * This class handles dual-write functionality and engine switching during the + * migration from Typesense to OpenSearch. It's designed to be completely removed + * once the migration is complete. + * + * CLEANUP INSTRUCTIONS: + * 1. Delete this entire file: SearchMigrationManager.java + * 2. Update SearchServiceAdapter to call OpenSearchService directly + * 3. Remove migration-related configuration properties + * 4. Remove migration-related admin endpoints and UI + */ +@Component +public class SearchMigrationManager { + + private static final Logger logger = LoggerFactory.getLogger(SearchMigrationManager.class); + + @Autowired(required = false) + private TypesenseService typesenseService; + + @Autowired(required = false) + private OpenSearchService openSearchService; + + @Value("${storycove.search.engine:typesense}") + private String primaryEngine; + + @Value("${storycove.search.dual-write:false}") + private boolean dualWrite; + + // =============================== + // READ OPERATIONS (single engine) + // =============================== + + public SearchResultDto searchStories(String query, List tags, String author, + String series, Integer minWordCount, Integer maxWordCount, + Float minRating, Boolean isRead, Boolean isFavorite, + String sortBy, String sortOrder, int page, int size, + List facetBy, + // Advanced filters + String createdAfter, String createdBefore, + String lastReadAfter, String lastReadBefore, + Boolean unratedOnly, String readingStatus, + Boolean hasReadingProgress, Boolean hasCoverImage, + String sourceDomain, String seriesFilter, + Integer minTagCount, Boolean popularOnly, + Boolean hiddenGemsOnly) { + boolean openSearchAvailable = openSearchService != null; + boolean openSearchConnected = openSearchAvailable ? openSearchService.testConnection() : false; + boolean routingCondition = "opensearch".equalsIgnoreCase(primaryEngine) && openSearchAvailable; + + logger.info("SEARCH ROUTING DEBUG:"); + logger.info(" Primary engine: '{}'", primaryEngine); + logger.info(" OpenSearch available: {}", openSearchAvailable); + logger.info(" OpenSearch connected: {}", openSearchConnected); + logger.info(" Routing condition result: {}", routingCondition); + logger.info(" Will route to: {}", routingCondition ? "OpenSearch" : "Typesense"); + + if (routingCondition) { + logger.info("ROUTING TO OPENSEARCH"); + return openSearchService.searchStories(query, tags, author, series, minWordCount, maxWordCount, + minRating, isRead, isFavorite, sortBy, sortOrder, page, size, facetBy, + createdAfter, createdBefore, lastReadAfter, lastReadBefore, unratedOnly, readingStatus, + hasReadingProgress, hasCoverImage, sourceDomain, seriesFilter, minTagCount, popularOnly, + hiddenGemsOnly); + } else if (typesenseService != null) { + logger.info("ROUTING TO TYPESENSE"); + // Convert parameters to match TypesenseService signature + return typesenseService.searchStories( + query, page, size, tags, null, minWordCount, maxWordCount, + null, null, null, null, minRating != null ? minRating.intValue() : null, + null, null, sortBy, sortOrder, null, null, isRead, isFavorite, + author, series, null, null, null); + } else { + logger.error("No search service available! Primary engine: {}, OpenSearch: {}, Typesense: {}", + primaryEngine, openSearchService != null, typesenseService != null); + return new SearchResultDto<>(List.of(), 0, page, size, query != null ? query : "", 0); + } + } + + public List getRandomStories(int count, List tags, String author, + String series, Integer minWordCount, Integer maxWordCount, + Float minRating, Boolean isRead, Boolean isFavorite, + Long seed) { + logger.debug("Getting random stories using primary engine: {}", primaryEngine); + + if ("opensearch".equalsIgnoreCase(primaryEngine) && openSearchService != null) { + return openSearchService.getRandomStories(count, tags, author, series, minWordCount, maxWordCount, + minRating, isRead, isFavorite, seed); + } else if (typesenseService != null) { + // TypesenseService doesn't have getRandomStories, use random story ID approach + List results = new java.util.ArrayList<>(); + for (int i = 0; i < count; i++) { + var randomId = typesenseService.getRandomStoryId(null, tags, seed != null ? seed + i : null); + // Note: This is a simplified approach - full implementation would need story lookup + } + return results; + } else { + logger.error("No search service available for random stories"); + return List.of(); + } + } + + public String getRandomStoryId(Long seed) { + logger.debug("Getting random story ID using primary engine: {}", primaryEngine); + + if ("opensearch".equalsIgnoreCase(primaryEngine) && openSearchService != null) { + return openSearchService.getRandomStoryId(seed); + } else if (typesenseService != null) { + var randomId = typesenseService.getRandomStoryId(null, null, seed); + return randomId.map(UUID::toString).orElse(null); + } else { + logger.error("No search service available for random story ID"); + return null; + } + } + + public List searchAuthors(String query, int limit) { + logger.debug("Searching authors using primary engine: {}", primaryEngine); + + if ("opensearch".equalsIgnoreCase(primaryEngine) && openSearchService != null) { + return openSearchService.searchAuthors(query, limit); + } else if (typesenseService != null) { + var result = typesenseService.searchAuthors(query, 0, limit, null, null); + return result.getResults(); + } else { + logger.error("No search service available for author search"); + return List.of(); + } + } + + public List getTagSuggestions(String query, int limit) { + logger.debug("Getting tag suggestions using primary engine: {}", primaryEngine); + + if ("opensearch".equalsIgnoreCase(primaryEngine) && openSearchService != null) { + return openSearchService.getTagSuggestions(query, limit); + } else if (typesenseService != null) { + // TypesenseService may not have getTagSuggestions - return empty for now + logger.warn("Tag suggestions not implemented for Typesense"); + return List.of(); + } else { + logger.error("No search service available for tag suggestions"); + return List.of(); + } + } + + // =============================== + // WRITE OPERATIONS (dual-write capable) + // =============================== + + public void indexStory(Story story) { + logger.debug("Indexing story with dual-write: {}, primary engine: {}", dualWrite, primaryEngine); + + // Write to OpenSearch + if ("opensearch".equalsIgnoreCase(primaryEngine) || dualWrite) { + if (openSearchService != null) { + try { + openSearchService.indexStory(story); + logger.debug("Successfully indexed story {} in OpenSearch", story.getId()); + } catch (Exception e) { + logger.error("Failed to index story {} in OpenSearch", story.getId(), e); + } + } else { + logger.warn("OpenSearch service not available for indexing story {}", story.getId()); + } + } + + // Write to Typesense + if ("typesense".equalsIgnoreCase(primaryEngine) || dualWrite) { + if (typesenseService != null) { + try { + typesenseService.indexStory(story); + logger.debug("Successfully indexed story {} in Typesense", story.getId()); + } catch (Exception e) { + logger.error("Failed to index story {} in Typesense", story.getId(), e); + } + } else { + logger.warn("Typesense service not available for indexing story {}", story.getId()); + } + } + } + + public void updateStory(Story story) { + logger.debug("Updating story with dual-write: {}, primary engine: {}", dualWrite, primaryEngine); + + // Update in OpenSearch + if ("opensearch".equalsIgnoreCase(primaryEngine) || dualWrite) { + if (openSearchService != null) { + try { + openSearchService.updateStory(story); + logger.debug("Successfully updated story {} in OpenSearch", story.getId()); + } catch (Exception e) { + logger.error("Failed to update story {} in OpenSearch", story.getId(), e); + } + } + } + + // Update in Typesense + if ("typesense".equalsIgnoreCase(primaryEngine) || dualWrite) { + if (typesenseService != null) { + try { + typesenseService.updateStory(story); + logger.debug("Successfully updated story {} in Typesense", story.getId()); + } catch (Exception e) { + logger.error("Failed to update story {} in Typesense", story.getId(), e); + } + } + } + } + + public void deleteStory(UUID storyId) { + logger.debug("Deleting story with dual-write: {}, primary engine: {}", dualWrite, primaryEngine); + + // Delete from OpenSearch + if ("opensearch".equalsIgnoreCase(primaryEngine) || dualWrite) { + if (openSearchService != null) { + try { + openSearchService.deleteStory(storyId); + logger.debug("Successfully deleted story {} from OpenSearch", storyId); + } catch (Exception e) { + logger.error("Failed to delete story {} from OpenSearch", storyId, e); + } + } + } + + // Delete from Typesense + if ("typesense".equalsIgnoreCase(primaryEngine) || dualWrite) { + if (typesenseService != null) { + try { + typesenseService.deleteStory(storyId.toString()); + logger.debug("Successfully deleted story {} from Typesense", storyId); + } catch (Exception e) { + logger.error("Failed to delete story {} from Typesense", storyId, e); + } + } + } + } + + public void indexAuthor(Author author) { + logger.debug("Indexing author with dual-write: {}, primary engine: {}", dualWrite, primaryEngine); + + // Index in OpenSearch + if ("opensearch".equalsIgnoreCase(primaryEngine) || dualWrite) { + if (openSearchService != null) { + try { + openSearchService.indexAuthor(author); + logger.debug("Successfully indexed author {} in OpenSearch", author.getId()); + } catch (Exception e) { + logger.error("Failed to index author {} in OpenSearch", author.getId(), e); + } + } + } + + // Index in Typesense + if ("typesense".equalsIgnoreCase(primaryEngine) || dualWrite) { + if (typesenseService != null) { + try { + typesenseService.indexAuthor(author); + logger.debug("Successfully indexed author {} in Typesense", author.getId()); + } catch (Exception e) { + logger.error("Failed to index author {} in Typesense", author.getId(), e); + } + } + } + } + + public void updateAuthor(Author author) { + logger.debug("Updating author with dual-write: {}, primary engine: {}", dualWrite, primaryEngine); + + // Update in OpenSearch + if ("opensearch".equalsIgnoreCase(primaryEngine) || dualWrite) { + if (openSearchService != null) { + try { + openSearchService.updateAuthor(author); + logger.debug("Successfully updated author {} in OpenSearch", author.getId()); + } catch (Exception e) { + logger.error("Failed to update author {} in OpenSearch", author.getId(), e); + } + } + } + + // Update in Typesense + if ("typesense".equalsIgnoreCase(primaryEngine) || dualWrite) { + if (typesenseService != null) { + try { + typesenseService.updateAuthor(author); + logger.debug("Successfully updated author {} in Typesense", author.getId()); + } catch (Exception e) { + logger.error("Failed to update author {} in Typesense", author.getId(), e); + } + } + } + } + + public void deleteAuthor(UUID authorId) { + logger.debug("Deleting author with dual-write: {}, primary engine: {}", dualWrite, primaryEngine); + + // Delete from OpenSearch + if ("opensearch".equalsIgnoreCase(primaryEngine) || dualWrite) { + if (openSearchService != null) { + try { + openSearchService.deleteAuthor(authorId); + logger.debug("Successfully deleted author {} from OpenSearch", authorId); + } catch (Exception e) { + logger.error("Failed to delete author {} from OpenSearch", authorId, e); + } + } + } + + // Delete from Typesense + if ("typesense".equalsIgnoreCase(primaryEngine) || dualWrite) { + if (typesenseService != null) { + try { + typesenseService.deleteAuthor(authorId.toString()); + logger.debug("Successfully deleted author {} from Typesense", authorId); + } catch (Exception e) { + logger.error("Failed to delete author {} from Typesense", authorId, e); + } + } + } + } + + public void bulkIndexStories(List stories) { + logger.debug("Bulk indexing {} stories with dual-write: {}, primary engine: {}", + stories.size(), dualWrite, primaryEngine); + + // Bulk index in OpenSearch + if ("opensearch".equalsIgnoreCase(primaryEngine) || dualWrite) { + if (openSearchService != null) { + try { + openSearchService.bulkIndexStories(stories); + logger.info("Successfully bulk indexed {} stories in OpenSearch", stories.size()); + } catch (Exception e) { + logger.error("Failed to bulk index {} stories in OpenSearch", stories.size(), e); + } + } + } + + // Bulk index in Typesense + if ("typesense".equalsIgnoreCase(primaryEngine) || dualWrite) { + if (typesenseService != null) { + try { + typesenseService.bulkIndexStories(stories); + logger.info("Successfully bulk indexed {} stories in Typesense", stories.size()); + } catch (Exception e) { + logger.error("Failed to bulk index {} stories in Typesense", stories.size(), e); + } + } + } + } + + public void bulkIndexAuthors(List authors) { + logger.debug("Bulk indexing {} authors with dual-write: {}, primary engine: {}", + authors.size(), dualWrite, primaryEngine); + + // Bulk index in OpenSearch + if ("opensearch".equalsIgnoreCase(primaryEngine) || dualWrite) { + if (openSearchService != null) { + try { + openSearchService.bulkIndexAuthors(authors); + logger.info("Successfully bulk indexed {} authors in OpenSearch", authors.size()); + } catch (Exception e) { + logger.error("Failed to bulk index {} authors in OpenSearch", authors.size(), e); + } + } + } + + // Bulk index in Typesense + if ("typesense".equalsIgnoreCase(primaryEngine) || dualWrite) { + if (typesenseService != null) { + try { + typesenseService.bulkIndexAuthors(authors); + logger.info("Successfully bulk indexed {} authors in Typesense", authors.size()); + } catch (Exception e) { + logger.error("Failed to bulk index {} authors in Typesense", authors.size(), e); + } + } + } + } + + // =============================== + // UTILITY METHODS + // =============================== + + public boolean isSearchServiceAvailable() { + if ("opensearch".equalsIgnoreCase(primaryEngine)) { + return openSearchService != null && openSearchService.testConnection(); + } else { + return typesenseService != null; + } + } + + public String getCurrentSearchEngine() { + return primaryEngine; + } + + public boolean isDualWriteEnabled() { + return dualWrite; + } + + public boolean canSwitchToOpenSearch() { + return openSearchService != null && openSearchService.testConnection(); + } + + public boolean canSwitchToTypesense() { + return typesenseService != null; + } + + public OpenSearchService getOpenSearchService() { + return openSearchService; + } + + /** + * Update configuration at runtime (for admin interface) + * Note: This requires @RefreshScope to work properly + */ + public void updateConfiguration(String engine, boolean enableDualWrite) { + logger.info("Updating search configuration: engine={}, dualWrite={}", engine, enableDualWrite); + this.primaryEngine = engine; + this.dualWrite = enableDualWrite; + } + + /** + * Get current configuration status for admin interface + */ + public SearchMigrationStatus getStatus() { + return new SearchMigrationStatus( + primaryEngine, + dualWrite, + typesenseService != null, + openSearchService != null && openSearchService.testConnection() + ); + } + + /** + * DTO for search migration status + */ + public static class SearchMigrationStatus { + private final String primaryEngine; + private final boolean dualWrite; + private final boolean typesenseAvailable; + private final boolean openSearchAvailable; + + public SearchMigrationStatus(String primaryEngine, boolean dualWrite, + boolean typesenseAvailable, boolean openSearchAvailable) { + this.primaryEngine = primaryEngine; + this.dualWrite = dualWrite; + this.typesenseAvailable = typesenseAvailable; + this.openSearchAvailable = openSearchAvailable; + } + + public String getPrimaryEngine() { return primaryEngine; } + public boolean isDualWrite() { return dualWrite; } + public boolean isTypesenseAvailable() { return typesenseAvailable; } + public boolean isOpenSearchAvailable() { return openSearchAvailable; } + } +} \ No newline at end of file diff --git a/backend/src/main/java/com/storycove/service/SearchServiceAdapter.java b/backend/src/main/java/com/storycove/service/SearchServiceAdapter.java new file mode 100644 index 0000000..a8c2e7c --- /dev/null +++ b/backend/src/main/java/com/storycove/service/SearchServiceAdapter.java @@ -0,0 +1,196 @@ +package com.storycove.service; + +import com.storycove.dto.AuthorSearchDto; +import com.storycove.dto.SearchResultDto; +import com.storycove.dto.StorySearchDto; +import com.storycove.entity.Author; +import com.storycove.entity.Story; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.UUID; + +/** + * Service adapter that provides a unified interface for search operations. + * + * This adapter delegates to SearchMigrationManager during the migration period, + * which will be removed once Typesense is completely eliminated. + * + * POST-MIGRATION: This class will be simplified to call OpenSearchService directly. + */ +@Service +public class SearchServiceAdapter { + + private static final Logger logger = LoggerFactory.getLogger(SearchServiceAdapter.class); + + @Autowired + private SearchMigrationManager migrationManager; + + // =============================== + // SEARCH OPERATIONS + // =============================== + + /** + * Search stories with unified interface + */ + public SearchResultDto searchStories(String query, List tags, String author, + String series, Integer minWordCount, Integer maxWordCount, + Float minRating, Boolean isRead, Boolean isFavorite, + String sortBy, String sortOrder, int page, int size, + List facetBy, + // Advanced filters + String createdAfter, String createdBefore, + String lastReadAfter, String lastReadBefore, + Boolean unratedOnly, String readingStatus, + Boolean hasReadingProgress, Boolean hasCoverImage, + String sourceDomain, String seriesFilter, + Integer minTagCount, Boolean popularOnly, + Boolean hiddenGemsOnly) { + return migrationManager.searchStories(query, tags, author, series, minWordCount, maxWordCount, + minRating, isRead, isFavorite, sortBy, sortOrder, page, size, facetBy, + createdAfter, createdBefore, lastReadAfter, lastReadBefore, unratedOnly, readingStatus, + hasReadingProgress, hasCoverImage, sourceDomain, seriesFilter, minTagCount, popularOnly, + hiddenGemsOnly); + } + + /** + * Get random stories with unified interface + */ + public List getRandomStories(int count, List tags, String author, + String series, Integer minWordCount, Integer maxWordCount, + Float minRating, Boolean isRead, Boolean isFavorite, + Long seed) { + return migrationManager.getRandomStories(count, tags, author, series, minWordCount, maxWordCount, + minRating, isRead, isFavorite, seed); + } + + /** + * Get random story ID with unified interface + */ + public String getRandomStoryId(Long seed) { + return migrationManager.getRandomStoryId(seed); + } + + /** + * Search authors with unified interface + */ + public List searchAuthors(String query, int limit) { + return migrationManager.searchAuthors(query, limit); + } + + /** + * Get tag suggestions with unified interface + */ + public List getTagSuggestions(String query, int limit) { + return migrationManager.getTagSuggestions(query, limit); + } + + // =============================== + // INDEX OPERATIONS + // =============================== + + /** + * Index a story with unified interface (supports dual-write) + */ + public void indexStory(Story story) { + migrationManager.indexStory(story); + } + + /** + * Update a story in the index with unified interface (supports dual-write) + */ + public void updateStory(Story story) { + migrationManager.updateStory(story); + } + + /** + * Delete a story from the index with unified interface (supports dual-write) + */ + public void deleteStory(UUID storyId) { + migrationManager.deleteStory(storyId); + } + + /** + * Index an author with unified interface (supports dual-write) + */ + public void indexAuthor(Author author) { + migrationManager.indexAuthor(author); + } + + /** + * Update an author in the index with unified interface (supports dual-write) + */ + public void updateAuthor(Author author) { + migrationManager.updateAuthor(author); + } + + /** + * Delete an author from the index with unified interface (supports dual-write) + */ + public void deleteAuthor(UUID authorId) { + migrationManager.deleteAuthor(authorId); + } + + /** + * Bulk index stories with unified interface (supports dual-write) + */ + public void bulkIndexStories(List stories) { + migrationManager.bulkIndexStories(stories); + } + + /** + * Bulk index authors with unified interface (supports dual-write) + */ + public void bulkIndexAuthors(List authors) { + migrationManager.bulkIndexAuthors(authors); + } + + // =============================== + // UTILITY METHODS + // =============================== + + /** + * Check if search service is available and healthy + */ + public boolean isSearchServiceAvailable() { + return migrationManager.isSearchServiceAvailable(); + } + + /** + * Get current search engine name + */ + public String getCurrentSearchEngine() { + return migrationManager.getCurrentSearchEngine(); + } + + /** + * Check if dual-write is enabled + */ + public boolean isDualWriteEnabled() { + return migrationManager.isDualWriteEnabled(); + } + + /** + * Check if we can switch to OpenSearch + */ + public boolean canSwitchToOpenSearch() { + return migrationManager.canSwitchToOpenSearch(); + } + + /** + * Check if we can switch to Typesense + */ + public boolean canSwitchToTypesense() { + return migrationManager.canSwitchToTypesense(); + } + + /** + * Get current migration status for admin interface + */ + public SearchMigrationManager.SearchMigrationStatus getMigrationStatus() { + return migrationManager.getStatus(); + } +} \ No newline at end of file diff --git a/backend/src/main/resources/application.yml b/backend/src/main/resources/application.yml index 8f1d025..fa80a51 100644 --- a/backend/src/main/resources/application.yml +++ b/backend/src/main/resources/application.yml @@ -19,6 +19,12 @@ spring: max-file-size: 256MB # Increased for backup restore max-request-size: 260MB # Slightly higher to account for form data + jackson: + serialization: + write-dates-as-timestamps: false + deserialization: + adjust-dates-to-context-time-zone: false + server: port: 8080 @@ -34,6 +40,7 @@ storycove: password: ${APP_PASSWORD} # REQUIRED: No default password for security search: engine: ${SEARCH_ENGINE:typesense} # typesense or opensearch + dual-write: ${SEARCH_DUAL_WRITE:false} # enable dual-write during migration typesense: api-key: ${TYPESENSE_API_KEY:xyz} host: ${TYPESENSE_HOST:localhost} @@ -44,9 +51,9 @@ storycove: # Connection settings host: ${OPENSEARCH_HOST:localhost} port: ${OPENSEARCH_PORT:9200} - scheme: ${OPENSEARCH_SCHEME:https} - username: ${OPENSEARCH_USERNAME:admin} - password: ${OPENSEARCH_PASSWORD} # REQUIRED when using OpenSearch + scheme: ${OPENSEARCH_SCHEME:http} + username: ${OPENSEARCH_USERNAME:} + password: ${OPENSEARCH_PASSWORD:} # Empty when security is disabled # Environment-specific configuration profile: ${SPRING_PROFILES_ACTIVE:development} # development, staging, production diff --git a/cookies.txt b/cookies.txt index c31d989..e63421a 100644 --- a/cookies.txt +++ b/cookies.txt @@ -2,3 +2,4 @@ # https://curl.se/docs/http-cookies.html # This file was generated by libcurl! Edit at your own risk. +#HttpOnly_localhost FALSE / FALSE 1758433252 token eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiJ1c2VyIiwiaWF0IjoxNzU4MzQ2ODUyLCJleHAiOjE3NTg0MzMyNTIsImxpYnJhcnlJZCI6InNlY3JldCJ9.zEAQT5_11-pxPxmIhufSQqE26hvHldde4kFNE2HWWgBa5lT_Wt7jwpoPUMkQGQfShQwDZ9N-hFX3R2ew8jD7WQ diff --git a/docker-compose.yml b/docker-compose.yml index ffed8ed..2c5e021 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,9 +39,8 @@ services: - TYPESENSE_PORT=8108 - OPENSEARCH_HOST=opensearch - OPENSEARCH_PORT=9200 - - OPENSEARCH_USERNAME=${OPENSEARCH_USERNAME:-admin} - - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD} - - SEARCH_ENGINE=${SEARCH_ENGINE:-typesense} + - OPENSEARCH_SCHEME=http + - SEARCH_ENGINE=${SEARCH_ENGINE:-opensearch} - IMAGE_STORAGE_PATH=/app/images - APP_PASSWORD=${APP_PASSWORD} - STORYCOVE_CORS_ALLOWED_ORIGINS=${STORYCOVE_CORS_ALLOWED_ORIGINS:-http://localhost:3000,http://localhost:6925} @@ -87,11 +86,10 @@ services: - cluster.name=storycove-opensearch - node.name=opensearch-node - discovery.type=single-node - - bootstrap.memory_lock=true - - "OPENSEARCH_JAVA_OPTS=-Xms256m -Xmx256m" + - bootstrap.memory_lock=false + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - "DISABLE_INSTALL_DEMO_CONFIG=true" - - "DISABLE_SECURITY_PLUGIN=false" - - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD}" + - "DISABLE_SECURITY_PLUGIN=true" ulimits: memlock: soft: -1 @@ -103,15 +101,15 @@ services: - opensearch_data:/usr/share/opensearch/data networks: - storycove-network + restart: unless-stopped opensearch-dashboards: image: opensearchproject/opensearch-dashboards:3.2.0 - # No port mapping - only accessible within the Docker network + ports: + - "5601:5601" # Expose OpenSearch Dashboard environment: - - OPENSEARCH_HOSTS=https://opensearch:9200 - - "OPENSEARCH_USERNAME=${OPENSEARCH_USERNAME:-admin}" - - "OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}" - - "DISABLE_SECURITY_DASHBOARDS_PLUGIN=false" + - OPENSEARCH_HOSTS=http://opensearch:9200 + - "DISABLE_SECURITY_DASHBOARDS_PLUGIN=true" depends_on: - opensearch networks: diff --git a/frontend/src/app/stories/[id]/page.tsx b/frontend/src/app/stories/[id]/page.tsx index d52bb03..27adafa 100644 --- a/frontend/src/app/stories/[id]/page.tsx +++ b/frontend/src/app/stories/[id]/page.tsx @@ -49,7 +49,7 @@ export default function StoryReadingPage() { )); // Convert to character position in the plain text content - const textLength = story.contentPlain?.length || story.contentHtml.length; + const textLength = story.contentPlain?.length || story.contentHtml?.length || 0; return Math.floor(scrollRatio * textLength); }, [story]); @@ -57,7 +57,7 @@ export default function StoryReadingPage() { const calculateReadingPercentage = useCallback((currentPosition: number): number => { if (!story) return 0; - const totalLength = story.contentPlain?.length || story.contentHtml.length; + const totalLength = story.contentPlain?.length || story.contentHtml?.length || 0; if (totalLength === 0) return 0; return Math.round((currentPosition / totalLength) * 100); @@ -67,7 +67,7 @@ export default function StoryReadingPage() { const scrollToCharacterPosition = useCallback((position: number) => { if (!contentRef.current || !story || hasScrolledToPosition) return; - const textLength = story.contentPlain?.length || story.contentHtml.length; + const textLength = story.contentPlain?.length || story.contentHtml?.length || 0; if (textLength === 0 || position === 0) return; const ratio = position / textLength; diff --git a/frontend/src/components/collections/CollectionReadingView.tsx b/frontend/src/components/collections/CollectionReadingView.tsx index 8d0d1f8..15ec6d9 100644 --- a/frontend/src/components/collections/CollectionReadingView.tsx +++ b/frontend/src/components/collections/CollectionReadingView.tsx @@ -40,7 +40,7 @@ export default function CollectionReadingView({ )); // Convert to character position in the plain text content - const textLength = story.contentPlain?.length || story.contentHtml.length; + const textLength = story.contentPlain?.length || story.contentHtml?.length || 0; return Math.floor(scrollRatio * textLength); }, [story]); @@ -48,7 +48,7 @@ export default function CollectionReadingView({ const calculateReadingPercentage = useCallback((currentPosition: number): number => { if (!story) return 0; - const totalLength = story.contentPlain?.length || story.contentHtml.length; + const totalLength = story.contentPlain?.length || story.contentHtml?.length || 0; if (totalLength === 0) return 0; return Math.round((currentPosition / totalLength) * 100); @@ -58,7 +58,7 @@ export default function CollectionReadingView({ const scrollToCharacterPosition = useCallback((position: number) => { if (!contentRef.current || !story || hasScrolledToPosition) return; - const textLength = story.contentPlain?.length || story.contentHtml.length; + const textLength = story.contentPlain?.length || story.contentHtml?.length || 0; if (textLength === 0 || position === 0) return; const ratio = position / textLength; diff --git a/frontend/src/components/library/AdvancedFilters.tsx b/frontend/src/components/library/AdvancedFilters.tsx index 0c01a57..b850b1f 100644 --- a/frontend/src/components/library/AdvancedFilters.tsx +++ b/frontend/src/components/library/AdvancedFilters.tsx @@ -127,29 +127,6 @@ const FILTER_PRESETS: FilterPreset[] = [ description: 'Stories that are part of a series', filters: { seriesFilter: 'series' }, category: 'content' - }, - - // Organization presets - { - id: 'well-tagged', - label: '3+ tags', - description: 'Well-tagged stories with 3 or more tags', - filters: { minTagCount: 3 }, - category: 'organization' - }, - { - id: 'popular', - label: 'Popular', - description: 'Stories with above-average ratings', - filters: { popularOnly: true }, - category: 'organization' - }, - { - id: 'hidden-gems', - label: 'Hidden Gems', - description: 'Underrated or unrated stories to discover', - filters: { hiddenGemsOnly: true }, - category: 'organization' } ]; diff --git a/frontend/src/components/settings/SystemSettings.tsx b/frontend/src/components/settings/SystemSettings.tsx index 746e097..64624b7 100644 --- a/frontend/src/components/settings/SystemSettings.tsx +++ b/frontend/src/components/settings/SystemSettings.tsx @@ -1,14 +1,39 @@ 'use client'; -import { useState } from 'react'; +import React, { useState, useEffect } from 'react'; import Button from '../ui/Button'; -import { storyApi, authorApi, databaseApi, configApi } from '../../lib/api'; +import { storyApi, authorApi, databaseApi, configApi, searchAdminApi } from '../../lib/api'; interface SystemSettingsProps { // No props needed - this component manages its own state } export default function SystemSettings({}: SystemSettingsProps) { + const [searchEngineStatus, setSearchEngineStatus] = useState<{ + currentEngine: string; + dualWrite: boolean; + typesenseAvailable: boolean; + openSearchAvailable: boolean; + loading: boolean; + message: string; + success?: boolean; + }>({ + currentEngine: 'typesense', + dualWrite: false, + typesenseAvailable: false, + openSearchAvailable: false, + loading: false, + message: '' + }); + + const [openSearchStatus, setOpenSearchStatus] = useState<{ + reindex: { loading: boolean; message: string; success?: boolean }; + recreate: { loading: boolean; message: string; success?: boolean }; + }>({ + reindex: { loading: false, message: '' }, + recreate: { loading: false, message: '' } + }); + const [typesenseStatus, setTypesenseStatus] = useState<{ reindex: { loading: boolean; message: string; success?: boolean }; recreate: { loading: boolean; message: string; success?: boolean }; @@ -419,13 +444,323 @@ export default function SystemSettings({}: SystemSettingsProps) { }, 10000); }; + // Search Engine Management Functions + const loadSearchEngineStatus = async () => { + try { + const status = await searchAdminApi.getStatus(); + setSearchEngineStatus(prev => ({ + ...prev, + currentEngine: status.primaryEngine, + dualWrite: status.dualWrite, + typesenseAvailable: status.typesenseAvailable, + openSearchAvailable: status.openSearchAvailable, + })); + } catch (error: any) { + console.error('Failed to load search engine status:', error); + } + }; + + const handleSwitchEngine = async (engine: string) => { + setSearchEngineStatus(prev => ({ ...prev, loading: true, message: `Switching to ${engine}...` })); + + try { + const result = engine === 'opensearch' + ? await searchAdminApi.switchToOpenSearch() + : await searchAdminApi.switchToTypesense(); + + setSearchEngineStatus(prev => ({ + ...prev, + loading: false, + message: result.message, + success: true, + currentEngine: engine + })); + + setTimeout(() => { + setSearchEngineStatus(prev => ({ ...prev, message: '', success: undefined })); + }, 5000); + } catch (error: any) { + setSearchEngineStatus(prev => ({ + ...prev, + loading: false, + message: error.message || 'Failed to switch engine', + success: false + })); + + setTimeout(() => { + setSearchEngineStatus(prev => ({ ...prev, message: '', success: undefined })); + }, 5000); + } + }; + + const handleToggleDualWrite = async () => { + const newDualWrite = !searchEngineStatus.dualWrite; + setSearchEngineStatus(prev => ({ + ...prev, + loading: true, + message: `${newDualWrite ? 'Enabling' : 'Disabling'} dual-write...` + })); + + try { + const result = newDualWrite + ? await searchAdminApi.enableDualWrite() + : await searchAdminApi.disableDualWrite(); + + setSearchEngineStatus(prev => ({ + ...prev, + loading: false, + message: result.message, + success: true, + dualWrite: newDualWrite + })); + + setTimeout(() => { + setSearchEngineStatus(prev => ({ ...prev, message: '', success: undefined })); + }, 5000); + } catch (error: any) { + setSearchEngineStatus(prev => ({ + ...prev, + loading: false, + message: error.message || 'Failed to toggle dual-write', + success: false + })); + + setTimeout(() => { + setSearchEngineStatus(prev => ({ ...prev, message: '', success: undefined })); + }, 5000); + } + }; + + const handleOpenSearchReindex = async () => { + setOpenSearchStatus(prev => ({ + ...prev, + reindex: { loading: true, message: 'Reindexing OpenSearch...', success: undefined } + })); + + try { + const result = await searchAdminApi.reindexOpenSearch(); + + setOpenSearchStatus(prev => ({ + ...prev, + reindex: { + loading: false, + message: result.success ? result.message : (result.error || 'Reindex failed'), + success: result.success + } + })); + + setTimeout(() => { + setOpenSearchStatus(prev => ({ + ...prev, + reindex: { loading: false, message: '', success: undefined } + })); + }, 8000); + } catch (error: any) { + setOpenSearchStatus(prev => ({ + ...prev, + reindex: { + loading: false, + message: error.message || 'Network error occurred', + success: false + } + })); + + setTimeout(() => { + setOpenSearchStatus(prev => ({ + ...prev, + reindex: { loading: false, message: '', success: undefined } + })); + }, 8000); + } + }; + + const handleOpenSearchRecreate = async () => { + setOpenSearchStatus(prev => ({ + ...prev, + recreate: { loading: true, message: 'Recreating OpenSearch indices...', success: undefined } + })); + + try { + const result = await searchAdminApi.recreateOpenSearchIndices(); + + setOpenSearchStatus(prev => ({ + ...prev, + recreate: { + loading: false, + message: result.success ? result.message : (result.error || 'Recreation failed'), + success: result.success + } + })); + + setTimeout(() => { + setOpenSearchStatus(prev => ({ + ...prev, + recreate: { loading: false, message: '', success: undefined } + })); + }, 8000); + } catch (error: any) { + setOpenSearchStatus(prev => ({ + ...prev, + recreate: { + loading: false, + message: error.message || 'Network error occurred', + success: false + } + })); + + setTimeout(() => { + setOpenSearchStatus(prev => ({ + ...prev, + recreate: { loading: false, message: '', success: undefined } + })); + }, 8000); + } + }; + + // Load status on component mount + useEffect(() => { + loadSearchEngineStatus(); + }, []); + return (
- {/* Typesense Search Management */} + {/* Search Engine Management */}
-

Search Index Management

+

Search Engine Migration

- Manage all Typesense search indexes (stories, authors, collections, etc.). Use these tools if search functionality isn't working properly. + Manage the transition from Typesense to OpenSearch. Switch between engines, enable dual-write mode, and perform maintenance operations. +

+ +
+ {/* Current Status */} +
+

Current Configuration

+
+
+ Primary Engine: + + {searchEngineStatus.currentEngine.charAt(0).toUpperCase() + searchEngineStatus.currentEngine.slice(1)} + +
+
+ Dual-Write: + + {searchEngineStatus.dualWrite ? 'Enabled' : 'Disabled'} + +
+
+ Typesense: + + {searchEngineStatus.typesenseAvailable ? 'Available' : 'Unavailable'} + +
+
+ OpenSearch: + + {searchEngineStatus.openSearchAvailable ? 'Available' : 'Unavailable'} + +
+
+
+ + {/* Engine Switching */} +
+

Engine Controls

+
+ + + +
+ + {searchEngineStatus.message && ( +
+ {searchEngineStatus.message} +
+ )} +
+ + {/* OpenSearch Operations */} +
+

OpenSearch Operations

+

+ Perform maintenance operations on OpenSearch indices. Use these if OpenSearch isn't returning expected results. +

+ +
+ + +
+ + {/* OpenSearch Status Messages */} + {openSearchStatus.reindex.message && ( +
+ {openSearchStatus.reindex.message} +
+ )} + + {openSearchStatus.recreate.message && ( +
+ {openSearchStatus.recreate.message} +
+ )} +
+
+
+ + {/* Legacy Typesense Management */} +
+

Legacy Typesense Management

+

+ Manage Typesense search indexes (for backwards compatibility and during migration). These tools will be removed once migration is complete.

diff --git a/frontend/src/components/stories/StoryCard.tsx b/frontend/src/components/stories/StoryCard.tsx index cbef4d0..1aa3338 100644 --- a/frontend/src/components/stories/StoryCard.tsx +++ b/frontend/src/components/stories/StoryCard.tsx @@ -17,17 +17,34 @@ interface StoryCardProps { onSelect?: () => void; } -export default function StoryCard({ - story, - viewMode, - onUpdate, - showSelection = false, - isSelected = false, - onSelect +export default function StoryCard({ + story, + viewMode, + onUpdate, + showSelection = false, + isSelected = false, + onSelect }: StoryCardProps) { const [rating, setRating] = useState(story.rating || 0); const [updating, setUpdating] = useState(false); + // Helper function to get tags from either tags array or tagNames array + const getTags = () => { + if (Array.isArray(story.tags) && story.tags.length > 0) { + return story.tags; + } + if (Array.isArray(story.tagNames) && story.tagNames.length > 0) { + // Convert tagNames to Tag objects for display compatibility + return story.tagNames.map((name, index) => ({ + id: `tag-${index}`, // Temporary ID for display + name: name + })); + } + return []; + }; + + const displayTags = getTags(); + const handleRatingClick = async (e: React.MouseEvent, newRating: number) => { // Prevent default and stop propagation to avoid triggering navigation e.preventDefault(); @@ -58,7 +75,7 @@ export default function StoryCard({ const calculateReadingPercentage = (story: Story): number => { if (!story.readingPosition) return 0; - const totalLength = story.contentPlain?.length || story.contentHtml.length; + const totalLength = story.contentPlain?.length || story.contentHtml?.length || 0; if (totalLength === 0) return 0; return Math.round((story.readingPosition / totalLength) * 100); @@ -124,9 +141,9 @@ export default function StoryCard({
{/* Tags */} - {Array.isArray(story.tags) && story.tags.length > 0 && ( + {displayTags.length > 0 && (
- {story.tags.slice(0, 3).map((tag) => ( + {displayTags.slice(0, 3).map((tag) => ( ))} - {story.tags.length > 3 && ( + {displayTags.length > 3 && ( - +{story.tags.length - 3} more + +{displayTags.length - 3} more )}
@@ -260,9 +277,9 @@ export default function StoryCard({
{/* Tags */} - {Array.isArray(story.tags) && story.tags.length > 0 && ( + {displayTags.length > 0 && (
- {story.tags.slice(0, 2).map((tag) => ( + {displayTags.slice(0, 2).map((tag) => ( ))} - {story.tags.length > 2 && ( + {displayTags.length > 2 && ( - +{story.tags.length - 2} + +{displayTags.length - 2} )}
diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 35a737f..32ccc5d 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -611,6 +611,79 @@ export const configApi = { }, }; +// Search Engine Management API +export const searchAdminApi = { + // Get migration status + getStatus: async (): Promise<{ + primaryEngine: string; + dualWrite: boolean; + typesenseAvailable: boolean; + openSearchAvailable: boolean; + }> => { + const response = await api.get('/admin/search/status'); + return response.data; + }, + + // Configure search engine + configure: async (config: { engine: string; dualWrite: boolean }): Promise<{ message: string }> => { + const response = await api.post('/admin/search/configure', config); + return response.data; + }, + + // Enable/disable dual-write + enableDualWrite: async (): Promise<{ message: string }> => { + const response = await api.post('/admin/search/dual-write/enable'); + return response.data; + }, + + disableDualWrite: async (): Promise<{ message: string }> => { + const response = await api.post('/admin/search/dual-write/disable'); + return response.data; + }, + + // Switch engines + switchToOpenSearch: async (): Promise<{ message: string }> => { + const response = await api.post('/admin/search/switch/opensearch'); + return response.data; + }, + + switchToTypesense: async (): Promise<{ message: string }> => { + const response = await api.post('/admin/search/switch/typesense'); + return response.data; + }, + + // Emergency rollback + emergencyRollback: async (): Promise<{ message: string }> => { + const response = await api.post('/admin/search/emergency-rollback'); + return response.data; + }, + + // OpenSearch operations + reindexOpenSearch: async (): Promise<{ + success: boolean; + message: string; + storiesCount?: number; + authorsCount?: number; + totalCount?: number; + error?: string; + }> => { + const response = await api.post('/admin/search/opensearch/reindex'); + return response.data; + }, + + recreateOpenSearchIndices: async (): Promise<{ + success: boolean; + message: string; + storiesCount?: number; + authorsCount?: number; + totalCount?: number; + error?: string; + }> => { + const response = await api.post('/admin/search/opensearch/recreate'); + return response.data; + }, +}; + // Collection endpoints export const collectionApi = { getCollections: async (params?: {