From 6ee2d67027af3db4acefd3294c9953c2fb73a745 Mon Sep 17 00:00:00 2001 From: Stefan Hardegger Date: Tue, 23 Sep 2025 14:42:38 +0200 Subject: [PATCH] solr migration button --- SOLR_LIBRARY_MIGRATION.md | 62 ++++++++- .../controller/AdminSearchController.java | 87 ++++++++++++- .../com/storycove/service/SolrService.java | 122 ++++++++++++++++++ frontend/src/lib/api.ts | 12 ++ 4 files changed, 272 insertions(+), 11 deletions(-) diff --git a/SOLR_LIBRARY_MIGRATION.md b/SOLR_LIBRARY_MIGRATION.md index 13ca226..2fdb8ce 100644 --- a/SOLR_LIBRARY_MIGRATION.md +++ b/SOLR_LIBRARY_MIGRATION.md @@ -35,6 +35,25 @@ docker-compose up -d **Best for**: Production environments where you need to preserve uptime. +**Method A: Automatic (Recommended)** +```bash +# Single endpoint that adds field and migrates data +curl -X POST "http://your-app-host/api/admin/search/solr/migrate-library-schema" \ + -H "Authorization: Bearer YOUR_JWT_TOKEN" +``` + +**Method B: Manual Steps** +```bash +# Step 1: Add libraryId field via app API +curl -X POST "http://your-app-host/api/admin/search/solr/add-library-field" \ + -H "Authorization: Bearer YOUR_JWT_TOKEN" + +# Step 2: Run migration +curl -X POST "http://your-app-host/api/admin/search/solr/migrate-library-schema" \ + -H "Authorization: Bearer YOUR_JWT_TOKEN" +``` + +**Method C: Direct Solr API (if app API fails)** ```bash # Add libraryId field to stories core curl -X POST "http://your-solr-host:8983/solr/storycove_stories/schema" \ @@ -62,13 +81,13 @@ curl -X POST "http://your-solr-host:8983/solr/storycove_authors/schema" \ } }' -# Then use the admin migration endpoint +# Then run the migration curl -X POST "http://your-app-host/api/admin/search/solr/migrate-library-schema" \ -H "Authorization: Bearer YOUR_JWT_TOKEN" ``` -**Pros**: No downtime, preserves service availability -**Cons**: More complex, requires API access +**Pros**: No downtime, preserves service availability, automatic field addition +**Cons**: Requires API access ### Option 3: Application-Level Migration (Recommended for Production) @@ -142,10 +161,39 @@ After migration, verify that library separation is working: ### "unknown field 'libraryId'" Error -This means the Solr schema wasn't updated. Solutions: -- Use Option 1 (volume reset) for clean restart -- Use Option 2 (Schema API) to add the field manually -- Check that schema files contain the libraryId field definition +**Problem**: `ERROR: [doc=xxx] unknown field 'libraryId'` + +**Cause**: The Solr schema doesn't have the libraryId field yet. + +**Solutions**: + +1. **Use the automated migration** (adds field automatically): + ```bash + curl -X POST "http://your-app/api/admin/search/solr/migrate-library-schema" + ``` + +2. **Add field manually first**: + ```bash + # Add field via app API + curl -X POST "http://your-app/api/admin/search/solr/add-library-field" + + # Then run migration + curl -X POST "http://your-app/api/admin/search/solr/migrate-library-schema" + ``` + +3. **Direct Solr API** (if app API fails): + ```bash + # Add to both cores + curl -X POST "http://solr:8983/solr/storycove_stories/schema" \ + -H "Content-Type: application/json" \ + -d '{"add-field":{"name":"libraryId","type":"string","indexed":true,"stored":true}}' + + curl -X POST "http://solr:8983/solr/storycove_authors/schema" \ + -H "Content-Type: application/json" \ + -d '{"add-field":{"name":"libraryId","type":"string","indexed":true,"stored":true}}' + ``` + +4. **For development**: Use Option 1 (volume reset) for clean restart ### Migration Endpoint Returns Error diff --git a/backend/src/main/java/com/storycove/controller/AdminSearchController.java b/backend/src/main/java/com/storycove/controller/AdminSearchController.java index 2846e86..f7d81d7 100644 --- a/backend/src/main/java/com/storycove/controller/AdminSearchController.java +++ b/backend/src/main/java/com/storycove/controller/AdminSearchController.java @@ -161,6 +161,58 @@ public class AdminSearchController { } } + /** + * Add libraryId field to Solr schema via Schema API. + * This is a prerequisite for library-aware indexing. + */ + @PostMapping("/solr/add-library-field") + public ResponseEntity> addLibraryField() { + try { + logger.info("Starting Solr libraryId field addition"); + + if (!searchServiceAdapter.isSearchServiceAvailable()) { + return ResponseEntity.badRequest().body(Map.of( + "success", false, + "error", "Solr is not available or healthy" + )); + } + + if (solrService == null) { + return ResponseEntity.badRequest().body(Map.of( + "success", false, + "error", "Solr service not available" + )); + } + + // Add the libraryId field to the schema + try { + solrService.addLibraryIdField(); + logger.info("libraryId field added successfully to schema"); + + return ResponseEntity.ok(Map.of( + "success", true, + "message", "libraryId field added successfully to both stories and authors cores", + "note", "You can now run the library schema migration" + )); + + } catch (Exception e) { + logger.error("Failed to add libraryId field to schema", e); + return ResponseEntity.internalServerError().body(Map.of( + "success", false, + "error", "Failed to add libraryId field to schema: " + e.getMessage(), + "details", "Check that Solr is accessible and schema is modifiable" + )); + } + + } catch (Exception e) { + logger.error("Error during libraryId field addition", e); + return ResponseEntity.internalServerError().body(Map.of( + "success", false, + "error", "libraryId field addition failed: " + e.getMessage() + )); + } + } + /** * Migrate to library-aware Solr schema. * This endpoint handles the migration from non-library-aware to library-aware indexing. @@ -185,13 +237,40 @@ public class AdminSearchController { )); } + logger.info("Adding libraryId field to Solr schema"); + + // First, add the libraryId field to the schema via Schema API + try { + solrService.addLibraryIdField(); + logger.info("libraryId field added successfully to schema"); + } catch (Exception e) { + logger.error("Failed to add libraryId field to schema", e); + return ResponseEntity.badRequest().body(Map.of( + "success", false, + "error", "Failed to add libraryId field to schema: " + e.getMessage(), + "details", "The schema must support the libraryId field before migration" + )); + } + logger.info("Clearing existing Solr data for library schema migration"); - // Note: This assumes the libraryId field has been added to the Solr schema - // Either manually or via schema restart with updated schema files - // Clear existing data that doesn't have libraryId - solrService.recreateIndices(); + try { + solrService.recreateIndices(); + } catch (Exception e) { + logger.warn("Could not recreate indices (expected in production): {}", e.getMessage()); + // In production, just clear the data instead + try { + solrService.clearAllDocuments(); + logger.info("Cleared all documents from Solr cores"); + } catch (Exception clearError) { + logger.error("Failed to clear documents", clearError); + return ResponseEntity.badRequest().body(Map.of( + "success", false, + "error", "Failed to clear existing data: " + clearError.getMessage() + )); + } + } // Get all data and reindex with library context List allStories = storyService.findAllWithAssociations(); diff --git a/backend/src/main/java/com/storycove/service/SolrService.java b/backend/src/main/java/com/storycove/service/SolrService.java index cba1d3a..a820c2c 100644 --- a/backend/src/main/java/com/storycove/service/SolrService.java +++ b/backend/src/main/java/com/storycove/service/SolrService.java @@ -379,6 +379,128 @@ public class SolrService { } } + /** + * Add libraryId field to Solr schema via Schema API + * This is required for library-aware indexing in production environments + */ + public void addLibraryIdField() throws Exception { + if (!isAvailable()) { + throw new IllegalStateException("Solr is not available"); + } + + try { + // Check if libraryId field already exists + if (hasLibraryIdField()) { + logger.info("libraryId field already exists in schema"); + return; + } + + logger.info("Adding libraryId field to Solr schema via Schema API"); + + // Add field to stories core + try { + var storiesRequest = new org.apache.solr.client.solrj.request.schema.SchemaRequest.AddField( + Map.of( + "name", "libraryId", + "type", "string", + "indexed", true, + "stored", true, + "required", false + ) + ); + var storiesResponse = storiesRequest.process(solrClient, properties.getCores().getStories()); + logger.info("Added libraryId field to stories core: {}", storiesResponse.getStatus()); + } catch (Exception e) { + if (e.getMessage() != null && e.getMessage().contains("already exists")) { + logger.info("libraryId field already exists in stories core"); + } else { + throw e; + } + } + + // Add field to authors core + try { + var authorsRequest = new org.apache.solr.client.solrj.request.schema.SchemaRequest.AddField( + Map.of( + "name", "libraryId", + "type", "string", + "indexed", true, + "stored", true, + "required", false + ) + ); + var authorsResponse = authorsRequest.process(solrClient, properties.getCores().getAuthors()); + logger.info("Added libraryId field to authors core: {}", authorsResponse.getStatus()); + } catch (Exception e) { + if (e.getMessage() != null && e.getMessage().contains("already exists")) { + logger.info("libraryId field already exists in authors core"); + } else { + throw e; + } + } + + logger.info("Successfully added libraryId field to both cores"); + + } catch (Exception e) { + logger.error("Failed to add libraryId field to schema", e); + throw new RuntimeException("Failed to add libraryId field to schema: " + e.getMessage(), e); + } + } + + /** + * Check if libraryId field exists in the schema + */ + public boolean hasLibraryIdField() { + try { + var request = new org.apache.solr.client.solrj.request.schema.SchemaRequest.Field("libraryId"); + request.process(solrClient, properties.getCores().getStories()); + return true; + } catch (Exception e) { + // Field doesn't exist or other error + return false; + } + } + + /** + * Clear all documents from both stories and authors cores + * Used for migration when core recreation isn't possible + */ + public void clearAllDocuments() throws Exception { + if (!isAvailable()) { + throw new IllegalStateException("Solr is not available"); + } + + try { + logger.info("Clearing all documents from Solr cores"); + + // Clear stories core + try { + solrClient.deleteByQuery(properties.getCores().getStories(), "*:*", + properties.getCommit().getCommitWithin()); + logger.info("Cleared all documents from stories core"); + } catch (Exception e) { + logger.error("Failed to clear stories core", e); + throw e; + } + + // Clear authors core + try { + solrClient.deleteByQuery(properties.getCores().getAuthors(), "*:*", + properties.getCommit().getCommitWithin()); + logger.info("Cleared all documents from authors core"); + } catch (Exception e) { + logger.error("Failed to clear authors core", e); + throw e; + } + + logger.info("Successfully cleared all documents from both cores"); + + } catch (Exception e) { + logger.error("Failed to clear all documents", e); + throw new RuntimeException("Failed to clear all documents: " + e.getMessage(), e); + } + } + // =============================== // SEARCH OPERATIONS // =============================== diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 612a716..79da394 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -637,6 +637,18 @@ export const searchAdminApi = { return response.data; }, + // Add libraryId field to schema + addLibraryField: async (): Promise<{ + success: boolean; + message: string; + error?: string; + details?: string; + note?: string; + }> => { + const response = await api.post('/admin/search/solr/add-library-field'); + return response.data; + }, + // Migrate to library-aware schema migrateLibrarySchema: async (): Promise<{ success: boolean;