shardegger/storycove
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

solr migration button

Browse Source
This commit is contained in:
Stefan Hardegger
2025-09-23 14:42:38 +02:00
parent 9472210d8b
commit 6ee2d67027
4 changed files with 272 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
SOLR_LIBRARY_MIGRATION.md
View File

@@ -35,6 +35,25 @@ docker-compose up -d
**Best for**: Production environments where you need to preserve uptime.
**Method A: Automatic (Recommended)**
```bash
# Single endpoint that adds field and migrates data
curl -X POST "http://your-app-host/api/admin/search/solr/migrate-library-schema" \
-H "Authorization: Bearer YOUR_JWT_TOKEN"
```
**Method B: Manual Steps**
```bash
# Step 1: Add libraryId field via app API
curl -X POST "http://your-app-host/api/admin/search/solr/add-library-field" \
-H "Authorization: Bearer YOUR_JWT_TOKEN"
# Step 2: Run migration
curl -X POST "http://your-app-host/api/admin/search/solr/migrate-library-schema" \
-H "Authorization: Bearer YOUR_JWT_TOKEN"
```
**Method C: Direct Solr API (if app API fails)**
```bash
# Add libraryId field to stories core
curl -X POST "http://your-solr-host:8983/solr/storycove_stories/schema" \
@@ -62,13 +81,13 @@ curl -X POST "http://your-solr-host:8983/solr/storycove_authors/schema" \
}
}'
# Then use the admin migration endpoint
# Then run the migration
curl -X POST "http://your-app-host/api/admin/search/solr/migrate-library-schema" \
-H "Authorization: Bearer YOUR_JWT_TOKEN"
```
**Pros**: No downtime, preserves service availability
**Cons**: More complex, requires API access
**Pros**: No downtime, preserves service availability, automatic field addition
**Cons**: Requires API access
### Option 3: Application-Level Migration (Recommended for Production)
@@ -142,10 +161,39 @@ After migration, verify that library separation is working:
### "unknown field 'libraryId'" Error
This means the Solr schema wasn't updated. Solutions:
- Use Option 1 (volume reset) for clean restart
- Use Option 2 (Schema API) to add the field manually
- Check that schema files contain the libraryId field definition
**Problem**: `ERROR: [doc=xxx] unknown field 'libraryId'`
**Cause**: The Solr schema doesn't have the libraryId field yet.
**Solutions**:
1. **Use the automated migration** (adds field automatically):
```bash
curl -X POST "http://your-app/api/admin/search/solr/migrate-library-schema"
```
2. **Add field manually first**:
```bash
# Add field via app API
curl -X POST "http://your-app/api/admin/search/solr/add-library-field"
# Then run migration
curl -X POST "http://your-app/api/admin/search/solr/migrate-library-schema"
```
3. **Direct Solr API** (if app API fails):
```bash
# Add to both cores
curl -X POST "http://solr:8983/solr/storycove_stories/schema" \
-H "Content-Type: application/json" \
-d '{"add-field":{"name":"libraryId","type":"string","indexed":true,"stored":true}}'
curl -X POST "http://solr:8983/solr/storycove_authors/schema" \
-H "Content-Type: application/json" \
-d '{"add-field":{"name":"libraryId","type":"string","indexed":true,"stored":true}}'
```
4. **For development**: Use Option 1 (volume reset) for clean restart
### Migration Endpoint Returns Error

87
backend/src/main/java/com/storycove/controller/AdminSearchController.java
View File

@@ -161,6 +161,58 @@ public class AdminSearchController {
}
}
/**
* Add libraryId field to Solr schema via Schema API.
* This is a prerequisite for library-aware indexing.
*/
@PostMapping("/solr/add-library-field")
public ResponseEntity<Map<String, Object>> addLibraryField() {
try {
logger.info("Starting Solr libraryId field addition");
if (!searchServiceAdapter.isSearchServiceAvailable()) {
return ResponseEntity.badRequest().body(Map.of(
"success", false,
"error", "Solr is not available or healthy"
));
}
if (solrService == null) {
return ResponseEntity.badRequest().body(Map.of(
"success", false,
"error", "Solr service not available"
));
}
// Add the libraryId field to the schema
try {
solrService.addLibraryIdField();
logger.info("libraryId field added successfully to schema");
return ResponseEntity.ok(Map.of(
"success", true,
"message", "libraryId field added successfully to both stories and authors cores",
"note", "You can now run the library schema migration"
));
} catch (Exception e) {
logger.error("Failed to add libraryId field to schema", e);
return ResponseEntity.internalServerError().body(Map.of(
"success", false,
"error", "Failed to add libraryId field to schema: " + e.getMessage(),
"details", "Check that Solr is accessible and schema is modifiable"
));
}
} catch (Exception e) {
logger.error("Error during libraryId field addition", e);
return ResponseEntity.internalServerError().body(Map.of(
"success", false,
"error", "libraryId field addition failed: " + e.getMessage()
));
}
}
/**
* Migrate to library-aware Solr schema.
* This endpoint handles the migration from non-library-aware to library-aware indexing.
@@ -185,13 +237,40 @@ public class AdminSearchController {
));
}
logger.info("Adding libraryId field to Solr schema");
// First, add the libraryId field to the schema via Schema API
try {
solrService.addLibraryIdField();
logger.info("libraryId field added successfully to schema");
} catch (Exception e) {
logger.error("Failed to add libraryId field to schema", e);
return ResponseEntity.badRequest().body(Map.of(
"success", false,
"error", "Failed to add libraryId field to schema: " + e.getMessage(),
"details", "The schema must support the libraryId field before migration"
));
}
logger.info("Clearing existing Solr data for library schema migration");
// Note: This assumes the libraryId field has been added to the Solr schema
// Either manually or via schema restart with updated schema files
// Clear existing data that doesn't have libraryId
solrService.recreateIndices();
try {
solrService.recreateIndices();
} catch (Exception e) {
logger.warn("Could not recreate indices (expected in production): {}", e.getMessage());
// In production, just clear the data instead
try {
solrService.clearAllDocuments();
logger.info("Cleared all documents from Solr cores");
} catch (Exception clearError) {
logger.error("Failed to clear documents", clearError);
return ResponseEntity.badRequest().body(Map.of(
"success", false,
"error", "Failed to clear existing data: " + clearError.getMessage()
));
}
}
// Get all data and reindex with library context
List<Story> allStories = storyService.findAllWithAssociations();

122
backend/src/main/java/com/storycove/service/SolrService.java
View File

@@ -379,6 +379,128 @@ public class SolrService {
}
}
/**
* Add libraryId field to Solr schema via Schema API
* This is required for library-aware indexing in production environments
*/
public void addLibraryIdField() throws Exception {
if (!isAvailable()) {
throw new IllegalStateException("Solr is not available");
}
try {
// Check if libraryId field already exists
if (hasLibraryIdField()) {
logger.info("libraryId field already exists in schema");
return;
}
logger.info("Adding libraryId field to Solr schema via Schema API");
// Add field to stories core
try {
var storiesRequest = new org.apache.solr.client.solrj.request.schema.SchemaRequest.AddField(
Map.of(
"name", "libraryId",
"type", "string",
"indexed", true,
"stored", true,
"required", false
)
);
var storiesResponse = storiesRequest.process(solrClient, properties.getCores().getStories());
logger.info("Added libraryId field to stories core: {}", storiesResponse.getStatus());
} catch (Exception e) {
if (e.getMessage() != null && e.getMessage().contains("already exists")) {
logger.info("libraryId field already exists in stories core");
} else {
throw e;
}
}
// Add field to authors core
try {
var authorsRequest = new org.apache.solr.client.solrj.request.schema.SchemaRequest.AddField(
Map.of(
"name", "libraryId",
"type", "string",
"indexed", true,
"stored", true,
"required", false
)
);
var authorsResponse = authorsRequest.process(solrClient, properties.getCores().getAuthors());
logger.info("Added libraryId field to authors core: {}", authorsResponse.getStatus());
} catch (Exception e) {
if (e.getMessage() != null && e.getMessage().contains("already exists")) {
logger.info("libraryId field already exists in authors core");
} else {
throw e;
}
}
logger.info("Successfully added libraryId field to both cores");
} catch (Exception e) {
logger.error("Failed to add libraryId field to schema", e);
throw new RuntimeException("Failed to add libraryId field to schema: " + e.getMessage(), e);
}
}
/**
* Check if libraryId field exists in the schema
*/
public boolean hasLibraryIdField() {
try {
var request = new org.apache.solr.client.solrj.request.schema.SchemaRequest.Field("libraryId");
request.process(solrClient, properties.getCores().getStories());
return true;
} catch (Exception e) {
// Field doesn't exist or other error
return false;
}
}
/**
* Clear all documents from both stories and authors cores
* Used for migration when core recreation isn't possible
*/
public void clearAllDocuments() throws Exception {
if (!isAvailable()) {
throw new IllegalStateException("Solr is not available");
}
try {
logger.info("Clearing all documents from Solr cores");
// Clear stories core
try {
solrClient.deleteByQuery(properties.getCores().getStories(), "*:*",
properties.getCommit().getCommitWithin());
logger.info("Cleared all documents from stories core");
} catch (Exception e) {
logger.error("Failed to clear stories core", e);
throw e;
}
// Clear authors core
try {
solrClient.deleteByQuery(properties.getCores().getAuthors(), "*:*",
properties.getCommit().getCommitWithin());
logger.info("Cleared all documents from authors core");
} catch (Exception e) {
logger.error("Failed to clear authors core", e);
throw e;
}
logger.info("Successfully cleared all documents from both cores");
} catch (Exception e) {
logger.error("Failed to clear all documents", e);
throw new RuntimeException("Failed to clear all documents: " + e.getMessage(), e);
}
}
// ===============================
// SEARCH OPERATIONS
// ===============================

12
frontend/src/lib/api.ts
View File

@@ -637,6 +637,18 @@ export const searchAdminApi = {
return response.data;
},
// Add libraryId field to schema
addLibraryField: async (): Promise<{
success: boolean;
message: string;
error?: string;
details?: string;
note?: string;
}> => {
const response = await api.post('/admin/search/solr/add-library-field');
return response.data;
},
// Migrate to library-aware schema
migrateLibrarySchema: async (): Promise<{
success: boolean;