backup / restore improvement
This commit is contained in:
Stefan Hardegger
@@ -2,8 +2,8 @@ FROM openjdk:17-jdk-slim
|
|||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
# Install Maven
|
# Install Maven and PostgreSQL client tools
|
||||||
RUN apt-get update && apt-get install -y maven && rm -rf /var/lib/apt/lists/*
|
RUN apt-get update && apt-get install -y maven postgresql-client && rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
# Copy source code
|
# Copy source code
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|||||||
@@ -70,6 +70,75 @@ public class DatabaseManagementService implements ApplicationContextAware {
|
|||||||
this.applicationContext = applicationContext;
|
this.applicationContext = applicationContext;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Helper methods to extract database connection details
|
||||||
|
private String extractDatabaseUrl() {
|
||||||
|
try (Connection connection = getDataSource().getConnection()) {
|
||||||
|
return connection.getMetaData().getURL();
|
||||||
|
} catch (SQLException e) {
|
||||||
|
throw new RuntimeException("Failed to extract database URL", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractDatabaseHost() {
|
||||||
|
String url = extractDatabaseUrl();
|
||||||
|
// Extract host from jdbc:postgresql://host:port/database
|
||||||
|
if (url.startsWith("jdbc:postgresql://")) {
|
||||||
|
String hostPort = url.substring("jdbc:postgresql://".length());
|
||||||
|
if (hostPort.contains("/")) {
|
||||||
|
hostPort = hostPort.substring(0, hostPort.indexOf("/"));
|
||||||
|
}
|
||||||
|
if (hostPort.contains(":")) {
|
||||||
|
return hostPort.substring(0, hostPort.indexOf(":"));
|
||||||
|
}
|
||||||
|
return hostPort;
|
||||||
|
}
|
||||||
|
return "localhost"; // fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractDatabasePort() {
|
||||||
|
String url = extractDatabaseUrl();
|
||||||
|
// Extract port from jdbc:postgresql://host:port/database
|
||||||
|
if (url.startsWith("jdbc:postgresql://")) {
|
||||||
|
String hostPort = url.substring("jdbc:postgresql://".length());
|
||||||
|
if (hostPort.contains("/")) {
|
||||||
|
hostPort = hostPort.substring(0, hostPort.indexOf("/"));
|
||||||
|
}
|
||||||
|
if (hostPort.contains(":")) {
|
||||||
|
return hostPort.substring(hostPort.indexOf(":") + 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "5432"; // default PostgreSQL port
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractDatabaseName() {
|
||||||
|
String url = extractDatabaseUrl();
|
||||||
|
// Extract database name from jdbc:postgresql://host:port/database
|
||||||
|
if (url.startsWith("jdbc:postgresql://")) {
|
||||||
|
String remaining = url.substring("jdbc:postgresql://".length());
|
||||||
|
if (remaining.contains("/")) {
|
||||||
|
String dbPart = remaining.substring(remaining.indexOf("/") + 1);
|
||||||
|
// Remove any query parameters
|
||||||
|
if (dbPart.contains("?")) {
|
||||||
|
dbPart = dbPart.substring(0, dbPart.indexOf("?"));
|
||||||
|
}
|
||||||
|
return dbPart;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "storycove"; // fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractDatabaseUsername() {
|
||||||
|
// Get from environment variable or default
|
||||||
|
return System.getenv("SPRING_DATASOURCE_USERNAME") != null ?
|
||||||
|
System.getenv("SPRING_DATASOURCE_USERNAME") : "storycove";
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractDatabasePassword() {
|
||||||
|
// Get from environment variable or default
|
||||||
|
return System.getenv("SPRING_DATASOURCE_PASSWORD") != null ?
|
||||||
|
System.getenv("SPRING_DATASOURCE_PASSWORD") : "password";
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a comprehensive backup including database and files in ZIP format
|
* Create a comprehensive backup including database and files in ZIP format
|
||||||
*/
|
*/
|
||||||
@@ -172,175 +241,177 @@ public class DatabaseManagementService implements ApplicationContextAware {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public Resource createBackup() throws SQLException, IOException {
|
public Resource createBackup() throws SQLException, IOException {
|
||||||
StringBuilder sqlDump = new StringBuilder();
|
// Use PostgreSQL's native pg_dump for reliable backup
|
||||||
|
String dbHost = extractDatabaseHost();
|
||||||
|
String dbPort = extractDatabasePort();
|
||||||
|
String dbName = extractDatabaseName();
|
||||||
|
String dbUser = extractDatabaseUsername();
|
||||||
|
String dbPassword = extractDatabasePassword();
|
||||||
|
|
||||||
try (Connection connection = getDataSource().getConnection()) {
|
// Create temporary file for backup
|
||||||
// Add header
|
Path tempBackupFile = Files.createTempFile("storycove_backup_", ".sql");
|
||||||
sqlDump.append("-- StoryCove Database Backup\n");
|
|
||||||
sqlDump.append("-- Generated at: ").append(new java.util.Date()).append("\n\n");
|
|
||||||
|
|
||||||
// Disable foreign key checks during restore (PostgreSQL syntax)
|
try {
|
||||||
sqlDump.append("SET session_replication_role = replica;\n\n");
|
// Build pg_dump command
|
||||||
|
ProcessBuilder pb = new ProcessBuilder(
|
||||||
// List of tables in dependency order (parents first for insertion)
|
"pg_dump",
|
||||||
List<String> insertTables = Arrays.asList(
|
"--host=" + dbHost,
|
||||||
"authors", "series", "tags", "collections",
|
"--port=" + dbPort,
|
||||||
"stories", "story_tags", "author_urls", "collection_stories"
|
"--username=" + dbUser,
|
||||||
|
"--dbname=" + dbName,
|
||||||
|
"--no-password",
|
||||||
|
"--verbose",
|
||||||
|
"--clean",
|
||||||
|
"--if-exists",
|
||||||
|
"--create",
|
||||||
|
"--file=" + tempBackupFile.toString()
|
||||||
);
|
);
|
||||||
|
|
||||||
// TRUNCATE in reverse order (children first)
|
// Set PGPASSWORD environment variable
|
||||||
List<String> truncateTables = Arrays.asList(
|
Map<String, String> env = pb.environment();
|
||||||
"collection_stories", "author_urls", "story_tags",
|
env.put("PGPASSWORD", dbPassword);
|
||||||
"stories", "collections", "tags", "series", "authors"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Generate DELETE statements for each table (safer than TRUNCATE CASCADE)
|
System.err.println("Starting PostgreSQL backup using pg_dump...");
|
||||||
for (String tableName : truncateTables) {
|
Process process = pb.start();
|
||||||
sqlDump.append("-- Clear Table: ").append(tableName).append("\n");
|
|
||||||
sqlDump.append("DELETE FROM \"").append(tableName).append("\";\n");
|
|
||||||
|
|
||||||
// Reset auto-increment sequences for tables with ID columns
|
// Capture output
|
||||||
if (Arrays.asList("authors", "series", "tags", "collections", "stories").contains(tableName)) {
|
try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getErrorStream()))) {
|
||||||
sqlDump.append("SELECT setval(pg_get_serial_sequence('\"").append(tableName).append("\"', 'id'), 1, false);\n");
|
String line;
|
||||||
|
while ((line = reader.readLine()) != null) {
|
||||||
|
System.err.println("pg_dump: " + line);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
sqlDump.append("\n");
|
|
||||||
|
|
||||||
// Generate INSERT statements in dependency order
|
int exitCode = process.waitFor();
|
||||||
for (String tableName : insertTables) {
|
if (exitCode != 0) {
|
||||||
sqlDump.append("-- Data for Table: ").append(tableName).append("\n");
|
throw new RuntimeException("pg_dump failed with exit code: " + exitCode);
|
||||||
|
|
||||||
// Get table data
|
|
||||||
try (PreparedStatement stmt = connection.prepareStatement("SELECT * FROM \"" + tableName + "\"");
|
|
||||||
ResultSet rs = stmt.executeQuery()) {
|
|
||||||
|
|
||||||
ResultSetMetaData metaData = rs.getMetaData();
|
|
||||||
int columnCount = metaData.getColumnCount();
|
|
||||||
|
|
||||||
// Build column names for INSERT statement
|
|
||||||
StringBuilder columnNames = new StringBuilder();
|
|
||||||
for (int i = 1; i <= columnCount; i++) {
|
|
||||||
if (i > 1) columnNames.append(", ");
|
|
||||||
columnNames.append("\"").append(metaData.getColumnName(i)).append("\"");
|
|
||||||
}
|
|
||||||
|
|
||||||
while (rs.next()) {
|
|
||||||
sqlDump.append("INSERT INTO \"").append(tableName).append("\" (")
|
|
||||||
.append(columnNames).append(") VALUES (");
|
|
||||||
|
|
||||||
for (int i = 1; i <= columnCount; i++) {
|
|
||||||
if (i > 1) sqlDump.append(", ");
|
|
||||||
|
|
||||||
Object value = rs.getObject(i);
|
|
||||||
sqlDump.append(formatSqlValue(value));
|
|
||||||
}
|
|
||||||
|
|
||||||
sqlDump.append(");\n");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
sqlDump.append("\n");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Re-enable foreign key checks (PostgreSQL syntax)
|
System.err.println("PostgreSQL backup completed successfully");
|
||||||
sqlDump.append("SET session_replication_role = DEFAULT;\n");
|
|
||||||
|
// Read the backup file into memory
|
||||||
|
byte[] backupData = Files.readAllBytes(tempBackupFile);
|
||||||
|
return new ByteArrayResource(backupData);
|
||||||
|
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
throw new RuntimeException("Backup process was interrupted", e);
|
||||||
|
} finally {
|
||||||
|
// Clean up temporary file
|
||||||
|
try {
|
||||||
|
Files.deleteIfExists(tempBackupFile);
|
||||||
|
} catch (IOException e) {
|
||||||
|
System.err.println("Warning: Could not delete temporary backup file: " + e.getMessage());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
byte[] backupData = sqlDump.toString().getBytes(StandardCharsets.UTF_8);
|
|
||||||
return new ByteArrayResource(backupData);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Transactional(timeout = 1800) // 30 minutes timeout for large backup restores
|
@Transactional(timeout = 1800) // 30 minutes timeout for large backup restores
|
||||||
public void restoreFromBackup(InputStream backupStream) throws IOException, SQLException {
|
public void restoreFromBackup(InputStream backupStream) throws IOException, SQLException {
|
||||||
// Read the SQL file
|
// Use PostgreSQL's native psql for reliable restore
|
||||||
StringBuilder sqlContent = new StringBuilder();
|
String dbHost = extractDatabaseHost();
|
||||||
try (BufferedReader reader = new BufferedReader(new InputStreamReader(backupStream, StandardCharsets.UTF_8))) {
|
String dbPort = extractDatabasePort();
|
||||||
String line;
|
String dbName = extractDatabaseName();
|
||||||
while ((line = reader.readLine()) != null) {
|
String dbUser = extractDatabaseUsername();
|
||||||
// Skip comments and empty lines
|
String dbPassword = extractDatabasePassword();
|
||||||
if (!line.trim().startsWith("--") && !line.trim().isEmpty()) {
|
|
||||||
sqlContent.append(line).append("\n");
|
// Create temporary file for the backup
|
||||||
|
Path tempBackupFile = Files.createTempFile("storycove_restore_", ".sql");
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Write backup stream to temporary file
|
||||||
|
System.err.println("Writing backup data to temporary file...");
|
||||||
|
try (InputStream input = backupStream;
|
||||||
|
OutputStream output = Files.newOutputStream(tempBackupFile)) {
|
||||||
|
byte[] buffer = new byte[8192];
|
||||||
|
int bytesRead;
|
||||||
|
while ((bytesRead = input.read(buffer)) != -1) {
|
||||||
|
output.write(buffer, 0, bytesRead);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Execute the SQL statements
|
System.err.println("Starting PostgreSQL restore using psql...");
|
||||||
try (Connection connection = getDataSource().getConnection()) {
|
|
||||||
connection.setAutoCommit(false);
|
|
||||||
|
|
||||||
try {
|
// Build psql command to restore the backup
|
||||||
// Ensure database schema exists before restoring data
|
ProcessBuilder pb = new ProcessBuilder(
|
||||||
ensureDatabaseSchemaExists(connection);
|
"psql",
|
||||||
|
"--host=" + dbHost,
|
||||||
|
"--port=" + dbPort,
|
||||||
|
"--username=" + dbUser,
|
||||||
|
"--dbname=" + dbName,
|
||||||
|
"--no-password",
|
||||||
|
"--echo-errors",
|
||||||
|
"--file=" + tempBackupFile.toString()
|
||||||
|
);
|
||||||
|
|
||||||
// Parse SQL statements properly (handle semicolons inside string literals)
|
// Set PGPASSWORD environment variable
|
||||||
List<String> statements = parseStatements(sqlContent.toString());
|
Map<String, String> env = pb.environment();
|
||||||
System.err.println("Parsed " + statements.size() + " SQL statements. Starting execution...");
|
env.put("PGPASSWORD", dbPassword);
|
||||||
|
|
||||||
int successCount = 0;
|
Process process = pb.start();
|
||||||
for (String statement : statements) {
|
|
||||||
String trimmedStatement = statement.trim();
|
|
||||||
if (!trimmedStatement.isEmpty()) {
|
|
||||||
try (PreparedStatement stmt = connection.prepareStatement(trimmedStatement)) {
|
|
||||||
stmt.setQueryTimeout(300); // 5 minute timeout per statement
|
|
||||||
stmt.executeUpdate();
|
|
||||||
successCount++;
|
|
||||||
|
|
||||||
// Progress logging and batch commits for large restores
|
// Capture output
|
||||||
if (successCount % 100 == 0) {
|
try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getErrorStream()));
|
||||||
System.err.println("Executed " + successCount + "/" + statements.size() + " statements...");
|
BufferedReader outputReader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
|
||||||
}
|
|
||||||
|
|
||||||
// Commit every 500 statements to avoid huge transactions
|
// Read stderr in a separate thread
|
||||||
if (successCount % 500 == 0) {
|
Thread errorThread = new Thread(() -> {
|
||||||
connection.commit();
|
try {
|
||||||
System.err.println("Committed batch at " + successCount + " statements");
|
String line;
|
||||||
}
|
while ((line = reader.readLine()) != null) {
|
||||||
} catch (SQLException e) {
|
System.err.println("psql stderr: " + line);
|
||||||
// Log detailed error information for failed statements
|
|
||||||
System.err.println("ERROR: Failed to execute SQL statement #" + (successCount + 1));
|
|
||||||
System.err.println("Error: " + e.getMessage());
|
|
||||||
System.err.println("SQL State: " + e.getSQLState());
|
|
||||||
System.err.println("Error Code: " + e.getErrorCode());
|
|
||||||
|
|
||||||
// Show the problematic statement (first 500 chars)
|
|
||||||
String statementPreview = trimmedStatement.length() > 500 ?
|
|
||||||
trimmedStatement.substring(0, 500) + "..." : trimmedStatement;
|
|
||||||
System.err.println("Statement: " + statementPreview);
|
|
||||||
|
|
||||||
throw e; // Re-throw to trigger rollback
|
|
||||||
}
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
System.err.println("Error reading psql stderr: " + e.getMessage());
|
||||||
}
|
}
|
||||||
|
});
|
||||||
|
errorThread.start();
|
||||||
|
|
||||||
|
// Read stdout
|
||||||
|
String line;
|
||||||
|
while ((line = outputReader.readLine()) != null) {
|
||||||
|
System.err.println("psql stdout: " + line);
|
||||||
}
|
}
|
||||||
|
|
||||||
connection.commit();
|
errorThread.join();
|
||||||
System.err.println("Restore completed successfully. Executed " + successCount + " SQL statements.");
|
}
|
||||||
|
|
||||||
// Reindex search after successful restore
|
int exitCode = process.waitFor();
|
||||||
try {
|
if (exitCode != 0) {
|
||||||
String currentLibraryId = libraryService.getCurrentLibraryId();
|
throw new RuntimeException("psql restore failed with exit code: " + exitCode);
|
||||||
System.err.println("Starting search reindex after successful restore for library: " + currentLibraryId);
|
}
|
||||||
if (currentLibraryId == null) {
|
|
||||||
System.err.println("ERROR: No current library set during restore - cannot reindex search!");
|
|
||||||
throw new IllegalStateException("No current library active during restore");
|
|
||||||
}
|
|
||||||
|
|
||||||
// Manually trigger reindexing using the correct database connection
|
System.err.println("PostgreSQL restore completed successfully");
|
||||||
System.err.println("Triggering manual reindex from library-specific database for library: " + currentLibraryId);
|
|
||||||
reindexStoriesAndAuthorsFromCurrentDatabase();
|
|
||||||
|
|
||||||
// Note: Collections collection will be recreated when needed by the service
|
// Reindex search after successful restore
|
||||||
System.err.println("Search reindex completed successfully for library: " + currentLibraryId);
|
try {
|
||||||
} catch (Exception e) {
|
String currentLibraryId = libraryService.getCurrentLibraryId();
|
||||||
// Log the error but don't fail the restore
|
System.err.println("Starting search reindex after successful restore for library: " + currentLibraryId);
|
||||||
System.err.println("Warning: Failed to reindex search after restore: " + e.getMessage());
|
if (currentLibraryId == null) {
|
||||||
e.printStackTrace();
|
System.err.println("ERROR: No current library set during restore - cannot reindex search!");
|
||||||
|
throw new IllegalStateException("No current library active during restore");
|
||||||
}
|
}
|
||||||
|
|
||||||
} catch (SQLException e) {
|
// Manually trigger reindexing using the correct database connection
|
||||||
connection.rollback();
|
System.err.println("Triggering manual reindex from library-specific database for library: " + currentLibraryId);
|
||||||
throw e;
|
reindexStoriesAndAuthorsFromCurrentDatabase();
|
||||||
} finally {
|
|
||||||
connection.setAutoCommit(true);
|
// Note: Collections collection will be recreated when needed by the service
|
||||||
|
System.err.println("Search reindex completed successfully for library: " + currentLibraryId);
|
||||||
|
} catch (Exception e) {
|
||||||
|
// Log the error but don't fail the restore
|
||||||
|
System.err.println("Warning: Failed to reindex search after restore: " + e.getMessage());
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
throw new RuntimeException("Restore process was interrupted", e);
|
||||||
|
} finally {
|
||||||
|
// Clean up temporary file
|
||||||
|
try {
|
||||||
|
Files.deleteIfExists(tempBackupFile);
|
||||||
|
} catch (IOException e) {
|
||||||
|
System.err.println("Warning: Could not delete temporary restore file: " + e.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user