shardegger/storycove
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scraping and improvements

Browse Source
This commit is contained in:
Stefan Hardegger
2025-07-28 13:52:09 +02:00
parent f95d7aa8bb
commit fcad028959
31 changed files with 3788 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
backend/src/main/java/com/storycove/controller/AuthorController.java
View File

@@ -65,10 +65,12 @@ public class AuthorController {
@PostMapping
public ResponseEntity<AuthorDto> createAuthor(@Valid @RequestBody CreateAuthorRequest request) {
logger.info("Creating new author: {}", request.getName());
Author author = new Author();
updateAuthorFromRequest(author, request);
Author savedAuthor = authorService.create(author);
logger.info("Successfully created author: {} (ID: {})", savedAuthor.getName(), savedAuthor.getId());
return ResponseEntity.status(HttpStatus.CREATED).body(convertToDto(savedAuthor));
}
@@ -81,13 +83,7 @@ public class AuthorController {
@RequestParam(required = false, name = "authorRating") Integer rating,
@RequestParam(required = false, name = "avatar") MultipartFile avatarFile) {
System.out.println("DEBUG: MULTIPART PUT called with:");
System.out.println(" - name: " + name);
System.out.println(" - notes: " + notes);
System.out.println(" - urls: " + urls);
System.out.println(" - rating: " + rating);
System.out.println(" - avatar: " + (avatarFile != null ? avatarFile.getOriginalFilename() : "null"));
logger.info("Updating author with multipart data (ID: {})", id);
try {
Author existingAuthor = authorService.findById(id);
@@ -104,7 +100,6 @@ public class AuthorController {
// Handle rating update
if (rating != null) {
System.out.println("DEBUG: Setting author rating via PUT: " + rating);
existingAuthor.setAuthorRating(rating);
}
@@ -115,6 +110,7 @@ public class AuthorController {
}
Author updatedAuthor = authorService.update(id, existingAuthor);
logger.info("Successfully updated author: {} via multipart", updatedAuthor.getName());
return ResponseEntity.ok(convertToDto(updatedAuthor));
} catch (Exception e) {
@@ -125,31 +121,27 @@ public class AuthorController {
@PutMapping(value = "/{id}", consumes = "application/json")
public ResponseEntity<AuthorDto> updateAuthorJson(@PathVariable UUID id,
@Valid @RequestBody UpdateAuthorRequest request) {
System.out.println("DEBUG: JSON PUT called with:");
System.out.println(" - name: " + request.getName());
System.out.println(" - notes: " + request.getNotes());
System.out.println(" - urls: " + request.getUrls());
System.out.println(" - rating: " + request.getRating());
logger.info("Updating author with JSON data: {} (ID: {})", request.getName(), id);
Author existingAuthor = authorService.findById(id);
updateAuthorFromRequest(existingAuthor, request);
Author updatedAuthor = authorService.update(id, existingAuthor);
logger.info("Successfully updated author: {} via JSON", updatedAuthor.getName());
return ResponseEntity.ok(convertToDto(updatedAuthor));
}
@PutMapping("/{id}")
public ResponseEntity<String> updateAuthorGeneric(@PathVariable UUID id, HttpServletRequest request) {
System.out.println("DEBUG: GENERIC PUT called!");
System.out.println(" - Content-Type: " + request.getContentType());
System.out.println(" - Method: " + request.getMethod());
return ResponseEntity.status(415).body("Unsupported Media Type. Expected multipart/form-data or application/json");
}
@DeleteMapping("/{id}")
public ResponseEntity<?> deleteAuthor(@PathVariable UUID id) {
logger.info("Deleting author with ID: {}", id);
authorService.delete(id);
logger.info("Successfully deleted author with ID: {}", id);
return ResponseEntity.ok(Map.of("message", "Author deleted successfully"));
}
@@ -177,11 +169,8 @@ public class AuthorController {
@PostMapping("/{id}/rating")
public ResponseEntity<AuthorDto> rateAuthor(@PathVariable UUID id, @RequestBody RatingRequest request) {
System.out.println("DEBUG: Rating author " + id + " with rating " + request.getRating());
Author author = authorService.setRating(id, request.getRating());
System.out.println("DEBUG: After setRating, author rating is: " + author.getAuthorRating());
AuthorDto dto = convertToDto(author);
System.out.println("DEBUG: Final DTO rating is: " + dto.getAuthorRating());
return ResponseEntity.ok(dto);
}
@@ -211,9 +200,7 @@ public class AuthorController {
@PostMapping("/{id}/test-rating/{rating}")
public ResponseEntity<Map<String, Object>> testSetRating(@PathVariable UUID id, @PathVariable Integer rating) {
try {
System.out.println("DEBUG: Test setting rating " + rating + " for author " + id);
Author author = authorService.setRating(id, rating);
System.out.println("DEBUG: After test setRating, got: " + author.getAuthorRating());
return ResponseEntity.ok(Map.of(
"success", true,
@@ -231,13 +218,11 @@ public class AuthorController {
@PostMapping("/{id}/test-put-rating")
public ResponseEntity<Map<String, Object>> testPutWithRating(@PathVariable UUID id, @RequestParam Integer rating) {
try {
System.out.println("DEBUG: Test PUT with rating " + rating + " for author " + id);
Author existingAuthor = authorService.findById(id);
existingAuthor.setAuthorRating(rating);
Author updatedAuthor = authorService.update(id, existingAuthor);
System.out.println("DEBUG: After PUT update, rating is: " + updatedAuthor.getAuthorRating());
return ResponseEntity.ok(Map.of(
"success", true,
@@ -389,7 +374,6 @@ public class AuthorController {
author.setUrls(updateReq.getUrls());
}
if (updateReq.getRating() != null) {
System.out.println("DEBUG: Setting author rating via JSON: " + updateReq.getRating());
author.setAuthorRating(updateReq.getRating());
}
}
@@ -402,9 +386,6 @@ public class AuthorController {
dto.setNotes(author.getNotes());
dto.setAvatarImagePath(author.getAvatarImagePath());
// Debug logging for author rating
System.out.println("DEBUG: Converting author " + author.getName() +
" with rating: " + author.getAuthorRating());
dto.setAuthorRating(author.getAuthorRating());
dto.setUrls(author.getUrls());
@@ -415,7 +396,6 @@ public class AuthorController {
// Calculate and set average story rating
dto.setAverageStoryRating(authorService.calculateAverageStoryRating(author.getId()));
System.out.println("DEBUG: DTO authorRating set to: " + dto.getAuthorRating());
return dto;
}

9
backend/src/main/java/com/storycove/controller/CollectionController.java
View File

@@ -56,8 +56,6 @@ public class CollectionController {
@RequestParam(required = false) List<String> tags,
@RequestParam(defaultValue = "false") boolean archived) {
logger.info("COLLECTIONS: Search request - search='{}', tags={}, archived={}, page={}, limit={}",
search, tags, archived, page, limit);
// MANDATORY: Use Typesense for all search/filter operations
SearchResultDto<Collection> results = collectionService.searchCollections(search, tags, archived, page, limit);
@@ -94,13 +92,14 @@ public class CollectionController {
*/
@PostMapping
public ResponseEntity<Collection> createCollection(@Valid @RequestBody CreateCollectionRequest request) {
logger.info("Creating new collection: {}", request.getName());
Collection collection = collectionService.createCollection(
request.getName(),
request.getDescription(),
request.getTagNames(),
request.getStoryIds()
);
logger.info("Successfully created collection: {} (ID: {})", collection.getName(), collection.getId());
return ResponseEntity.status(HttpStatus.CREATED).body(collection);
}
@@ -115,6 +114,7 @@ public class CollectionController {
@RequestParam(required = false) List<UUID> storyIds,
@RequestParam(required = false, name = "coverImage") MultipartFile coverImage) {
logger.info("Creating new collection with image: {}", name);
try {
// Create collection first
Collection collection = collectionService.createCollection(name, description, tags, storyIds);
@@ -128,6 +128,7 @@ public class CollectionController {
);
}
logger.info("Successfully created collection with image: {} (ID: {})", collection.getName(), collection.getId());
return ResponseEntity.status(HttpStatus.CREATED).body(collection);
} catch (Exception e) {
@@ -160,7 +161,9 @@ public class CollectionController {
*/
@DeleteMapping("/{id}")
public ResponseEntity<Map<String, String>> deleteCollection(@PathVariable UUID id) {
logger.info("Deleting collection with ID: {}", id);
collectionService.deleteCollection(id);
logger.info("Successfully deleted collection with ID: {}", id);
return ResponseEntity.ok(Map.of("message", "Collection deleted successfully"));
}

7
backend/src/main/java/com/storycove/controller/StoryController.java
View File

@@ -86,23 +86,29 @@ public class StoryController {
@PostMapping
public ResponseEntity<StoryDto> createStory(@Valid @RequestBody CreateStoryRequest request) {
logger.info("Creating new story: {}", request.getTitle());
Story story = new Story();
updateStoryFromRequest(story, request);
Story savedStory = storyService.createWithTagNames(story, request.getTagNames());
logger.info("Successfully created story: {} (ID: {})", savedStory.getTitle(), savedStory.getId());
return ResponseEntity.status(HttpStatus.CREATED).body(convertToDto(savedStory));
}
@PutMapping("/{id}")
public ResponseEntity<StoryDto> updateStory(@PathVariable UUID id,
@Valid @RequestBody UpdateStoryRequest request) {
logger.info("Updating story: {} (ID: {})", request.getTitle(), id);
Story updatedStory = storyService.updateWithTagNames(id, request);
logger.info("Successfully updated story: {}", updatedStory.getTitle());
return ResponseEntity.ok(convertToDto(updatedStory));
}
@DeleteMapping("/{id}")
public ResponseEntity<?> deleteStory(@PathVariable UUID id) {
logger.info("Deleting story with ID: {}", id);
storyService.delete(id);
logger.info("Successfully deleted story with ID: {}", id);
return ResponseEntity.ok(Map.of("message", "Story deleted successfully"));
}
@@ -212,7 +218,6 @@ public class StoryController {
@RequestParam(required = false) String sortBy,
@RequestParam(required = false) String sortDir) {
logger.info("CONTROLLER DEBUG: Search request - query='{}', tags={}, authors={}", query, tags, authors);
if (typesenseService != null) {
SearchResultDto<StorySearchDto> results = typesenseService.searchStories(

82
backend/src/main/java/com/storycove/service/AuthorService.java
View File

@@ -31,7 +31,7 @@ public class AuthorService {
private final TypesenseService typesenseService;
@Autowired
public AuthorService(AuthorRepository authorRepository, TypesenseService typesenseService) {
public AuthorService(AuthorRepository authorRepository, @Autowired(required = false) TypesenseService typesenseService) {
this.authorRepository = authorRepository;
this.typesenseService = typesenseService;
}
@@ -133,10 +133,12 @@ public class AuthorService {
Author savedAuthor = authorRepository.save(author);
// Index in Typesense
try {
typesenseService.indexAuthor(savedAuthor);
} catch (Exception e) {
logger.warn("Failed to index author in Typesense: " + savedAuthor.getName(), e);
if (typesenseService != null) {
try {
typesenseService.indexAuthor(savedAuthor);
} catch (Exception e) {
logger.warn("Failed to index author in Typesense: " + savedAuthor.getName(), e);
}
}
return savedAuthor;
@@ -155,10 +157,12 @@ public class AuthorService {
Author savedAuthor = authorRepository.save(existingAuthor);
// Update in Typesense
try {
typesenseService.updateAuthor(savedAuthor);
} catch (Exception e) {
logger.warn("Failed to update author in Typesense: " + savedAuthor.getName(), e);
if (typesenseService != null) {
try {
typesenseService.updateAuthor(savedAuthor);
} catch (Exception e) {
logger.warn("Failed to update author in Typesense: " + savedAuthor.getName(), e);
}
}
return savedAuthor;
@@ -175,10 +179,12 @@ public class AuthorService {
authorRepository.delete(author);
// Remove from Typesense
try {
typesenseService.deleteAuthor(id.toString());
} catch (Exception e) {
logger.warn("Failed to delete author from Typesense: " + author.getName(), e);
if (typesenseService != null) {
try {
typesenseService.deleteAuthor(id.toString());
} catch (Exception e) {
logger.warn("Failed to delete author from Typesense: " + author.getName(), e);
}
}
}
@@ -188,10 +194,12 @@ public class AuthorService {
Author savedAuthor = authorRepository.save(author);
// Update in Typesense
try {
typesenseService.updateAuthor(savedAuthor);
} catch (Exception e) {
logger.warn("Failed to update author in Typesense after adding URL: " + savedAuthor.getName(), e);
if (typesenseService != null) {
try {
typesenseService.updateAuthor(savedAuthor);
} catch (Exception e) {
logger.warn("Failed to update author in Typesense after adding URL: " + savedAuthor.getName(), e);
}
}
return savedAuthor;
@@ -203,10 +211,12 @@ public class AuthorService {
Author savedAuthor = authorRepository.save(author);
// Update in Typesense
try {
typesenseService.updateAuthor(savedAuthor);
} catch (Exception e) {
logger.warn("Failed to update author in Typesense after removing URL: " + savedAuthor.getName(), e);
if (typesenseService != null) {
try {
typesenseService.updateAuthor(savedAuthor);
} catch (Exception e) {
logger.warn("Failed to update author in Typesense after removing URL: " + savedAuthor.getName(), e);
}
}
return savedAuthor;
@@ -242,10 +252,12 @@ public class AuthorService {
refreshedAuthor.getAuthorRating(), refreshedAuthor.getName());
// Update in Typesense
try {
typesenseService.updateAuthor(refreshedAuthor);
} catch (Exception e) {
logger.warn("Failed to update author in Typesense after rating: " + refreshedAuthor.getName(), e);
if (typesenseService != null) {
try {
typesenseService.updateAuthor(refreshedAuthor);
} catch (Exception e) {
logger.warn("Failed to update author in Typesense after rating: " + refreshedAuthor.getName(), e);
}
}
return refreshedAuthor;
@@ -290,10 +302,12 @@ public class AuthorService {
Author savedAuthor = authorRepository.save(author);
// Update in Typesense
try {
typesenseService.updateAuthor(savedAuthor);
} catch (Exception e) {
logger.warn("Failed to update author in Typesense after setting avatar: " + savedAuthor.getName(), e);
if (typesenseService != null) {
try {
typesenseService.updateAuthor(savedAuthor);
} catch (Exception e) {
logger.warn("Failed to update author in Typesense after setting avatar: " + savedAuthor.getName(), e);
}
}
return savedAuthor;
@@ -305,10 +319,12 @@ public class AuthorService {
Author savedAuthor = authorRepository.save(author);
// Update in Typesense
try {
typesenseService.updateAuthor(savedAuthor);
} catch (Exception e) {
logger.warn("Failed to update author in Typesense after removing avatar: " + savedAuthor.getName(), e);
if (typesenseService != null) {
try {
typesenseService.updateAuthor(savedAuthor);
} catch (Exception e) {
logger.warn("Failed to update author in Typesense after removing avatar: " + savedAuthor.getName(), e);
}
}
return savedAuthor;

19
backend/src/main/java/com/storycove/service/TypesenseService.java
View File

@@ -209,8 +209,6 @@ public class TypesenseService {
try {
long startTime = System.currentTimeMillis();
logger.info("SEARCH DEBUG: searchStories called with query='{}', tagFilters={}, authorFilters={}",
query, tagFilters, authorFilters);
// Convert 0-based page (frontend/backend) to 1-based page (Typesense)
int typesensePage = page + 1;
@@ -242,15 +240,12 @@ public class TypesenseService {
}
if (tagFilters != null && !tagFilters.isEmpty()) {
logger.info("SEARCH DEBUG: Processing {} tag filters: {}", tagFilters.size(), tagFilters);
// Use AND logic for multiple tags - items must have ALL selected tags
for (String tag : tagFilters) {
String escaped = escapeTypesenseValue(tag);
String condition = "tagNames:=" + escaped;
logger.info("SEARCH DEBUG: Tag '{}' -> escaped '{}' -> condition '{}'", tag, escaped, condition);
filterConditions.add(condition);
}
logger.info("SEARCH DEBUG: Added {} individual tag filter conditions", tagFilters.size());
}
if (minRating != null) {
@@ -263,17 +258,14 @@ public class TypesenseService {
if (!filterConditions.isEmpty()) {
String finalFilter = String.join(" && ", filterConditions);
logger.info("SEARCH DEBUG: Final filter condition: '{}'", finalFilter);
searchParameters.filterBy(finalFilter);
} else {
logger.info("SEARCH DEBUG: No filter conditions applied");
}
SearchResult searchResult = typesenseClient.collections(STORIES_COLLECTION)
.documents()
.search(searchParameters);
logger.info("SEARCH DEBUG: Typesense returned {} results", searchResult.getFound());
List<StorySearchDto> results = convertSearchResult(searchResult);
long searchTime = System.currentTimeMillis() - startTime;
@@ -377,10 +369,8 @@ public class TypesenseService {
List<String> tagNames = story.getTags().stream()
.map(tag -> tag.getName())
.collect(Collectors.toList());
logger.debug("INDEXING DEBUG: Story '{}' has tags: {}", story.getTitle(), tagNames);
document.put("tagNames", tagNames);
} else {
logger.debug("INDEXING DEBUG: Story '{}' has no tags", story.getTitle());
}
document.put("rating", story.getRating() != null ? story.getRating() : 0);
@@ -746,8 +736,6 @@ public class TypesenseService {
public SearchResultDto<AuthorSearchDto> searchAuthors(String query, int page, int perPage, String sortBy, String sortOrder) {
try {
logger.info("AUTHORS SEARCH DEBUG: Searching collection '{}' with query='{}', sortBy='{}', sortOrder='{}'",
AUTHORS_COLLECTION, query, sortBy, sortOrder);
SearchParameters searchParameters = new SearchParameters()
.q(query != null && !query.trim().isEmpty() ? query : "*")
.queryBy("name,notes")
@@ -759,8 +747,6 @@ public class TypesenseService {
String sortDirection = "desc".equalsIgnoreCase(sortOrder) ? "desc" : "asc";
String sortField = mapAuthorSortField(sortBy);
String sortString = sortField + ":" + sortDirection;
logger.info("AUTHORS SEARCH DEBUG: Original sortBy='{}', mapped to='{}', full sort string='{}'",
sortBy, sortField, sortString);
searchParameters.sortBy(sortString);
}
@@ -771,17 +757,12 @@ public class TypesenseService {
.search(searchParameters);
} catch (Exception sortException) {
// If sorting fails (likely due to schema issues), retry without sorting
logger.error("SORTING ERROR DEBUG: Full exception details", sortException);
logger.warn("Sorting failed for authors search, retrying without sort: " + sortException.getMessage());
// Try to get collection info for debugging
try {
CollectionResponse collection = typesenseClient.collections(AUTHORS_COLLECTION).retrieve();
logger.error("COLLECTION DEBUG: Collection '{}' exists with {} documents and {} fields",
collection.getName(), collection.getNumDocuments(), collection.getFields().size());
logger.error("COLLECTION DEBUG: Fields: {}", collection.getFields());
} catch (Exception debugException) {
logger.error("COLLECTION DEBUG: Failed to retrieve collection info", debugException);
}
searchParameters = new SearchParameters()

135
backend/src/test/java/com/storycove/service/AuthorServiceTest.java
View File

@@ -1,6 +1,7 @@
package com.storycove.service;
import com.storycove.entity.Author;
import com.storycove.entity.Story;
import com.storycove.repository.AuthorRepository;
import com.storycove.service.exception.DuplicateResourceException;
import com.storycove.service.exception.ResourceNotFoundException;
@@ -24,6 +25,7 @@ import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.*;
import static org.mockito.Mockito.times;
@ExtendWith(MockitoExtension.class)
@DisplayName("Author Service Unit Tests")
@@ -32,7 +34,6 @@ class AuthorServiceTest {
@Mock
private AuthorRepository authorRepository;
@InjectMocks
private AuthorService authorService;
private Author testAuthor;
@@ -44,6 +45,9 @@ class AuthorServiceTest {
testAuthor = new Author("Test Author");
testAuthor.setId(testId);
testAuthor.setNotes("Test notes");
// Initialize service with null TypesenseService (which is allowed)
authorService = new AuthorService(authorRepository, null);
}
@Test
@@ -307,4 +311,133 @@ class AuthorServiceTest {
assertEquals(5L, count);
verify(authorRepository).countRecentAuthors(any(java.time.LocalDateTime.class));
}
@Test
@DisplayName("Should set author rating with validation")
void shouldSetAuthorRating() {
when(authorRepository.findById(testId)).thenReturn(Optional.of(testAuthor));
when(authorRepository.save(any(Author.class))).thenReturn(testAuthor);
Author result = authorService.setRating(testId, 4);
assertEquals(4, testAuthor.getAuthorRating());
verify(authorRepository, times(2)).findById(testId); // Called twice: once initially, once after flush
verify(authorRepository).save(testAuthor);
verify(authorRepository).flush();
}
@Test
@DisplayName("Should throw exception for invalid rating range")
void shouldThrowExceptionForInvalidRating() {
assertThrows(IllegalArgumentException.class, () -> authorService.setRating(testId, 0));
assertThrows(IllegalArgumentException.class, () -> authorService.setRating(testId, 6));
verify(authorRepository, never()).findById(any());
verify(authorRepository, never()).save(any());
}
@Test
@DisplayName("Should handle null rating")
void shouldHandleNullRating() {
when(authorRepository.findById(testId)).thenReturn(Optional.of(testAuthor));
when(authorRepository.save(any(Author.class))).thenReturn(testAuthor);
Author result = authorService.setRating(testId, null);
assertNull(testAuthor.getAuthorRating());
verify(authorRepository, times(2)).findById(testId); // Called twice: once initially, once after flush
verify(authorRepository).save(testAuthor);
}
@Test
@DisplayName("Should find all authors with stories")
void shouldFindAllAuthorsWithStories() {
List<Author> authors = List.of(testAuthor);
when(authorRepository.findAll()).thenReturn(authors);
List<Author> result = authorService.findAllWithStories();
assertEquals(1, result.size());
verify(authorRepository).findAll();
}
@Test
@DisplayName("Should get author rating from database")
void shouldGetAuthorRatingFromDb() {
when(authorRepository.findAuthorRatingById(testId)).thenReturn(4);
Integer rating = authorService.getAuthorRatingFromDb(testId);
assertEquals(4, rating);
verify(authorRepository).findAuthorRatingById(testId);
}
@Test
@DisplayName("Should calculate average story rating")
void shouldCalculateAverageStoryRating() {
// Setup test author with stories
Story story1 = new Story("Story 1");
story1.setRating(4);
Story story2 = new Story("Story 2");
story2.setRating(5);
testAuthor.getStories().add(story1);
testAuthor.getStories().add(story2);
when(authorRepository.findById(testId)).thenReturn(Optional.of(testAuthor));
Double avgRating = authorService.calculateAverageStoryRating(testId);
assertEquals(4.5, avgRating);
verify(authorRepository).findById(testId);
}
@Test
@DisplayName("Should find authors with stories using repository method")
void shouldFindAuthorsWithStoriesFromRepository() {
List<Author> authors = List.of(testAuthor);
when(authorRepository.findAuthorsWithStories()).thenReturn(authors);
List<Author> result = authorService.findAuthorsWithStories();
assertEquals(1, result.size());
verify(authorRepository).findAuthorsWithStories();
}
@Test
@DisplayName("Should find top rated authors")
void shouldFindTopRatedAuthors() {
List<Author> authors = List.of(testAuthor);
when(authorRepository.findTopRatedAuthors()).thenReturn(authors);
List<Author> result = authorService.findTopRatedAuthors();
assertEquals(1, result.size());
verify(authorRepository).findTopRatedAuthors();
}
@Test
@DisplayName("Should find most prolific authors")
void shouldFindMostProlificAuthors() {
List<Author> authors = List.of(testAuthor);
when(authorRepository.findMostProlificAuthors()).thenReturn(authors);
List<Author> result = authorService.findMostProlificAuthors();
assertEquals(1, result.size());
verify(authorRepository).findMostProlificAuthors();
}
@Test
@DisplayName("Should find authors by URL domain")
void shouldFindAuthorsByUrlDomain() {
List<Author> authors = List.of(testAuthor);
when(authorRepository.findByUrlDomain("example.com")).thenReturn(authors);
List<Author> result = authorService.findByUrlDomain("example.com");
assertEquals(1, result.size());
verify(authorRepository).findByUrlDomain("example.com");
}
}