diff --git a/backend/src/main/java/com/storycove/entity/Story.java b/backend/src/main/java/com/storycove/entity/Story.java index dde96e5..e87d129 100644 --- a/backend/src/main/java/com/storycove/entity/Story.java +++ b/backend/src/main/java/com/storycove/entity/Story.java @@ -8,6 +8,7 @@ import org.hibernate.annotations.UpdateTimestamp; import org.jsoup.Jsoup; import com.fasterxml.jackson.annotation.JsonManagedReference; import com.fasterxml.jackson.annotation.JsonBackReference; +import com.storycove.util.TextSanitizer; import java.time.LocalDateTime; import java.util.HashSet; @@ -171,8 +172,8 @@ public class Story { } public void setContentHtml(String contentHtml) { - this.contentHtml = contentHtml; - this.setContentPlain(Jsoup.parse(contentHtml).text()); + this.contentHtml = TextSanitizer.sanitize(contentHtml); + this.setContentPlain(Jsoup.parse(this.contentHtml).text()); updateWordCount(); } diff --git a/backend/src/main/java/com/storycove/util/TextSanitizer.java b/backend/src/main/java/com/storycove/util/TextSanitizer.java new file mode 100644 index 0000000..2c62e1a --- /dev/null +++ b/backend/src/main/java/com/storycove/util/TextSanitizer.java @@ -0,0 +1,105 @@ +package com.storycove.util; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.TextNode; + +/** + * Normalizes the text nodes of story HTML so that invisible Unicode cruft (commonly left behind by + * EPUB/Calibre conversions) can never reach persisted story content. Only text node content is touched; + * tag structure and attributes pass through unchanged. + */ +public final class TextSanitizer { + + private static final int MIN_COLLAPSIBLE_RUN = 3; + + private TextSanitizer() { + } + + /** + * Strips Unicode "Cf" (format) characters and collapses runs of 3+ whitespace-like characters + * down to a single space, within text nodes only. + */ + public static String sanitize(String html) { + if (html == null || html.trim().isEmpty()) { + return html; + } + + Document doc = Jsoup.parseBodyFragment(html); + doc.outputSettings().prettyPrint(false); + + doc.body().traverse((node, depth) -> { + // Skip text nodes that are pure structural whitespace (indentation/formatting + // between tags, e.g. " ", "\n "): isBlank() only matches ' ', '\t', '\n', '\r', + // '\f', so any node containing real prose, Cf chars, or NBSP spam still gets processed. + if (node instanceof TextNode textNode && !textNode.isBlank()) { + String original = textNode.getWholeText(); + String normalized = normalizeText(original); + if (!normalized.equals(original)) { + textNode.text(normalized); + } + } + }); + + return doc.body().html(); + } + + static String normalizeText(String text) { + String withoutFormatChars = stripFormatCharacters(text); + return collapseLongWhitespaceRuns(withoutFormatChars); + } + + private static String stripFormatCharacters(String text) { + StringBuilder result = new StringBuilder(text.length()); + int i = 0; + while (i < text.length()) { + int codePoint = text.codePointAt(i); + int charCount = Character.charCount(codePoint); + if (Character.getType(codePoint) != Character.FORMAT) { + result.appendCodePoint(codePoint); + } + i += charCount; + } + return result.toString(); + } + + private static String collapseLongWhitespaceRuns(String text) { + StringBuilder result = new StringBuilder(text.length()); + int len = text.length(); + int i = 0; + while (i < len) { + int codePoint = text.codePointAt(i); + if (isSpaceLike(codePoint)) { + int runStart = i; + int runLength = 0; + int j = i; + while (j < len) { + int cp = text.codePointAt(j); + if (!isSpaceLike(cp)) { + break; + } + runLength++; + j += Character.charCount(cp); + } + if (runLength >= MIN_COLLAPSIBLE_RUN) { + result.append(' '); + } else { + result.append(text, runStart, j); + } + i = j; + } else { + result.appendCodePoint(codePoint); + i += Character.charCount(codePoint); + } + } + return result.toString(); + } + + /** + * Mirrors Python's str.isspace(): Unicode space separators (Zs/Zl/Zp, which includes NBSP) + * plus the ASCII/control whitespace characters (tab, newline, etc.). + */ + private static boolean isSpaceLike(int codePoint) { + return Character.isWhitespace(codePoint) || Character.isSpaceChar(codePoint); + } +} diff --git a/backend/src/test/java/com/storycove/util/TextSanitizerTest.java b/backend/src/test/java/com/storycove/util/TextSanitizerTest.java new file mode 100644 index 0000000..44abe45 --- /dev/null +++ b/backend/src/test/java/com/storycove/util/TextSanitizerTest.java @@ -0,0 +1,138 @@ +package com.storycove.util; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +@DisplayName("TextSanitizer Tests") +class TextSanitizerTest { + + private static final String ZWSP = ""; // zero width space (Cf) + private static final String WORD_JOINER = ""; // Cf + private static final String BOM = ""; // zero width no-break space / BOM (Cf) + private static final String NBSP = " "; // non-breaking space (Zs, not Cf) + + @Test + @DisplayName("Should strip a large run of zero-width spaces embedded mid-sentence") + void shouldStripLargeRunOfZeroWidthSpacesMidSentence() { + String garbageRun = ZWSP.repeat(5000); + String html = "
The hero drew" + garbageRun + "his sword and charged.
"; + + String sanitized = TextSanitizer.sanitize(html); + + assertFalse(sanitized.contains(ZWSP), "Zero-width spaces should be fully stripped"); + assertEquals("The hero drewhis sword and charged.
", sanitized); + } + + @Test + @DisplayName("Should strip mixed Cf characters (word joiner, BOM) anywhere in text") + void shouldStripMixedFormatCharacters() { + String html = "Hello" + BOM + " " + WORD_JOINER + "world.
"; + + String sanitized = TextSanitizer.sanitize(html); + + assertFalse(sanitized.contains(BOM)); + assertFalse(sanitized.contains(WORD_JOINER)); + assertEquals("Hello world.
", sanitized); + } + + @Test + @DisplayName("Should collapse a long run of regular spaces used as a formatting hack") + void shouldCollapseLongRunOfRegularSpaces() { + String html = "He paused" + " ".repeat(200) + "then spoke.
"; + + String sanitized = TextSanitizer.sanitize(html); + + assertEquals("He paused then spoke.
", sanitized); + } + + @Test + @DisplayName("Should collapse a long run of NBSP characters down to a single space") + void shouldCollapseLongRunOfNbsp() { + String html = "Wait" + NBSP.repeat(500) + "for it.
"; + + String sanitized = TextSanitizer.sanitize(html); + + assertEquals("Wait for it.
", sanitized); + } + + @Test + @DisplayName("Should collapse a mixed run of tabs, NBSP, and regular spaces") + void shouldCollapseMixedWhitespaceRun() { + String html = "Stop" + " \t" + NBSP + " \t" + NBSP + "go.
"; + + String sanitized = TextSanitizer.sanitize(html); + + assertEquals("Stop go.
", sanitized); + } + + @Test + @DisplayName("Should leave a single space untouched") + void shouldLeaveSingleSpaceUntouched() { + String html = "One two three.
"; + + assertEquals(html, TextSanitizer.sanitize(html)); + } + + @Test + @DisplayName("Should leave a run of exactly two whitespace characters untouched") + void shouldLeaveTwoWhitespaceCharsUntouched() { + String html = "One two.
"; // two regular spaces + + assertEquals(html, TextSanitizer.sanitize(html)); + } + + @Test + @DisplayName("Should not mangle accented / non-ASCII characters") + void shouldPreserveAccentedCharacters() { + String html = "Café, naïve, über, 日本語, 😀 stays intact.
"; + + assertEquals(html, TextSanitizer.sanitize(html)); + } + + @Test + @DisplayName("Should preserve paragraph structure expressed via a double newline") + void shouldPreserveDoubleNewlineParagraphBreaks() { + String html = "First paragraph.\n\nSecond paragraph.
"; + + assertEquals(html, TextSanitizer.sanitize(html)); + } + + @Test + @DisplayName("Should preserve paragraph structure expressed viatags") + void shouldPreserveParagraphTags() { + String html = "
First paragraph.
Second paragraph.
"; + + assertEquals(html, TextSanitizer.sanitize(html)); + } + + @Test + @DisplayName("Should not alter whitespace used purely for HTML indentation between tags") + void shouldNotCollapseInterTagIndentation() { + // The "\n " nodes betweentags are pure structural indentation, not prose, + // and must be left completely untouched even though they are 3+ char whitespace runs. + String html = "
Once upon a time.
\nThe end.
\nThe old café on the corner of Rue de Bellévue had seen better days, " + + "but Anaïs still loved it.
" + + "\"We'll meet at dawn,\" she whispered, \"just like we always do.\"
"; + + assertEquals(html, TextSanitizer.sanitize(html)); + } + + @Test + @DisplayName("Should handle null and blank input gracefully") + void shouldHandleNullAndBlankInput() { + assertNull(TextSanitizer.sanitize(null)); + assertEquals("", TextSanitizer.sanitize("")); + } +}