Add watermark normalization to fetch-lyrics

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

tools/src/pop_fem_audit_tools/commands/fetch_lyrics.py

@@ -57,6 +57,49 @@ SLEEP_SECONDS: float = 1.0
 """The delay between consecutive HTTP requests, in seconds."""
 
 
+def _build_normalization() -> dict[int, str | None]:
+    """Build the lyrics normalization translation table.
+
+    :return: The codepoint-to-replacement mapping, a replacement
+        of None meaning removal.
+    """
+    table: dict[int, str | None] = {}
+    codepoint: int
+    for codepoint in range(0x80, 0xa0):
+        try:
+            table[codepoint] = bytes([codepoint]).decode("cp1252")
+        except UnicodeDecodeError:
+            table[codepoint] = None
+    table[0x0435] = "e"
+    table[0x03cc] = "ó"
+    for codepoint in (0x2005, 0x205f, 0x200a):
+        table[codepoint] = " "
+    for codepoint in (0x200b, 0x200c, 0x200d, 0xfeff):
+        table[codepoint] = None
+    return table
+
+
+NORMALIZATION: dict[int, str | None] = _build_normalization()
+"""The codepoint-to-replacement mapping applied to fetched
+lyrics: cp1252-mojibake restoration for U+0080-U+009F (with the
+five byte values undefined in cp1252 removed), homoglyph
+restoration for the Cyrillic "e" and the Greek "o" with tonos,
+ASCII-space restoration for exotic space variants, and removal
+of zero-width characters.  A replacement of None removes the
+codepoint."""
+
+
+def normalize_lyrics(text: str) -> str:
+    """Restore or remove watermark and mojibake characters.
+
+    :param text: The lyrics text as fetched from an API.
+    :return: The text with the codepoints in
+        :data:`NORMALIZATION` replaced or removed; every other
+        character is unchanged.
+    """
+    return text.translate(NORMALIZATION)
+
+
 def parse_args(argv: list[str] | None) -> argparse.Namespace:
     """Parse the command-line arguments.
 
@@ -187,6 +230,9 @@ def save_lyrics(lyrics_dir: Path, song_id: int,
 
     The cache directory is created when missing.
 
+    The lyrics text is normalized with :func:`normalize_lyrics`
+    before being written.
+
     :param lyrics_dir: The lyrics cache directory.
     :param song_id: The song ID.
     :param lyrics: The lyrics text.
@@ -195,7 +241,7 @@ def save_lyrics(lyrics_dir: Path, song_id: int,
     """
     lyrics_dir.mkdir(parents=True, exist_ok=True)
     (lyrics_dir / f"{song_id}.txt").write_text(
-        lyrics, encoding="utf-8")
+        normalize_lyrics(lyrics), encoding="utf-8")
 
 
 def append_provenance(path: Path, song_id: int,

tools/tests/test_fetch_lyrics.py

@@ -317,6 +317,79 @@ class TestFetchLyrics(unittest.TestCase):
         self.assertEqual([x[0] for x in rows[1:]],
                          ["1", "2", "2"])
 
+    def test_normalize_cp1252_mojibake(self) -> None:
+        """Test that cp1252 mojibake codepoints are restored."""
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics("wait…"),
+            "wait…")
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics(
+                "‘quote’"),
+            "‘quote’")
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics(
+                "“quote”"),
+            "“quote”")
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics("dash—line"),
+            "dash—line")
+
+    def test_normalize_undefined_cp1252_removed(self) -> None:
+        """Test that undefined cp1252 byte values are removed."""
+        text: str = ("abcdef")
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics(text), "abcdef")
+
+    def test_normalize_homoglyphs(self) -> None:
+        """Test that watermark homoglyphs are restored."""
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics("likе that"),
+            "like that")
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics("lό que soy"),
+            "ló que soy")
+
+    def test_normalize_space_variants(self) -> None:
+        """Test that exotic space variants become ASCII space."""
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics(
+                "a b c d"),
+            "a b c d")
+
+    def test_normalize_zero_width_removed(self) -> None:
+        """Test that zero-width characters are removed."""
+        text: str = (
+            "a​b‌c‍de")
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics(text), "abcde")
+
+    def test_normalize_ascii_unchanged(self) -> None:
+        """Test that plain ASCII text passes through unchanged."""
+        text: str = "Hello, it's me\n"
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics(text), text)
+
+    def test_normalize_legitimate_non_ascii_unchanged(self) -> None:
+        """Test that legitimate non-ASCII content is unchanged."""
+        text: str = "¿cómo estás? 안녕하세요\n"
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics(text), text)
+
+    def test_fetched_lyrics_saved_normalized(self) -> None:
+        """Test that a fetched lyric is normalized before saving."""
+        self.__seed([("Hello", "Adele")])
+        with mock.patch(
+                "urllib.request.urlopen",
+                side_effect=[self.__response(
+                    {"lyrics": "wait… likе"
+                               " that now​\n"})]):
+            status: int = self.__run_fetch()[0]
+        self.assertEqual(status, 0)
+        self.assertEqual(
+            (self.__lyrics / "1.txt")
+            .read_text(encoding="utf-8"),
+            "wait… like that now\n")
+
     def test_no_store_fails(self) -> None:
         """Test that a missing working store fails the run."""
         urlopen: mock.Mock