Add watermark normalization to fetch-lyrics

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

docs/decision-log.md

@@ -374,3 +374,17 @@
   層；(3) 複雜仲裁機械無以自證其益（見前條）。驗證改為
   確定性格式檢查（完整分割、組名唯一、cap ≤ 50），違規
   依協定修訂定義檔重跑。
+
+## 2026-08-05
+
+- **歌詞捕捉加浮水印／誤碼正規化**：全語料普查（程式窮舉
+  非 ASCII）發現歌詞站浮水印與 cp1252 誤碼——同形字（西里爾
+  е 160 處／78 檔、希臘 ό 6 處）、異體空格（U+2005 211 處、
+  U+205F 80 處、U+200A 6 處）、零寬字元（U+200B 3 處）、C1
+  誤碼（U+0091/92/93/94/97 共 58 處；先前手修之 U+0085 同
+  類）；受影響 81/883 檔全部來自 pilot 匯入，新抓取檔乾淨。
+  正規化規則四條（C1 依 cp1252 本義還原、同形字還原、異體
+  空格→ASCII 空格、零寬刪除）實作於 fetch-lyrics 的
+  normalize_lyrics()，寫入時套用；既有語料以同一函式一次性
+  套用。倒放行等歌曲本體內容不動——策展界線：清傳輸雜質，
+  不改內容。

tools/src/pop_fem_audit_tools/commands/fetch_lyrics.py

@@ -57,6 +57,49 @@ SLEEP_SECONDS: float = 1.0
 """The delay between consecutive HTTP requests, in seconds."""
 
 
+def _build_normalization() -> dict[int, str | None]:
+    """Build the lyrics normalization translation table.
+
+    :return: The codepoint-to-replacement mapping, a replacement
+        of None meaning removal.
+    """
+    table: dict[int, str | None] = {}
+    codepoint: int
+    for codepoint in range(0x80, 0xa0):
+        try:
+            table[codepoint] = bytes([codepoint]).decode("cp1252")
+        except UnicodeDecodeError:
+            table[codepoint] = None
+    table[0x0435] = "e"
+    table[0x03cc] = "ó"
+    for codepoint in (0x2005, 0x205f, 0x200a):
+        table[codepoint] = " "
+    for codepoint in (0x200b, 0x200c, 0x200d, 0xfeff):
+        table[codepoint] = None
+    return table
+
+
+NORMALIZATION: dict[int, str | None] = _build_normalization()
+"""The codepoint-to-replacement mapping applied to fetched
+lyrics: cp1252-mojibake restoration for U+0080-U+009F (with the
+five byte values undefined in cp1252 removed), homoglyph
+restoration for the Cyrillic "e" and the Greek "o" with tonos,
+ASCII-space restoration for exotic space variants, and removal
+of zero-width characters.  A replacement of None removes the
+codepoint."""
+
+
+def normalize_lyrics(text: str) -> str:
+    """Restore or remove watermark and mojibake characters.
+
+    :param text: The lyrics text as fetched from an API.
+    :return: The text with the codepoints in
+        :data:`NORMALIZATION` replaced or removed; every other
+        character is unchanged.
+    """
+    return text.translate(NORMALIZATION)
+
+
 def parse_args(argv: list[str] | None) -> argparse.Namespace:
     """Parse the command-line arguments.
 
@@ -187,6 +230,9 @@ def save_lyrics(lyrics_dir: Path, song_id: int,
 
     The cache directory is created when missing.
 
+    The lyrics text is normalized with :func:`normalize_lyrics`
+    before being written.
+
     :param lyrics_dir: The lyrics cache directory.
     :param song_id: The song ID.
     :param lyrics: The lyrics text.
@@ -195,7 +241,7 @@ def save_lyrics(lyrics_dir: Path, song_id: int,
     """
     lyrics_dir.mkdir(parents=True, exist_ok=True)
     (lyrics_dir / f"{song_id}.txt").write_text(
-        lyrics, encoding="utf-8")
+        normalize_lyrics(lyrics), encoding="utf-8")
 
 
 def append_provenance(path: Path, song_id: int,

tools/tests/test_fetch_lyrics.py

@@ -317,6 +317,79 @@ class TestFetchLyrics(unittest.TestCase):
         self.assertEqual([x[0] for x in rows[1:]],
                          ["1", "2", "2"])
 
+    def test_normalize_cp1252_mojibake(self) -> None:
+        """Test that cp1252 mojibake codepoints are restored."""
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics("wait…"),
+            "wait…")
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics(
+                "‘quote’"),
+            "‘quote’")
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics(
+                "“quote”"),
+            "“quote”")
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics("dash—line"),
+            "dash—line")
+
+    def test_normalize_undefined_cp1252_removed(self) -> None:
+        """Test that undefined cp1252 byte values are removed."""
+        text: str = ("abcdef")
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics(text), "abcdef")
+
+    def test_normalize_homoglyphs(self) -> None:
+        """Test that watermark homoglyphs are restored."""
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics("likе that"),
+            "like that")
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics("lό que soy"),
+            "ló que soy")
+
+    def test_normalize_space_variants(self) -> None:
+        """Test that exotic space variants become ASCII space."""
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics(
+                "a b c d"),
+            "a b c d")
+
+    def test_normalize_zero_width_removed(self) -> None:
+        """Test that zero-width characters are removed."""
+        text: str = (
+            "a​b‌c‍de")
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics(text), "abcde")
+
+    def test_normalize_ascii_unchanged(self) -> None:
+        """Test that plain ASCII text passes through unchanged."""
+        text: str = "Hello, it's me\n"
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics(text), text)
+
+    def test_normalize_legitimate_non_ascii_unchanged(self) -> None:
+        """Test that legitimate non-ASCII content is unchanged."""
+        text: str = "¿cómo estás? 안녕하세요\n"
+        self.assertEqual(
+            fetch_lyrics.normalize_lyrics(text), text)
+
+    def test_fetched_lyrics_saved_normalized(self) -> None:
+        """Test that a fetched lyric is normalized before saving."""
+        self.__seed([("Hello", "Adele")])
+        with mock.patch(
+                "urllib.request.urlopen",
+                side_effect=[self.__response(
+                    {"lyrics": "wait… likе"
+                               " that now​\n"})]):
+            status: int = self.__run_fetch()[0]
+        self.assertEqual(status, 0)
+        self.assertEqual(
+            (self.__lyrics / "1.txt")
+            .read_text(encoding="utf-8"),
+            "wait… like that now\n")
+
     def test_no_store_fails(self) -> None:
         """Test that a missing working store fails the run."""
         urlopen: mock.Mock