diff --git a/docs/decision-log.md b/docs/decision-log.md index 635cac0..67e27ca 100644 --- a/docs/decision-log.md +++ b/docs/decision-log.md @@ -374,3 +374,17 @@ 層;(3) 複雜仲裁機械無以自證其益(見前條)。驗證改為 確定性格式檢查(完整分割、組名唯一、cap ≤ 50),違規 依協定修訂定義檔重跑。 + +## 2026-08-05 + +- **歌詞捕捉加浮水印/誤碼正規化**:全語料普查(程式窮舉 + 非 ASCII)發現歌詞站浮水印與 cp1252 誤碼——同形字(西里爾 + е 160 處/78 檔、希臘 ό 6 處)、異體空格(U+2005 211 處、 + U+205F 80 處、U+200A 6 處)、零寬字元(U+200B 3 處)、C1 + 誤碼(U+0091/92/93/94/97 共 58 處;先前手修之 U+0085 同 + 類);受影響 81/883 檔全部來自 pilot 匯入,新抓取檔乾淨。 + 正規化規則四條(C1 依 cp1252 本義還原、同形字還原、異體 + 空格→ASCII 空格、零寬刪除)實作於 fetch-lyrics 的 + normalize_lyrics(),寫入時套用;既有語料以同一函式一次性 + 套用。倒放行等歌曲本體內容不動——策展界線:清傳輸雜質, + 不改內容。 diff --git a/tools/src/pop_fem_audit_tools/commands/fetch_lyrics.py b/tools/src/pop_fem_audit_tools/commands/fetch_lyrics.py index 7a607e9..a22b244 100644 --- a/tools/src/pop_fem_audit_tools/commands/fetch_lyrics.py +++ b/tools/src/pop_fem_audit_tools/commands/fetch_lyrics.py @@ -57,6 +57,49 @@ SLEEP_SECONDS: float = 1.0 """The delay between consecutive HTTP requests, in seconds.""" +def _build_normalization() -> dict[int, str | None]: + """Build the lyrics normalization translation table. + + :return: The codepoint-to-replacement mapping, a replacement + of None meaning removal. + """ + table: dict[int, str | None] = {} + codepoint: int + for codepoint in range(0x80, 0xa0): + try: + table[codepoint] = bytes([codepoint]).decode("cp1252") + except UnicodeDecodeError: + table[codepoint] = None + table[0x0435] = "e" + table[0x03cc] = "ó" + for codepoint in (0x2005, 0x205f, 0x200a): + table[codepoint] = " " + for codepoint in (0x200b, 0x200c, 0x200d, 0xfeff): + table[codepoint] = None + return table + + +NORMALIZATION: dict[int, str | None] = _build_normalization() +"""The codepoint-to-replacement mapping applied to fetched +lyrics: cp1252-mojibake restoration for U+0080-U+009F (with the +five byte values undefined in cp1252 removed), homoglyph +restoration for the Cyrillic "e" and the Greek "o" with tonos, +ASCII-space restoration for exotic space variants, and removal +of zero-width characters. A replacement of None removes the +codepoint.""" + + +def normalize_lyrics(text: str) -> str: + """Restore or remove watermark and mojibake characters. + + :param text: The lyrics text as fetched from an API. + :return: The text with the codepoints in + :data:`NORMALIZATION` replaced or removed; every other + character is unchanged. + """ + return text.translate(NORMALIZATION) + + def parse_args(argv: list[str] | None) -> argparse.Namespace: """Parse the command-line arguments. @@ -187,6 +230,9 @@ def save_lyrics(lyrics_dir: Path, song_id: int, The cache directory is created when missing. + The lyrics text is normalized with :func:`normalize_lyrics` + before being written. + :param lyrics_dir: The lyrics cache directory. :param song_id: The song ID. :param lyrics: The lyrics text. @@ -195,7 +241,7 @@ def save_lyrics(lyrics_dir: Path, song_id: int, """ lyrics_dir.mkdir(parents=True, exist_ok=True) (lyrics_dir / f"{song_id}.txt").write_text( - lyrics, encoding="utf-8") + normalize_lyrics(lyrics), encoding="utf-8") def append_provenance(path: Path, song_id: int, diff --git a/tools/tests/test_fetch_lyrics.py b/tools/tests/test_fetch_lyrics.py index 0744b94..daa5af4 100644 --- a/tools/tests/test_fetch_lyrics.py +++ b/tools/tests/test_fetch_lyrics.py @@ -317,6 +317,79 @@ class TestFetchLyrics(unittest.TestCase): self.assertEqual([x[0] for x in rows[1:]], ["1", "2", "2"]) + def test_normalize_cp1252_mojibake(self) -> None: + """Test that cp1252 mojibake codepoints are restored.""" + self.assertEqual( + fetch_lyrics.normalize_lyrics("wait…"), + "wait…") + self.assertEqual( + fetch_lyrics.normalize_lyrics( + "‘quote’"), + "‘quote’") + self.assertEqual( + fetch_lyrics.normalize_lyrics( + "“quote”"), + "“quote”") + self.assertEqual( + fetch_lyrics.normalize_lyrics("dash—line"), + "dash—line") + + def test_normalize_undefined_cp1252_removed(self) -> None: + """Test that undefined cp1252 byte values are removed.""" + text: str = ("abcdef") + self.assertEqual( + fetch_lyrics.normalize_lyrics(text), "abcdef") + + def test_normalize_homoglyphs(self) -> None: + """Test that watermark homoglyphs are restored.""" + self.assertEqual( + fetch_lyrics.normalize_lyrics("likе that"), + "like that") + self.assertEqual( + fetch_lyrics.normalize_lyrics("lό que soy"), + "ló que soy") + + def test_normalize_space_variants(self) -> None: + """Test that exotic space variants become ASCII space.""" + self.assertEqual( + fetch_lyrics.normalize_lyrics( + "a b c d"), + "a b c d") + + def test_normalize_zero_width_removed(self) -> None: + """Test that zero-width characters are removed.""" + text: str = ( + "a​b‌c‍de") + self.assertEqual( + fetch_lyrics.normalize_lyrics(text), "abcde") + + def test_normalize_ascii_unchanged(self) -> None: + """Test that plain ASCII text passes through unchanged.""" + text: str = "Hello, it's me\n" + self.assertEqual( + fetch_lyrics.normalize_lyrics(text), text) + + def test_normalize_legitimate_non_ascii_unchanged(self) -> None: + """Test that legitimate non-ASCII content is unchanged.""" + text: str = "¿cómo estás? 안녕하세요\n" + self.assertEqual( + fetch_lyrics.normalize_lyrics(text), text) + + def test_fetched_lyrics_saved_normalized(self) -> None: + """Test that a fetched lyric is normalized before saving.""" + self.__seed([("Hello", "Adele")]) + with mock.patch( + "urllib.request.urlopen", + side_effect=[self.__response( + {"lyrics": "wait… likе" + " that now​\n"})]): + status: int = self.__run_fetch()[0] + self.assertEqual(status, 0) + self.assertEqual( + (self.__lyrics / "1.txt") + .read_text(encoding="utf-8"), + "wait… like that now\n") + def test_no_store_fails(self) -> None: """Test that a missing working store fails the run.""" urlopen: mock.Mock