From 81975b72e43711a5553e807801931dadf004b10d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BE=9D=E7=91=AA=E8=B2=93?= Date: Tue, 4 Aug 2026 13:05:11 +0800 Subject: [PATCH] Remove the missing-lyrics report from fetch-lyrics Co-Authored-By: Claude Fable 5 --- docs/decision_log.md | 6 ++ docs/project_structure.md | 1 - tools/src/pop_fem_audit_tools/fetch_lyrics.py | 66 ++----------------- tools/tests/test_fetch_lyrics.py | 14 +--- 4 files changed, 13 insertions(+), 74 deletions(-) diff --git a/docs/decision_log.md b/docs/decision_log.md index b74ae9c..5ed68ab 100644 --- a/docs/decision_log.md +++ b/docs/decision_log.md @@ -201,3 +201,9 @@ 拆成個人後，男女混合是歌曲層（song_artists＋各歌手 gender）可推導的事實，不屬歌手實體。非人非團體者 （Pinkfong）type 留空由人工判定，維持現狀。 +- **fetch-lyrics 移除缺漏報表**：刪去 `missing_csv` 位置引數 + 與缺漏報表 CSV 輸出。理由：該報表只寫不讀（指令從未讀取 + 它），缺漏數字每次執行皆由工作儲存比對歌詞目錄重新算出； + worklist 角色由重跑冪等指令本身承擔；完整性已由 provenance + CSV 與 build-db 統計數字佐證；該檔案不承載任何無法重新 + 產生的資訊。 diff --git a/docs/project_structure.md b/docs/project_structure.md index df0a8e5..ea3b88a 100644 --- a/docs/project_structure.md +++ b/docs/project_structure.md @@ -17,7 +17,6 @@ pop-fem-audit/ │ │ │ # 私人匯入腳本寫入 │ │ ├── artists-wikidata.csv # Wikidata 快照 │ │ ├── lyrics_provenance.csv # 歌詞出處 -│ │ ├── lyrics_missing.csv # 歌詞缺漏報表 │ │ └── lyrics/ # 歌詞 .txt 快取 │ │ # （gitignored，版權） │ ├── manual/ # 人工著作：只由研究者手寫 diff --git a/tools/src/pop_fem_audit_tools/fetch_lyrics.py b/tools/src/pop_fem_audit_tools/fetch_lyrics.py index 93ea6bf..e8a092e 100644 --- a/tools/src/pop_fem_audit_tools/fetch_lyrics.py +++ b/tools/src/pop_fem_audit_tools/fetch_lyrics.py @@ -18,10 +18,8 @@ artist name, the same APIs are queried again with the artist credit, to catch songs cataloged only under a joint credit such as "Dan + Shay". -A song that every API misses on both queries is reported in the -missing lyrics CSV, also given as a positional command-line -argument, which is rewritten on every run to reflect the -current status. Misses are expected and do not fail the run. +A song that every API misses on both queries is reported on the +standard error, but does not fail the run; misses are expected. """ import argparse import csv @@ -32,7 +30,6 @@ import time import urllib.parse import urllib.request from collections.abc import Sequence -from dataclasses import dataclass from pathlib import Path from typing import Any @@ -50,9 +47,6 @@ from .models import ( PROVENANCE_FIELDS: Sequence[str] = ( "song_id", "source", "method", "acquired_at", "note") """The header columns of the lyrics provenance CSV file.""" -MISSING_FIELDS: Sequence[str] = ( - "song_id", "title", "artist_credit", "reason") -"""The header columns of the missing lyrics report CSV file.""" USER_AGENT: str = ("pop-fem-audit-tools" " (https://github.com/imacat/pop-fem-audit)") """The User-Agent header sent on every HTTP request.""" @@ -62,28 +56,6 @@ SLEEP_SECONDS: float = 1.0 """The delay between consecutive HTTP requests, in seconds.""" -@dataclass -class MissingLyrics: - """One row of the missing lyrics report CSV file.""" - - song_id: int - """The song ID.""" - title: str - """The song title.""" - artist_credit: str - """The artist credit of the song.""" - reason: str - """The reason the lyrics are missing.""" - - def to_row(self) -> list[str]: - """Return this report entry as a CSV row. - - :return: The row values, in the column order. - """ - return [str(self.song_id), self.title, - self.artist_credit, self.reason] - - def parse_args(argv: list[str] | None) -> argparse.Namespace: """Parse the command-line arguments. @@ -100,9 +72,6 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: parser.add_argument( "provenance_csv", type=Path, help="the lyrics provenance CSV file") - parser.add_argument( - "missing_csv", type=Path, - help="the missing lyrics report CSV file") return parser.parse_args(argv) @@ -251,27 +220,6 @@ def append_provenance(path: Path, song_id: int, datetime.date.today().isoformat(), ""]) -def write_missing(path: Path, - misses: Sequence[MissingLyrics]) -> None: - """Rewrite the missing lyrics report CSV file. - - The previous content is replaced, so the file reflects the - current misses only. - - :param path: The missing lyrics report CSV file. - :param misses: The report entries of the songs still without - lyrics. - :return: None. - :raises OSError: When the file cannot be written. - """ - path.parent.mkdir(parents=True, exist_ok=True) - with open(path, "w", encoding="utf-8", - newline="") as file: - writer: Any = csv.writer(file) - writer.writerow(MISSING_FIELDS) - writer.writerows(x.to_row() for x in misses) - - def main(argv: list[str] | None = None) -> int: """Fetch the missing song lyrics from the public APIs. @@ -283,7 +231,7 @@ def main(argv: list[str] | None = None) -> int: args: argparse.Namespace = parse_args(argv) fetcher: LyricsFetcher = LyricsFetcher() fetched: int = 0 - misses: list[MissingLyrics] = [] + missed: int = 0 session: Session = ds.get_db() try: song: Song @@ -298,10 +246,7 @@ def main(argv: list[str] | None = None) -> int: result = fetcher.fetch( song.artist_credit, song.title) if result is None: - misses.append(MissingLyrics( - song_id=song.id, title=song.title, - artist_credit=song.artist_credit, - reason="not found")) + missed += 1 print(f"song {song.id} \"{song.title}\": miss", file=sys.stderr) continue @@ -314,12 +259,11 @@ def main(argv: list[str] | None = None) -> int: fetched += 1 print(f"song {song.id} \"{song.title}\": {source}", file=sys.stderr) - write_missing(args.missing_csv, misses) except (OSError, sa.exc.SQLAlchemyError) as error: print(f"error: {error}", file=sys.stderr) return 1 finally: session.close() - print(f"done: {fetched} fetched, {len(misses)} missed", + print(f"done: {fetched} fetched, {missed} missed", file=sys.stderr) return 0 diff --git a/tools/tests/test_fetch_lyrics.py b/tools/tests/test_fetch_lyrics.py index 430cc86..582bed7 100644 --- a/tools/tests/test_fetch_lyrics.py +++ b/tools/tests/test_fetch_lyrics.py @@ -32,9 +32,6 @@ class TestFetchLyrics(unittest.TestCase): PROVENANCE_HEADER: list[str] = [ "song_id", "source", "method", "acquired_at", "note"] """The expected header row of the provenance CSV file.""" - MISSING_HEADER: list[str] = [ - "song_id", "title", "artist_credit", "reason"] - """The expected header row of the missing report CSV file.""" def setUp(self) -> None: """Create a temporary capture directory with the store.""" @@ -45,7 +42,6 @@ class TestFetchLyrics(unittest.TestCase): self.__lyrics: Path = self.__dir / "lyrics" self.__provenance: Path = \ self.__dir / "lyrics_provenance.csv" - self.__missing: Path = self.__dir / "lyrics_missing.csv" url: str = f"sqlite:///{self.__dir}/store.sqlite3" config.set_settings(config.Settings( SQLALCHEMY_DATABASE_URL=url, @@ -133,8 +129,7 @@ class TestFetchLyrics(unittest.TestCase): stderr: io.StringIO = io.StringIO() with redirect_stderr(stderr): status: int = fetch_lyrics.main( - [str(self.__lyrics), str(self.__provenance), - str(self.__missing)]) + [str(self.__lyrics), str(self.__provenance)]) return status, stderr.getvalue() @staticmethod @@ -251,10 +246,7 @@ class TestFetchLyrics(unittest.TestCase): self.assertEqual(status, 0) self.assertFalse((self.__lyrics / "1.txt").exists()) self.assertFalse(self.__provenance.exists()) - rows: list[list[str]] = self.__read_rows(self.__missing) - self.assertEqual(len(rows), 2) - self.assertEqual(rows[0], self.MISSING_HEADER) - self.assertEqual(rows[1][:3], ["1", "Hello", "Adele"]) + self.assertIn("song 1 \"Hello\": miss", stderr) self.assertIn("0 fetched, 1 missed", stderr) def test_cached_song_skipped(self) -> None: @@ -272,8 +264,6 @@ class TestFetchLyrics(unittest.TestCase): (self.__lyrics / "1.txt") .read_text(encoding="utf-8"), "cached\n") - rows: list[list[str]] = self.__read_rows(self.__missing) - self.assertEqual(rows, [self.MISSING_HEADER]) def test_url_encoding(self) -> None: """Test the percent-encoding of the artist and title."""