Remove the retired artist overrides layer and the mixed artist type

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-04 15:15:49 +08:00
co-authored by Claude Fable 5
parent 93aa06703b
commit 15f207ffc6
8 changed files with 24 additions and 70 deletions
+1 -2
View File
@@ -24,8 +24,7 @@ Hot 10020162025)為例的內容分析。
../data/source/yearend_hot100_2016_2025.csv
../data/derived
--lyrics-dir ../data/captures/lyrics
--wikidata-csv ../data/captures/artists-wikidata.csv
--overrides-csv ../data/manual/artists_overrides.csv`)。
--wikidata-csv ../data/captures/artists-wikidata.csv`)。
LLM 步驟使用 `claude-sonnet-4-6`、temperature=0、
thinking 關閉;每步驟獨立執行兩次後由仲裁步驟合併
(「2+1」協定)。
+15
View File
@@ -180,3 +180,18 @@
推測不確定且無佐證者列疑慮清單;查證屬資料策展而非
分析,以 Claude Code 輔助、不走分析 APIQS 批次以
私人工作檔留存,不隨論文發布。
## 2026-08-04
- **移除人工 overrides 層**`build-db` 刪去
`--overrides-csv` 選項與套用邏輯,文件同步移除
`data/manual/artists_overrides.csv`。理由:「先補完
Wikidata、再捕捉快照」工作流實跑後,469 位歌手全數
在上游查證補齊,快照即完整,本地覆蓋層已無存在事實;
`data/manual/` 層保留(供日後黃金標準編碼)。
- **歌手型態刪去 mixed 值**`ArtistType` 只留 sologroup。
mixed 是先導研究「男/女/混合團體」單一欄位的殘留,
正式設計拆成 gender+type 後從未定義其指涉;署名一律
拆成個人後,男女混合是歌曲層(song_artists+各歌手
gender)可推導的事實,不屬歌手實體。非人非團體者
Pinkfong)type 留空由人工判定,維持現狀。
+1 -1
View File
@@ -21,7 +21,7 @@ pop-fem-audit/
│ │ └── lyrics/ # 歌詞 .txt 快取
│ │ # gitignored,版權)
│ ├── manual/ # 人工著作:只由研究者手寫
│ │ └── artists_overrides.csv # 人工核定 / 深度背景
│ │ # (黃金標準編碼等)
│ └── derived/ # 衍生:只由 build-db 寫入
│ ├── songs.csv # 歌曲報表(人讀;進 git)
│ └── artists.csv # 歌手報表(人讀;進 git)
+2 -2
View File
@@ -33,7 +33,7 @@
——一律以文字格式 commit。格式跟著層次走,不跟著偏好走。
- **分層**
- 源頭:原始榜單 CSV(進 git)。
- 捕捉:Wikidata 快照 CSV、人工 overrides CSV、你的編碼
- 捕捉:Wikidata 快照 CSV、你的編碼
CSV、`runs/` JSONL(皆進 git);歌詞 `.txt` 快取
(版權因素 gitignored,為已知的稽核缺口)。
- 工作儲存:SQLite 單檔(`tools/instance/`generated、
@@ -79,7 +79,7 @@
| 階段 | 內容 | 方式 | 時程 |
|---|---|---|---|
| 0 | 基礎建設:git init、目錄結構、.gitignore、決策日誌、runner script(含 Batch API)、codebook v0 骨架 | script + 討論 | 7/307/31 |
| 1 | 資料準備:`run_llm` 改走統一設定 → `build-db`(解析榜單成 songs/chart_entries/artists/song_artists)→ `import-lyrics`pilot 20182025)→ `fetch-lyrics`201617 與缺漏,Lyrics.ovh / LRCLIB)→ `fetch-artists`Wikidata 快照 + 人工 overrides)→ `export-llm-input` | 子命令 | 7/318/3 |
| 1 | 資料準備:`run_llm` 改走統一設定 → `build-db`(解析榜單成 songs/chart_entries/artists/song_artists)→ `import-lyrics`pilot 20182025)→ `fetch-lyrics`201617 與缺漏,Lyrics.ovh / LRCLIB)→ `fetch-artists`Wikidata 快照)→ `export-llm-input` | 子命令 | 7/318/3 |
| 2 | 候選篩選:全部唯一歌曲高召回 women-power 候選篩選(寧可多抓,人工剔除) | API 2+1 | 8/28/3 |
| 3 | 黃金標準:依 codebook 人工逐首判定 genuine/peripheral/fake,附引用歌詞證據表(LLM 只做摘錄,不給判定建議);先以 10–15 首校準樣本試編並修訂 codebook 後凍結;同批校準樣本實測 Sonnet 4.6 vs Opus 5 一致率 | 人工 + script 輔助 | 8/48/8 |
| 4 | 受控比較(盲點實驗):條件 A(詞彙層提示)vs 條件 B(框架感知提示),各 2+1,對照黃金標準計算假陽/假陰率 | API | 8/68/9 |
+4 -11
View File
@@ -8,8 +8,8 @@ Rebuilds the working store from scratch out of the committed
inputs: the year-end chart CSV and the output directory for the
review CSV files, given as the two positional command-line
arguments, and the optional capture inputs, each given as an
option: the lyrics cache directory, the Wikidata artist snapshot
CSV, and the manual artist overrides CSV. An omitted option
option: the lyrics cache directory and the Wikidata artist
snapshot CSV. An omitted option
leaves its capture layer unloaded; a given option whose path does
not exist fails the build. Missing tables
are created on a fresh store; existing tables are never altered,
@@ -188,9 +188,6 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
parser.add_argument(
"--wikidata-csv", type=Path, default=None,
help="the Wikidata artist snapshot CSV file to apply")
parser.add_argument(
"--overrides-csv", type=Path, default=None,
help="the manual artist override CSV file to apply")
return parser.parse_args(argv)
@@ -215,8 +212,7 @@ def parse_artist_credit(credit: str) -> list[tuple[str, Role]]:
word or punctuation.
Known limitation: a compound act name that contains one of the
delimiters, other than the protected names, is over-split;
such cases are corrected later via the human override layer.
delimiters, other than the protected names, is over-split.
:param credit: The combined artist credit string.
:return: The (name, role) pairs in credit order, primary side
@@ -407,8 +403,7 @@ def apply_artist_csv(session: Session, path: Path) -> None:
"""Apply an artist attribute CSV onto the artist rows.
Artists match by exact name. Only the non-empty cells are
applied, so a later CSV overrides an earlier one field by
field. The note column is ignored.
applied, field by field. The note column is ignored.
:param session: The database session, with the artists
flushed.
@@ -727,8 +722,6 @@ def main(argv: list[str] | None = None) -> int:
load_lyrics(session, args.lyrics_dir)
if args.wikidata_csv is not None:
apply_artist_csv(session, args.wikidata_csv)
if args.overrides_csv is not None:
apply_artist_csv(session, args.overrides_csv)
session.flush()
violations: list[str] = find_violations(
session, YEARS, RANKS_PER_YEAR)
@@ -106,9 +106,6 @@ class ArtistType(enum.StrEnum):
"""A solo artist: a human."""
GROUP = "group"
"""A musical ensemble."""
MIXED = "mixed"
"""A mixed act, assigned manually via the overrides; never
derived by the fetcher."""
@dataclass
+1 -1
View File
@@ -78,7 +78,7 @@ class Artist(Base):
gender: Mapped[str | None]
"""The gender of the artist."""
type: Mapped[str | None]
"""The artist type: solo, group, or mixed."""
"""The artist type: solo or group."""
genre: Mapped[str | None]
"""The music genre of the artist."""
country: Mapped[str | None]
-50
View File
@@ -258,8 +258,6 @@ class TestBuildDB(unittest.TestCase):
self.__lyrics: Path = self.__dir / "lyrics"
self.__wikidata: Path = \
self.__dir / "artists_wikidata.csv"
self.__overrides: Path = \
self.__dir / "artists_overrides.csv"
self.__write_chart(self.CHART_CSV)
url: str = f"sqlite:///{self.__dir}/store.sqlite3"
config.set_settings(config.Settings(
@@ -573,42 +571,6 @@ class TestBuildDB(unittest.TestCase):
list(session.scalars(sa.select(ChartEntry))), [])
self.assertEqual(list(session.scalars(sa.select(Song))), [])
def test_overrides_apply_over_wikidata(self) -> None:
"""Test that the overrides win over the Wikidata snapshot."""
self.__wikidata.write_text(
"name,qid,gender,type,genre,country,note\n"
"Adele,Q2831,female,solo,pop,GB,\n",
encoding="utf-8")
self.__overrides.write_text(
"name,qid,gender,type,genre,country,note\n"
"Adele,,,,soul,,manually checked\n",
encoding="utf-8")
self.assertEqual(self.__run_build(
"--wikidata-csv", str(self.__wikidata),
"--overrides-csv", str(self.__overrides))[0], 0)
session: Session = self.__session()
artist: Artist | None = session.scalar(
sa.select(Artist).where(Artist.name == "Adele"))
assert artist is not None
self.assertEqual(artist.genre, "soul")
self.assertEqual(artist.gender, "female")
self.assertEqual(artist.wikidata_qid, "Q2831")
self.assertEqual(artist.country, "GB")
def test_unknown_override_name_fails(self) -> None:
"""Test that an unknown override name fails the build."""
self.__overrides.write_text(
"name,qid,gender,type,genre,country,note\n"
"Adel,,female,,,,typo\n", encoding="utf-8")
status: int
stderr: str
status, stderr = self.__run_build(
"--overrides-csv", str(self.__overrides))
self.assertNotEqual(status, 0)
self.assertIn("Adel", stderr)
session: Session = self.__session()
self.assertEqual(list(session.scalars(sa.select(Song))), [])
def test_lyrics_loaded(self) -> None:
"""Test loading the lyrics cache into the songs."""
self.__lyrics.mkdir()
@@ -676,18 +638,6 @@ class TestBuildDB(unittest.TestCase):
session: Session = self.__session()
self.assertEqual(list(session.scalars(sa.select(Song))), [])
def test_missing_overrides_csv_fails(self) -> None:
"""Test that a given but missing override CSV fails."""
status: int
stderr: str
status, stderr = self.__run_build(
"--overrides-csv", str(self.__overrides))
self.assertNotEqual(status, 0)
self.assertIn("error:", stderr)
self.assertIn(str(self.__overrides), stderr)
session: Session = self.__session()
self.assertEqual(list(session.scalars(sa.select(Song))), [])
REVIEW_CHART_CSV: str = (
"year,rank,title,artist\n"
"2016,1,banana,Artist B\n"