diff --git a/data/derived/songs.csv b/data/derived/songs.csv index 3cd4b16..85860b4 100644 --- a/data/derived/songs.csv +++ b/data/derived/songs.csv @@ -67,7 +67,7 @@ Bam Bam,Camila Cabello Featuring Ed Sheeran,2022#66,mixed Bandit,Juice WRLD & YoungBoy Never Broke Again,2020#63,male Bang!,AJR,2021#56,male Bank Account,21 Savage,2017#48,male -Barbie World,Nicki Minaj & Ice Spice With Aqua,2023#46,mixed +Barbie World,Nicki Minaj & Ice Spice With Aqua,2023#46,female Bartier Cardi,Cardi B Featuring 21 Savage,2018#61,mixed Be Alright,Dean Lewis,2019#54,male Be Careful,Cardi B,2018#59,female @@ -122,7 +122,7 @@ Bury A Friend,Billie Eilish,2019#73,female Bury Me In Georgia,Kane Brown,2023#89,male Butter,BTS,2021#11,male Buy Dirt,Jordan Davis Featuring Luke Bryan,2022#49,male -"Bzrp Music Sessions, Vol. 53",Bizarrap & Shakira,2023#96,mixed +"Bzrp Music Sessions, Vol. 53",Bizarrap & Shakira,2023#96,female Cake By The Ocean,DNCE,2016#18,mixed Call Out My Name,The Weeknd,2018#78,male Calling My Phone,Lil Tjay Featuring 6LACK,2021#33,male @@ -251,7 +251,7 @@ Forever After All,Luke Combs,2021#18,male Fortnight,Taylor Swift Featuring Post Malone,2024#22,mixed Freaky Friday,Lil Dicky Featuring Chris Brown,2018#55,male Free,Rumi & JINU: EJAE & Andrew Choi,2025#92,mixed -Friends,Marshmello & Anne-Marie,2018#26,mixed +Friends,Marshmello & Anne-Marie,2018#26,female FTCU,Nicki Minaj,2024#88,female Fukumean,Gunna,2023#31/2024#85,male Gata Only,FloyyMenor X Cris Mj,2024#52,male diff --git a/data/manual/performer-gender-corrections.csv b/data/manual/performer-gender-corrections.csv new file mode 100644 index 0000000..424c3b0 --- /dev/null +++ b/data/manual/performer-gender-corrections.csv @@ -0,0 +1,4 @@ +Title,Artist Credit,Performer Gender,Note +Barbie World,Nicki Minaj & Ice Spice With Aqua,female,Aqua credited for the sample; no male vocal in the lyrics +"Bzrp Music Sessions, Vol. 53",Bizarrap & Shakira,female,Bizarrap produces; no vocal part +Friends,Marshmello & Anne-Marie,female,Marshmello produces; no vocal part diff --git a/docs/decision-log.md b/docs/decision-log.md index 76d81d4..d779973 100644 --- a/docs/decision-log.md +++ b/docs/decision-log.md @@ -817,3 +817,23 @@ 獨立;run1 之覆蓋=`run1`(144 首)∪`run1-rescue-444` (1 首),步驟 5-2 輸入組裝時 song-444 之 run1 份取自 補送歸檔,組裝腳本並驗證三輪歌曲集合一致(145 首)。 + +- **performer_gender 語意定為「演唱聲音的性別」,新增 + 手工修正機制**:原值由 `build-db` 自署名藝人的 + Wikidata 性別推導,將僅掛名不開口者(製作人、取樣 + 來源)也計入,致三首純女聲歌曲被判為 mixed: + 〈Friends〉(Marshmello 僅製作)、〈Barbie World〉 + (Aqua 為取樣掛名,Ken 聲部不在歌詞中)、〈Bzrp Music + Sessions, Vol. 53〉(Bizarrap 僅製作)。裁定:不另設 + 發話位置變項(單獨為男聲設變項說不通,女聲、非二元 + 聲部亦無對應變項),而是修正 performer_gender 本身—— + 其語意即演唱聲音的性別。機制:`build-db` 新增 + `--gender-corrections` 選項,於推導後套用 + `data/manual/performer-gender-corrections.csv`(研究者 + 手工編定,Title+Artist Credit 精確比對,查無即建置 + 失敗),鏡射 coding-corrections 前例;不直接改推導 + 規則,因署名層推導對其餘 880 首仍正確。下游數字隨 + 重建更新:wp 66 首成 female 50/mixed 12/male 3/ + genderfluid 1;fe 144 首成 female 98/mixed 30/male 13/ + genderfluid 2/non-binary 1;wp∪fe 145 首中署名含男性 + 且開口者 44 首。步驟 5 報告中三首之性別標示同步更正。 diff --git a/tools/src/pop_fem_audit_tools/commands/build_db.py b/tools/src/pop_fem_audit_tools/commands/build_db.py index 8c5522d..a5ebaf0 100644 --- a/tools/src/pop_fem_audit_tools/commands/build_db.py +++ b/tools/src/pop_fem_audit_tools/commands/build_db.py @@ -9,7 +9,8 @@ inputs: the year-end chart CSV and the output directory for the review CSV files, given as the two positional command-line arguments, and the optional inputs, each given as an option: the lyrics cache directory, the Wikidata artist -snapshot CSV, and the settled coding table CSV. An omitted option +snapshot CSV, the settled coding table CSV, and the gender +correction table CSV. An omitted option leaves its layer unloaded; a given option whose path does not exist fails the build. Missing tables are created on a fresh store; existing tables are never altered, @@ -48,6 +49,12 @@ producer collective -- is not a performing act: it has no voice, so its gender is inapplicable, and it takes no part in the derivation. See `PerformerGenderDeriver.performer_gender`. +Once the performer genders are derived, an optional gender +correction table CSV overrides the performer gender of the songs +it names, matched by exact title and exact artist credit, with +its performer gender column stored verbatim; see +`GenderCorrectionImporter`. + On a successful build, two review CSV files, ``songs.csv`` and ``artists.csv``, are (re)written under the given output directory, mirroring the stored songs and artists without their IDs; see @@ -122,6 +129,9 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: parser.add_argument( "--groups", type=Path, default=None, help="the settled code group table CSV file to import") + parser.add_argument( + "--gender-corrections", type=Path, default=None, + help="the gender correction table CSV file to apply") return parser.parse_args(argv) @@ -687,6 +697,98 @@ class PerformerGenderDeriver: return None +class GenderCorrectionImporter: + """The gender-correction job: overrides the derived performer + gender of the stored songs it names.""" + + COLUMNS: tuple[str, ...] = ( + "Title", "Artist Credit", "Performer Gender", "Note") + """The required columns of the gender correction CSV file.""" + + def __init__(self, session: Session) -> None: + """Initialize the importer. + + :param session: The database session. + """ + self.__session: Session = session + + def import_gender_corrections(self, path: Path | None) -> None: + """Apply the gender correction table onto the stored songs. + + A None input leaves the derived performer genders + untouched. Otherwise every row of the CSV file matches one + stored song by its exact title and exact artist credit and + sets ``Song.performer_gender`` to the row's performer + gender column verbatim; the note column is ignored. Apply + this after `PerformerGenderDeriver.derive_performer_genders` + has run, so a correction overrides the derived value. When + the method returns, the applied corrections are queryable + in the session. + + :param path: The gender correction table CSV file to apply, + or None to skip the corrections. + :return: None. + :raises BuildError: When the file lacks a required column, + or a row names a song that the store does not have. + :raises OSError: When the file cannot be read. + """ + if path is None: + return + songs: dict[tuple[str, str], Song] = { + (x.title, x.artist_credit): x + for x in self.__session.scalars(sa.select(Song))} + with open(path, encoding="utf-8", newline="") as file: + reader: csv.DictReader[str] = csv.DictReader(file) + self.__check_columns(path, reader.fieldnames) + row: dict[str, str] + for row in reader: + self.__apply_correction(path, songs, row) + self.__session.flush() + + @staticmethod + def __apply_correction(path: Path, + songs: dict[tuple[str, str], Song], + row: dict[str, str]) -> None: + """Apply one gender correction row. + + :param path: The gender correction CSV file, for the error + message. + :param songs: The stored songs, keyed by the title and the + artist credit. + :param row: The gender correction CSV row. + :return: None. + :raises BuildError: When the row names a song that the + store does not have. + """ + key: tuple[str, str] = ( + row["Title"], row["Artist Credit"]) + song: Song | None = songs.get(key) + if song is None: + raise BuildError( + f"{path}: no song \"{row['Title']}\" by" + f" \"{row['Artist Credit']}\"") + song.performer_gender = row["Performer Gender"] + + @classmethod + def __check_columns(cls, path: Path, + fieldnames: Sequence[str] | None) -> None: + """Verify the gender correction CSV file has the required + columns. + + :param path: The gender correction CSV file. + :param fieldnames: The header row of the file, or None + when the file is empty. + :return: None. + :raises BuildError: When a required column is absent. + """ + header: Sequence[str] = fieldnames or () + missing: list[str] = [ + x for x in cls.COLUMNS if x not in header] + if len(missing) > 0: + raise BuildError( + f"{path}: missing column(s): {', '.join(missing)}") + + class CodingImporter: """The coding-import job: loads the settled coding table onto the stored songs.""" @@ -1091,6 +1193,8 @@ def main(argv: list[str] | None = None) -> int: CaptureImporter(session).import_captures( args.lyrics_dir, args.wikidata_csv) PerformerGenderDeriver(session).derive_performer_genders() + GenderCorrectionImporter(session).import_gender_corrections( + args.gender_corrections) CodingImporter(session).import_codings(args.codings) GroupImporter(session).import_groups(args.groups) counts = StoreCounts.get_instance(session) diff --git a/tools/tests/test_build_db.py b/tools/tests/test_build_db.py index 5134ce4..9e9deec 100644 --- a/tools/tests/test_build_db.py +++ b/tools/tests/test_build_db.py @@ -270,6 +270,8 @@ class TestBuildDB(unittest.TestCase): self.__dir / "artists_wikidata.csv" self.__codings: Path = self.__dir / "codings.csv" self.__groups: Path = self.__dir / "groups.csv" + self.__gender_corrections: Path = \ + self.__dir / "gender_corrections.csv" self.__write_chart(self.CHART_CSV) config.set_settings(config.Settings( SQLALCHEMY_DATABASE_URL="sqlite://", @@ -1222,3 +1224,127 @@ class TestBuildDB(unittest.TestCase): "--groups", str(self.__groups)) self.assertNotEqual(status, 0) self.assertIn("missing column(s): Votes", stderr) + + def __write_gender_corrections(self, content: str) -> None: + """Write the gender correction CSV fixture. + + :param content: The CSV content. + :return: None. + """ + self.__gender_corrections.write_text( + content, encoding="utf-8") + + def test_gender_correction_overrides_derived_gender( + self) -> None: + """Test that a correction row overrides an already-derived + performer gender, stored verbatim.""" + self.__write_chart(self.GENDER_CHART_CSV) + self.__write_wikidata(self.GENDER_WIKIDATA_CSV) + self.__write_gender_corrections( + "Title,Artist Credit,Performer Gender,Note\n" + "Female Song,Adele & Taylor Swift,mixed,reviewed\n") + status: int + stderr: str + status, stderr = self.__run_build( + "--wikidata-csv", str(self.__wikidata), + "--gender-corrections", str(self.__gender_corrections)) + self.assertEqual(status, 0) + self.assertEqual( + self.__performer_genders()["Female Song"], "mixed") + + def test_gender_correction_sets_undetermined_gender( + self) -> None: + """Test that a correction row sets the performer gender of + a song the derivation left undetermined.""" + self.__write_chart(self.GENDER_CHART_CSV) + self.__write_wikidata(self.GENDER_WIKIDATA_CSV) + self.__write_gender_corrections( + "Title,Artist Credit,Performer Gender,Note\n" + "Unknown Song,Adele featuring Nobody,female," + "reviewed\n") + status: int + stderr: str + status, stderr = self.__run_build( + "--wikidata-csv", str(self.__wikidata), + "--gender-corrections", str(self.__gender_corrections)) + self.assertEqual(status, 0) + self.assertEqual( + self.__performer_genders()["Unknown Song"], "female") + + def test_gender_correction_in_songs_csv(self) -> None: + """Test that songs.csv mirrors the corrected performer + gender.""" + self.__write_chart(self.GENDER_CHART_CSV) + self.__write_wikidata(self.GENDER_WIKIDATA_CSV) + self.__write_gender_corrections( + "Title,Artist Credit,Performer Gender,Note\n" + "Female Song,Adele & Taylor Swift,mixed,reviewed\n") + self.assertEqual( + self.__run_build( + "--wikidata-csv", str(self.__wikidata), + "--gender-corrections", + str(self.__gender_corrections))[0], 0) + rows: list[list[str]] = self.__read_csv_rows("songs.csv") + self.assertIn(("Female Song", "mixed"), + {(row[0], row[3]) for row in rows}) + + def test_omitted_gender_corrections_leaves_derived_gender( + self) -> None: + """Test that an omitted correction option leaves the + derived performer genders untouched.""" + self.__write_chart(self.GENDER_CHART_CSV) + self.__write_wikidata(self.GENDER_WIKIDATA_CSV) + self.__write_gender_corrections( + "Title,Artist Credit,Performer Gender,Note\n" + "Female Song,Adele & Taylor Swift,mixed,reviewed\n") + status: int + stderr: str + status, stderr = self.__run_build( + "--wikidata-csv", str(self.__wikidata)) + self.assertEqual(status, 0) + self.assertEqual( + self.__performer_genders()["Female Song"], "female") + + def test_gender_correction_unknown_song_fails(self) -> None: + """Test that a correction row naming an unknown song fails + the build, naming the file and the offending title and + credit.""" + self.__write_chart(self.GENDER_CHART_CSV) + self.__write_wikidata(self.GENDER_WIKIDATA_CSV) + self.__write_gender_corrections( + "Title,Artist Credit,Performer Gender,Note\n" + "Nowhere,Nobody,female,reviewed\n") + status: int + stderr: str + status, stderr = self.__run_build( + "--wikidata-csv", str(self.__wikidata), + "--gender-corrections", str(self.__gender_corrections)) + self.assertNotEqual(status, 0) + self.assertIn(str(self.__gender_corrections), stderr) + self.assertIn("no song \"Nowhere\" by \"Nobody\"", stderr) + + def test_gender_corrections_missing_column_fails(self) -> None: + """Test that a gender correction CSV missing a required + column fails the build.""" + self.__write_gender_corrections( + "Title,Artist Credit,Performer Gender\n" + "Hello,Adele,female\n") + status: int + stderr: str + status, stderr = self.__run_build( + "--gender-corrections", str(self.__gender_corrections)) + self.assertNotEqual(status, 0) + self.assertIn("missing column(s): Note", stderr) + + def test_missing_gender_corrections_csv_fails(self) -> None: + """Test that a given but missing gender correction CSV + fails the build.""" + status: int + stderr: str + status, stderr = self.__run_build( + "--gender-corrections", str(self.__gender_corrections)) + self.assertNotEqual(status, 0) + self.assertIn("error:", stderr) + self.assertIn(str(self.__gender_corrections), stderr) + session: Session = self.__session() + self.assertEqual(list(session.scalars(sa.select(Song))), [])