Correct three performer genders via build-db --gender-corrections

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

data/derived/songs.csv

@@ -67,7 +67,7 @@ Bam Bam,Camila Cabello Featuring Ed Sheeran,2022#66,mixed
 Bandit,Juice WRLD & YoungBoy Never Broke Again,2020#63,male
 Bang!,AJR,2021#56,male
 Bank Account,21 Savage,2017#48,male
-Barbie World,Nicki Minaj & Ice Spice With Aqua,2023#46,mixed
+Barbie World,Nicki Minaj & Ice Spice With Aqua,2023#46,female
 Bartier Cardi,Cardi B Featuring 21 Savage,2018#61,mixed
 Be Alright,Dean Lewis,2019#54,male
 Be Careful,Cardi B,2018#59,female
@@ -122,7 +122,7 @@ Bury A Friend,Billie Eilish,2019#73,female
 Bury Me In Georgia,Kane Brown,2023#89,male
 Butter,BTS,2021#11,male
 Buy Dirt,Jordan Davis Featuring Luke Bryan,2022#49,male
-"Bzrp Music Sessions, Vol. 53",Bizarrap & Shakira,2023#96,mixed
+"Bzrp Music Sessions, Vol. 53",Bizarrap & Shakira,2023#96,female
 Cake By The Ocean,DNCE,2016#18,mixed
 Call Out My Name,The Weeknd,2018#78,male
 Calling My Phone,Lil Tjay Featuring 6LACK,2021#33,male
@@ -251,7 +251,7 @@ Forever After All,Luke Combs,2021#18,male
 Fortnight,Taylor Swift Featuring Post Malone,2024#22,mixed
 Freaky Friday,Lil Dicky Featuring Chris Brown,2018#55,male
 Free,Rumi & JINU: EJAE & Andrew Choi,2025#92,mixed
-Friends,Marshmello & Anne-Marie,2018#26,mixed
+Friends,Marshmello & Anne-Marie,2018#26,female
 FTCU,Nicki Minaj,2024#88,female
 Fukumean,Gunna,2023#31/2024#85,male
 Gata Only,FloyyMenor X Cris Mj,2024#52,male

data/manual/performer-gender-corrections.csv

@@ -0,0 +1,4 @@
+Title,Artist Credit,Performer Gender,Note
+Barbie World,Nicki Minaj & Ice Spice With Aqua,female,Aqua credited for the sample; no male vocal in the lyrics
+"Bzrp Music Sessions, Vol. 53",Bizarrap & Shakira,female,Bizarrap produces; no vocal part
+Friends,Marshmello & Anne-Marie,female,Marshmello produces; no vocal part

docs/decision-log.md

@@ -817,3 +817,23 @@
   獨立;run1 之覆蓋=`run1`(144 首)∪`run1-rescue-444`
   (1 首),步驟 5-2 輸入組裝時 song-444 之 run1 份取自
   補送歸檔,組裝腳本並驗證三輪歌曲集合一致(145 首)。
+
+- **performer_gender 語意定為「演唱聲音的性別」,新增
+  手工修正機制**:原值由 `build-db` 自署名藝人的
+  Wikidata 性別推導,將僅掛名不開口者(製作人、取樣
+  來源)也計入,致三首純女聲歌曲被判為 mixed:
+  〈Friends〉(Marshmello 僅製作)、〈Barbie World〉
+  (Aqua 為取樣掛名,Ken 聲部不在歌詞中)、〈Bzrp Music
+  Sessions, Vol. 53〉(Bizarrap 僅製作)。裁定:不另設
+  發話位置變項(單獨為男聲設變項說不通,女聲、非二元
+  聲部亦無對應變項),而是修正 performer_gender 本身——
+  其語意即演唱聲音的性別。機制:`build-db` 新增
+  `--gender-corrections` 選項,於推導後套用
+  `data/manual/performer-gender-corrections.csv`(研究者
+  手工編定,Title+Artist Credit 精確比對,查無即建置
+  失敗),鏡射 coding-corrections 前例;不直接改推導
+  規則,因署名層推導對其餘 880 首仍正確。下游數字隨
+  重建更新:wp 66 首成 female 50/mixed 12/male 3/
+  genderfluid 1;fe 144 首成 female 98/mixed 30/male 13/
+  genderfluid 2/non-binary 1;wp∪fe 145 首中署名含男性
+  且開口者 44 首。步驟 5 報告中三首之性別標示同步更正。

tools/src/pop_fem_audit_tools/commands/build_db.py

@@ -9,7 +9,8 @@ inputs: the year-end chart CSV and the output directory for the
 review CSV files, given as the two positional command-line
 arguments, and the optional inputs, each given as an
 option: the lyrics cache directory, the Wikidata artist
-snapshot CSV, and the settled coding table CSV.  An omitted option
+snapshot CSV, the settled coding table CSV, and the gender
+correction table CSV.  An omitted option
 leaves its layer unloaded; a given option whose path does
 not exist fails the build.  Missing tables
 are created on a fresh store; existing tables are never altered,
@@ -48,6 +49,12 @@ producer collective -- is not a performing act: it has no voice,
 so its gender is inapplicable, and it takes no part in the
 derivation.  See `PerformerGenderDeriver.performer_gender`.
 
+Once the performer genders are derived, an optional gender
+correction table CSV overrides the performer gender of the songs
+it names, matched by exact title and exact artist credit, with
+its performer gender column stored verbatim; see
+`GenderCorrectionImporter`.
+
 On a successful build, two review CSV files, ``songs.csv`` and
 ``artists.csv``, are (re)written under the given output directory,
 mirroring the stored songs and artists without their IDs; see
@@ -122,6 +129,9 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
     parser.add_argument(
         "--groups", type=Path, default=None,
         help="the settled code group table CSV file to import")
+    parser.add_argument(
+        "--gender-corrections", type=Path, default=None,
+        help="the gender correction table CSV file to apply")
     return parser.parse_args(argv)
 
 
@@ -687,6 +697,98 @@ class PerformerGenderDeriver:
         return None
 
 
+class GenderCorrectionImporter:
+    """The gender-correction job: overrides the derived performer
+    gender of the stored songs it names."""
+
+    COLUMNS: tuple[str, ...] = (
+        "Title", "Artist Credit", "Performer Gender", "Note")
+    """The required columns of the gender correction CSV file."""
+
+    def __init__(self, session: Session) -> None:
+        """Initialize the importer.
+
+        :param session: The database session.
+        """
+        self.__session: Session = session
+
+    def import_gender_corrections(self, path: Path | None) -> None:
+        """Apply the gender correction table onto the stored songs.
+
+        A None input leaves the derived performer genders
+        untouched.  Otherwise every row of the CSV file matches one
+        stored song by its exact title and exact artist credit and
+        sets ``Song.performer_gender`` to the row's performer
+        gender column verbatim; the note column is ignored.  Apply
+        this after `PerformerGenderDeriver.derive_performer_genders`
+        has run, so a correction overrides the derived value.  When
+        the method returns, the applied corrections are queryable
+        in the session.
+
+        :param path: The gender correction table CSV file to apply,
+            or None to skip the corrections.
+        :return: None.
+        :raises BuildError: When the file lacks a required column,
+            or a row names a song that the store does not have.
+        :raises OSError: When the file cannot be read.
+        """
+        if path is None:
+            return
+        songs: dict[tuple[str, str], Song] = {
+            (x.title, x.artist_credit): x
+            for x in self.__session.scalars(sa.select(Song))}
+        with open(path, encoding="utf-8", newline="") as file:
+            reader: csv.DictReader[str] = csv.DictReader(file)
+            self.__check_columns(path, reader.fieldnames)
+            row: dict[str, str]
+            for row in reader:
+                self.__apply_correction(path, songs, row)
+        self.__session.flush()
+
+    @staticmethod
+    def __apply_correction(path: Path,
+                           songs: dict[tuple[str, str], Song],
+                           row: dict[str, str]) -> None:
+        """Apply one gender correction row.
+
+        :param path: The gender correction CSV file, for the error
+            message.
+        :param songs: The stored songs, keyed by the title and the
+            artist credit.
+        :param row: The gender correction CSV row.
+        :return: None.
+        :raises BuildError: When the row names a song that the
+            store does not have.
+        """
+        key: tuple[str, str] = (
+            row["Title"], row["Artist Credit"])
+        song: Song | None = songs.get(key)
+        if song is None:
+            raise BuildError(
+                f"{path}: no song \"{row['Title']}\" by"
+                f" \"{row['Artist Credit']}\"")
+        song.performer_gender = row["Performer Gender"]
+
+    @classmethod
+    def __check_columns(cls, path: Path,
+                        fieldnames: Sequence[str] | None) -> None:
+        """Verify the gender correction CSV file has the required
+        columns.
+
+        :param path: The gender correction CSV file.
+        :param fieldnames: The header row of the file, or None
+            when the file is empty.
+        :return: None.
+        :raises BuildError: When a required column is absent.
+        """
+        header: Sequence[str] = fieldnames or ()
+        missing: list[str] = [
+            x for x in cls.COLUMNS if x not in header]
+        if len(missing) > 0:
+            raise BuildError(
+                f"{path}: missing column(s): {', '.join(missing)}")
+
+
 class CodingImporter:
     """The coding-import job: loads the settled coding table onto
     the stored songs."""
@@ -1091,6 +1193,8 @@ def main(argv: list[str] | None = None) -> int:
         CaptureImporter(session).import_captures(
             args.lyrics_dir, args.wikidata_csv)
         PerformerGenderDeriver(session).derive_performer_genders()
+        GenderCorrectionImporter(session).import_gender_corrections(
+            args.gender_corrections)
         CodingImporter(session).import_codings(args.codings)
         GroupImporter(session).import_groups(args.groups)
         counts = StoreCounts.get_instance(session)

tools/tests/test_build_db.py

@@ -270,6 +270,8 @@ class TestBuildDB(unittest.TestCase):
             self.__dir / "artists_wikidata.csv"
         self.__codings: Path = self.__dir / "codings.csv"
         self.__groups: Path = self.__dir / "groups.csv"
+        self.__gender_corrections: Path = \
+            self.__dir / "gender_corrections.csv"
         self.__write_chart(self.CHART_CSV)
         config.set_settings(config.Settings(
             SQLALCHEMY_DATABASE_URL="sqlite://",
@@ -1222,3 +1224,127 @@ class TestBuildDB(unittest.TestCase):
             "--groups", str(self.__groups))
         self.assertNotEqual(status, 0)
         self.assertIn("missing column(s): Votes", stderr)
+
+    def __write_gender_corrections(self, content: str) -> None:
+        """Write the gender correction CSV fixture.
+
+        :param content: The CSV content.
+        :return: None.
+        """
+        self.__gender_corrections.write_text(
+            content, encoding="utf-8")
+
+    def test_gender_correction_overrides_derived_gender(
+            self) -> None:
+        """Test that a correction row overrides an already-derived
+        performer gender, stored verbatim."""
+        self.__write_chart(self.GENDER_CHART_CSV)
+        self.__write_wikidata(self.GENDER_WIKIDATA_CSV)
+        self.__write_gender_corrections(
+            "Title,Artist Credit,Performer Gender,Note\n"
+            "Female Song,Adele & Taylor Swift,mixed,reviewed\n")
+        status: int
+        stderr: str
+        status, stderr = self.__run_build(
+            "--wikidata-csv", str(self.__wikidata),
+            "--gender-corrections", str(self.__gender_corrections))
+        self.assertEqual(status, 0)
+        self.assertEqual(
+            self.__performer_genders()["Female Song"], "mixed")
+
+    def test_gender_correction_sets_undetermined_gender(
+            self) -> None:
+        """Test that a correction row sets the performer gender of
+        a song the derivation left undetermined."""
+        self.__write_chart(self.GENDER_CHART_CSV)
+        self.__write_wikidata(self.GENDER_WIKIDATA_CSV)
+        self.__write_gender_corrections(
+            "Title,Artist Credit,Performer Gender,Note\n"
+            "Unknown Song,Adele featuring Nobody,female,"
+            "reviewed\n")
+        status: int
+        stderr: str
+        status, stderr = self.__run_build(
+            "--wikidata-csv", str(self.__wikidata),
+            "--gender-corrections", str(self.__gender_corrections))
+        self.assertEqual(status, 0)
+        self.assertEqual(
+            self.__performer_genders()["Unknown Song"], "female")
+
+    def test_gender_correction_in_songs_csv(self) -> None:
+        """Test that songs.csv mirrors the corrected performer
+        gender."""
+        self.__write_chart(self.GENDER_CHART_CSV)
+        self.__write_wikidata(self.GENDER_WIKIDATA_CSV)
+        self.__write_gender_corrections(
+            "Title,Artist Credit,Performer Gender,Note\n"
+            "Female Song,Adele & Taylor Swift,mixed,reviewed\n")
+        self.assertEqual(
+            self.__run_build(
+                "--wikidata-csv", str(self.__wikidata),
+                "--gender-corrections",
+                str(self.__gender_corrections))[0], 0)
+        rows: list[list[str]] = self.__read_csv_rows("songs.csv")
+        self.assertIn(("Female Song", "mixed"),
+                      {(row[0], row[3]) for row in rows})
+
+    def test_omitted_gender_corrections_leaves_derived_gender(
+            self) -> None:
+        """Test that an omitted correction option leaves the
+        derived performer genders untouched."""
+        self.__write_chart(self.GENDER_CHART_CSV)
+        self.__write_wikidata(self.GENDER_WIKIDATA_CSV)
+        self.__write_gender_corrections(
+            "Title,Artist Credit,Performer Gender,Note\n"
+            "Female Song,Adele & Taylor Swift,mixed,reviewed\n")
+        status: int
+        stderr: str
+        status, stderr = self.__run_build(
+            "--wikidata-csv", str(self.__wikidata))
+        self.assertEqual(status, 0)
+        self.assertEqual(
+            self.__performer_genders()["Female Song"], "female")
+
+    def test_gender_correction_unknown_song_fails(self) -> None:
+        """Test that a correction row naming an unknown song fails
+        the build, naming the file and the offending title and
+        credit."""
+        self.__write_chart(self.GENDER_CHART_CSV)
+        self.__write_wikidata(self.GENDER_WIKIDATA_CSV)
+        self.__write_gender_corrections(
+            "Title,Artist Credit,Performer Gender,Note\n"
+            "Nowhere,Nobody,female,reviewed\n")
+        status: int
+        stderr: str
+        status, stderr = self.__run_build(
+            "--wikidata-csv", str(self.__wikidata),
+            "--gender-corrections", str(self.__gender_corrections))
+        self.assertNotEqual(status, 0)
+        self.assertIn(str(self.__gender_corrections), stderr)
+        self.assertIn("no song \"Nowhere\" by \"Nobody\"", stderr)
+
+    def test_gender_corrections_missing_column_fails(self) -> None:
+        """Test that a gender correction CSV missing a required
+        column fails the build."""
+        self.__write_gender_corrections(
+            "Title,Artist Credit,Performer Gender\n"
+            "Hello,Adele,female\n")
+        status: int
+        stderr: str
+        status, stderr = self.__run_build(
+            "--gender-corrections", str(self.__gender_corrections))
+        self.assertNotEqual(status, 0)
+        self.assertIn("missing column(s): Note", stderr)
+
+    def test_missing_gender_corrections_csv_fails(self) -> None:
+        """Test that a given but missing gender correction CSV
+        fails the build."""
+        status: int
+        stderr: str
+        status, stderr = self.__run_build(
+            "--gender-corrections", str(self.__gender_corrections))
+        self.assertNotEqual(status, 0)
+        self.assertIn("error:", stderr)
+        self.assertIn(str(self.__gender_corrections), stderr)
+        session: Session = self.__session()
+        self.assertEqual(list(session.scalars(sa.select(Song))), [])