From 15cdb861ff42b063a04505f36d7c3a869f51172b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BE=9D=E7=91=AA=E8=B2=93?= Date: Sat, 15 Aug 2026 00:56:55 +0800 Subject: [PATCH] Import the settled code groups into the working store Co-Authored-By: Claude Fable 5 --- docs/decision-log.md | 5 + docs/methodology.md | 3 + .../pop_fem_audit_tools/commands/build_db.py | 112 +++++++++++++++++- tools/src/pop_fem_audit_tools/models.py | 14 +++ tools/tests/test_build_db.py | 101 ++++++++++++++++ 5 files changed, 232 insertions(+), 3 deletions(-) diff --git a/docs/decision-log.md b/docs/decision-log.md index 461fde4..831d9d9 100644 --- a/docs/decision-log.md +++ b/docs/decision-log.md @@ -744,3 +744,8 @@ 逐一替換;定案表重新計票,與舊檔之逐位元差異恰為 `\n`→「 / 」(14,664 列不變,含「 / 」之引述 1,658 筆);工作儲存重建後引述零換行字元。 + +- **定案分群匯入工作儲存**:`build-db` 增 `--groups` 選項, + 將 `results/groups.csv` 逐欄照存入 `groups` 資料表(群、 + 編碼、票數),納入重建摘要與清空範圍,供群層次查詢; + 重建命令自此帶 `--groups results/groups.csv`。 diff --git a/docs/methodology.md b/docs/methodology.md index 8991cf4..1148eec 100644 --- a/docs/methodology.md +++ b/docs/methodology.md @@ -138,6 +138,9 @@ 定案分群寫入 `results/groups.csv`，欄位 `Group`、 `Keyword`、`Votes`，列序先依群名、再依編碼，一律以 Unicode 碼位比較，換行為 CRLF。 +- **工作儲存**：`build-db --groups <定案分群 CSV>` 將定案 + 分群逐欄照存入 `groups` 資料表（群、編碼、票數），供 + 群層次查詢。 ## 女性力量候選集 diff --git a/tools/src/pop_fem_audit_tools/commands/build_db.py b/tools/src/pop_fem_audit_tools/commands/build_db.py index de22920..8c5522d 100644 --- a/tools/src/pop_fem_audit_tools/commands/build_db.py +++ b/tools/src/pop_fem_audit_tools/commands/build_db.py @@ -72,6 +72,7 @@ from ..database import Base, ds from ..models import ( Artist, ChartEntry, + CodeGroup, Coding, Role, Song, @@ -118,6 +119,9 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: parser.add_argument( "--codings", type=Path, default=None, help="the settled coding table CSV file to import") + parser.add_argument( + "--groups", type=Path, default=None, + help="the settled code group table CSV file to import") return parser.parse_args(argv) @@ -785,6 +789,101 @@ class CodingImporter: f"{path}: missing column(s): {', '.join(missing)}") +class GroupImporter: + """The group-import job: loads the settled code group table + into the working store.""" + + COLUMNS: tuple[str, ...] = ("Group", "Keyword", "Votes") + """The required columns of the group CSV file.""" + + def __init__(self, session: Session) -> None: + """Initialize the importer. + + :param session: The database session. + """ + self.__session: Session = session + + def import_groups(self, path: Path | None) -> None: + """Load the settled code group table into the store. + + A None input leaves the groups unloaded. Otherwise every + row of the CSV file yields one member keyword of one + group, with the group name, the keyword, and the integer + vote count stored verbatim. When the method returns, the + imported groups are queryable in the session. + + :param path: The settled code group table CSV file to + import, or None to skip the groups. + :return: None. + :raises BuildError: When the file lacks a required + column, a votes field is not an integer, or two rows + name the same group and keyword. + :raises OSError: When the file cannot be read. + """ + if path is None: + return + seen: set[tuple[str, str]] = set() + with open(path, encoding="utf-8", newline="") as file: + reader: csv.DictReader[str] = csv.DictReader(file) + self.__check_columns(path, reader.fieldnames) + row: dict[str, str] + for row in reader: + self.__import_group_row(path, seen, row) + self.__session.flush() + + def __import_group_row(self, path: Path, + seen: set[tuple[str, str]], + row: dict[str, str]) -> None: + """Store one group member row. + + :param path: The group CSV file, for the error messages. + :param seen: The (group, keyword) pairs already stored, + updated with the pair of this row. + :param row: The group CSV row. + :return: None. + :raises BuildError: When the votes field is not an + integer, or the group and keyword repeat an earlier + row. + """ + key: tuple[str, str] = (row["Group"], row["Keyword"]) + if key in seen: + raise BuildError( + f"{path}: duplicated group member: group" + f" \"{row['Group']}\", keyword" + f" \"{row['Keyword']}\"") + seen.add(key) + votes: int + try: + votes = int(row["Votes"]) + except ValueError as error: + raise BuildError( + f"{path}: group \"{row['Group']}\", keyword" + f" \"{row['Keyword']}\": votes" + f" \"{row['Votes']}\" is not an integer" + ) from error + self.__session.add(CodeGroup( + group=row["Group"], keyword=row["Keyword"], + votes=votes)) + + @classmethod + def __check_columns(cls, path: Path, + fieldnames: Sequence[str] | None) -> None: + """Verify the group CSV file has the required columns. + + :param path: The group CSV file. + :param fieldnames: The header row of the file, or None + when the file is empty. + :return: None. + :raises BuildError: When a required column is absent. + """ + header: Sequence[str] = fieldnames or () + missing: list[str] = [ + x for x in cls.COLUMNS if x not in header] + if len(missing) > 0: + raise BuildError( + f"{path}: missing column(s): {', '.join(missing)}") + + @dataclass class StoreCounts: """The row counts of the working store, for the build summary.""" @@ -795,6 +894,8 @@ class StoreCounts: """The number of the artists.""" codings: int """The number of the settled codings.""" + groups: int + """The number of the settled code group members.""" @classmethod def get_instance(cls, session: Session) -> Self: @@ -815,7 +916,9 @@ class StoreCounts: artists=count( sa.select(sa.func.count()).select_from(Artist)), codings=count( - sa.select(sa.func.count()).select_from(Coding))) + sa.select(sa.func.count()).select_from(Coding)), + groups=count( + sa.select(sa.func.count()).select_from(CodeGroup))) def reset_store(session: Session) -> None: @@ -825,7 +928,8 @@ def reset_store(session: Session) -> None: :return: None. """ model: type[Base] - for model in (Coding, SongArtist, ChartEntry, Song, Artist): + for model in (CodeGroup, Coding, SongArtist, ChartEntry, Song, + Artist): session.execute(sa.delete(model)) @@ -988,6 +1092,7 @@ def main(argv: list[str] | None = None) -> int: args.lyrics_dir, args.wikidata_csv) PerformerGenderDeriver(session).derive_performer_genders() CodingImporter(session).import_codings(args.codings) + GroupImporter(session).import_groups(args.groups) counts = StoreCounts.get_instance(session) CSVExporter(session, args.derived_dir).write() session.commit() @@ -999,6 +1104,7 @@ def main(argv: list[str] | None = None) -> int: session.close() elapsed: str = format_duration(time.monotonic() - started) print(f"Done. {counts.songs} songs/{counts.artists} artists" - f"/{counts.codings} codings. {elapsed} elapsed.", + f"/{counts.codings} codings/{counts.groups} group" + f" members. {elapsed} elapsed.", file=sys.stderr) return 0 diff --git a/tools/src/pop_fem_audit_tools/models.py b/tools/src/pop_fem_audit_tools/models.py index 95b0b57..ae53220 100644 --- a/tools/src/pop_fem_audit_tools/models.py +++ b/tools/src/pop_fem_audit_tools/models.py @@ -139,3 +139,17 @@ class Coding(Base): single "|", empty when the keyword carries no evidence.""" song: Mapped[Song] = relationship(back_populates="codings") """The coded song.""" + + +class CodeGroup(Base): + """A settled member keyword of a semantic code group.""" + __tablename__ = "groups" + """The table name.""" + + group: Mapped[str] = mapped_column(primary_key=True) + """The group name, as the settled group table carries it.""" + keyword: Mapped[str] = mapped_column(primary_key=True) + """The member coding keyword.""" + votes: Mapped[int] = mapped_column() + """The number of the selection runs that selected the + keyword for the group.""" diff --git a/tools/tests/test_build_db.py b/tools/tests/test_build_db.py index 1d65081..5134ce4 100644 --- a/tools/tests/test_build_db.py +++ b/tools/tests/test_build_db.py @@ -21,6 +21,7 @@ from pop_fem_audit_tools.database import DataSource from pop_fem_audit_tools.models import ( Artist, ChartEntry, + CodeGroup, Coding, Role, Song, @@ -268,6 +269,7 @@ class TestBuildDB(unittest.TestCase): self.__wikidata: Path = \ self.__dir / "artists_wikidata.csv" self.__codings: Path = self.__dir / "codings.csv" + self.__groups: Path = self.__dir / "groups.csv" self.__write_chart(self.CHART_CSV) config.set_settings(config.Settings( SQLALCHEMY_DATABASE_URL="sqlite://", @@ -1121,3 +1123,102 @@ class TestBuildDB(unittest.TestCase): self.__stored_codings(), {("Shape of You", "attraction"): "I'm in love with your body"}) + + GROUPS_CSV: str = ( + "Group,Keyword,Votes\n" + "masculine,dominance-and-power,3\n" + "masculine,family-and-fatherhood,2\n" + "women-power,women-power,3\n") + """The group CSV fixture: two groups, one two-vote member.""" + + def __write_groups(self, content: str) -> None: + """Write the group CSV fixture. + + :param content: The CSV content. + :return: None. + """ + self.__groups.write_text(content, encoding="utf-8") + + def __stored_groups(self) -> dict[tuple[str, str], int]: + """Read the stored group members keyed by group and keyword. + + :return: The stored votes, keyed by the group name and the + keyword. + """ + session: Session = self.__session() + return {(x.group, x.keyword): x.votes + for x in session.scalars(sa.select(CodeGroup))} + + def test_groups_imported(self) -> None: + """Test that the group CSV imports one row per group and + keyword, the votes stored as integers.""" + self.__write_groups(self.GROUPS_CSV) + status: int + stderr: str + status, stderr = self.__run_build( + "--groups", str(self.__groups)) + self.assertEqual(status, 0) + self.assertIn("3 group members", stderr) + self.assertEqual( + self.__stored_groups(), + {("masculine", "dominance-and-power"): 3, + ("masculine", "family-and-fatherhood"): 2, + ("women-power", "women-power"): 3}) + + def test_groups_replaced_on_rebuild(self) -> None: + """Test that a rebuild replaces the previous groups rather + than adding to them.""" + self.__write_groups(self.GROUPS_CSV) + self.assertEqual( + self.__run_build("--groups", str(self.__groups))[0], 0) + self.__write_groups( + "Group,Keyword,Votes\n" + "vulnerable,longing-and-loss,3\n") + status: int + stderr: str + status, stderr = self.__run_build( + "--groups", str(self.__groups)) + self.assertEqual(status, 0) + self.assertIn("1 group members", stderr) + self.assertEqual( + self.__stored_groups(), + {("vulnerable", "longing-and-loss"): 3}) + + def test_duplicated_group_member_fails(self) -> None: + """Test that two rows naming the same group and keyword + fail the build.""" + self.__write_groups( + "Group,Keyword,Votes\n" + "masculine,dominance-and-power,3\n" + "masculine,dominance-and-power,2\n") + status: int + stderr: str + status, stderr = self.__run_build( + "--groups", str(self.__groups)) + self.assertNotEqual(status, 0) + self.assertIn("duplicated group member", stderr) + + def test_group_votes_not_an_integer_fails(self) -> None: + """Test that a non-integer votes field fails the build.""" + self.__write_groups( + "Group,Keyword,Votes\n" + "masculine,dominance-and-power,three\n") + status: int + stderr: str + status, stderr = self.__run_build( + "--groups", str(self.__groups)) + self.assertNotEqual(status, 0) + self.assertIn("is not an integer", stderr) + + def test_groups_missing_column_fails(self) -> None: + """Test that a group CSV without a required column fails + the build.""" + self.__write_groups( + "Group,Keyword\n" + "masculine,dominance-and-power\n") + status: int + stderr: str + status, stderr = self.__run_build( + "--groups", str(self.__groups)) + self.assertNotEqual(status, 0) + self.assertIn("missing column(s): Votes", stderr)