diff --git a/docs/methodology.md b/docs/methodology.md index a9ca888..8991cf4 100644 --- a/docs/methodology.md +++ b/docs/methodology.md @@ -131,9 +131,13 @@ - **三次執行**:同一份定義檔、同一份輸入檔,獨立執行 三次,歸檔並列(`runs/04-group/run1`、`run2`、`run3`)。 - **多數決**:一個(群,編碼)配對,三次執行中至少兩次 - 入選即屬該群;不在 101 碼詞彙表內的輸出項無效。計票由 - 確定性子命令 `tally-groups` 完成,定案分群寫入 - `results/groups.csv`(欄位:群、編碼、票數)。 + 入選即屬該群;不在 101 碼詞彙表內的輸出項無效,每筆 + 丟棄印於標準錯誤。計票由確定性子命令 `tally-groups` + 完成:`tally-groups <執行歸檔 1> <執行歸檔 2> <執行歸檔 + 3> <合法碼清單> <輸出 CSV>`,合法碼清單之產法同步驟 3。 + 定案分群寫入 `results/groups.csv`,欄位 `Group`、 + `Keyword`、`Votes`,列序先依群名、再依編碼,一律以 + Unicode 碼位比較,換行為 CRLF。 ## 女性力量候選集 diff --git a/results/groups.csv b/results/groups.csv new file mode 100644 index 0000000..0ea8989 --- /dev/null +++ b/results/groups.csv @@ -0,0 +1,23 @@ +Group,Keyword,Votes +masculine,dominance-and-power,3 +masculine,family-and-fatherhood,2 +masculine,hustle-and-money,3 +masculine,rivalry-and-superiority,3 +masculine,self-confidence-and-braggadocio,3 +masculine,showing-off-and-impressing,3 +masculine,street-loyalty-and-danger,3 +masculine,violence-and-street-danger,3 +masculine,wealth-and-flexing,3 +misogyny,rejection-of-women,3 +vulnerable,disappointment-and-failure,3 +vulnerable,fear-of-losing-love,3 +vulnerable,heartbreak-and-grief,3 +vulnerable,hidden-emotional-struggle,3 +vulnerable,inner-mental-turmoil,3 +vulnerable,loneliness-and-isolation,3 +vulnerable,longing-and-loss,3 +vulnerable,past-trauma-and-healing,3 +vulnerable,self-worth-and-insecurity,3 +vulnerable,vulnerability-and-betrayal,3 +women-power,female-empowerment,3 +women-power,women-power,3 diff --git a/tools/README.rst b/tools/README.rst index 8716655..e025d22 100644 --- a/tools/README.rst +++ b/tools/README.rst @@ -73,6 +73,12 @@ tally-codings Settle the coding step by a majority of the three coding runs' archives, and write the final coding table, naming every song by its title and stored artist credit from the working store. Check ``pop-fem-audit-tools tally-codings -h`` for complete instructions on its usage. +tally-groups +------------ + +Settle the semantic code groups by a majority of the three group-selection runs' archives, and write the final group table, dropping any keyword outside the settled vocabulary. Check ``pop-fem-audit-tools tally-groups -h`` for complete instructions on its usage. + + Copyright ========= diff --git a/tools/docs/source/pop_fem_audit_tools.commands.rst b/tools/docs/source/pop_fem_audit_tools.commands.rst index 19771ff..ec9de86 100644 --- a/tools/docs/source/pop_fem_audit_tools.commands.rst +++ b/tools/docs/source/pop_fem_audit_tools.commands.rst @@ -60,6 +60,14 @@ pop\_fem\_audit\_tools.commands.tally\_codings module :show-inheritance: :undoc-members: +pop\_fem\_audit\_tools.commands.tally\_groups module +---------------------------------------------------- + +.. automodule:: pop_fem_audit_tools.commands.tally_groups + :members: + :show-inheritance: + :undoc-members: + Module contents --------------- diff --git a/tools/src/pop_fem_audit_tools/__main__.py b/tools/src/pop_fem_audit_tools/__main__.py index 1532753..131fdae 100644 --- a/tools/src/pop_fem_audit_tools/__main__.py +++ b/tools/src/pop_fem_audit_tools/__main__.py @@ -23,6 +23,7 @@ from .commands import ( fetch_lyrics_command, run_llm_command, tally_codings_command, + tally_groups_command, ) MODULE_PROG: str = "python -m pop_fem_audit_tools" @@ -36,6 +37,7 @@ SUBCOMMANDS: dict[str, Callable[[list[str] | None], int]] = { "fetch-lyrics": fetch_lyrics_command, "run-llm": run_llm_command, "tally-codings": tally_codings_command, + "tally-groups": tally_groups_command, } """The dispatch table from the subcommand name to the tool main.""" diff --git a/tools/src/pop_fem_audit_tools/commands/__init__.py b/tools/src/pop_fem_audit_tools/commands/__init__.py index adf9455..6bf3e9d 100644 --- a/tools/src/pop_fem_audit_tools/commands/__init__.py +++ b/tools/src/pop_fem_audit_tools/commands/__init__.py @@ -116,3 +116,23 @@ def tally_codings_command(argv: list[str] | None = None) -> int: """ from .tally_codings import main return main(argv) + + +def tally_groups_command(argv: list[str] | None = None) -> int: + """Settle the code groups by a majority of the three runs. + + Writes the final group table as the given CSV file, holding + the header row ``Group,Keyword,Votes`` and one row per + (group, keyword) pair at least two of the three selection + runs select, ordered by the group name and then by the + keyword. A selected item that is not in the valid keyword + list casts no vote. Nothing is written when the three + archives do not cover the same groups or a record is + malformed; the error message names what failed. + + :param argv: The command-line arguments, or None for + ``sys.argv``. + :return: The exit status: 0 on success, non-zero on failure. + """ + from .tally_groups import main + return main(argv) diff --git a/tools/src/pop_fem_audit_tools/commands/tally_groups.py b/tools/src/pop_fem_audit_tools/commands/tally_groups.py new file mode 100644 index 0000000..aa430f1 --- /dev/null +++ b/tools/src/pop_fem_audit_tools/commands/tally_groups.py @@ -0,0 +1,305 @@ +# Tools for A Feminist Audit of Pop Music. +# Copyright 2026 imacat. All rights reserved. +# Authors: +# imacat@mail.imacat.idv.tw (imacat), 2026/8/15 +# AI assistance: Claude Code (Anthropic) +"""The majority tally of the three group-selection runs. + +Settles the semantic code groups of step 4: the same group +selection definition file is run three times independently, and +this command counts the votes and writes the final group table +the paper cites, as the CSV file given as the last positional +command-line argument. A (group, keyword) pair is written out +when at least two of the three runs select it, so three votes +never tie. A selected item that is not in the valid keyword +list is invalid and casts no vote; every dropped occurrence is +reported on standard error. The group name is the record ID +with its ``group-`` prefix dropped. The rows are ordered by the +group name and then by the keyword, by Unicode code point, and +the file carries the header row ``Group,Keyword,Votes`` with +CRLF line endings per RFC 4180. + +The three archives must cover exactly the same set of group IDs, +every record ID must carry the ``group-`` prefix, and every +record's "text" must parse to a JSON array of strings; otherwise +the tally fails and nothing is written. +""" +import argparse +import csv +import json +import sys +import time +from pathlib import Path +from typing import Any + +from ..utils import format_duration + +GROUP_ID_PREFIX: str = "group-" +"""The prefix every group record ID must carry; the group name +is the rest of the ID.""" +MAJORITY: int = 2 +"""The number of runs that must select a keyword for a group for +that pair to be settled.""" +HEADER: tuple[str, str, str] = ("Group", "Keyword", "Votes") +"""The header row of the group table CSV file.""" + + +class TallyError(Exception): + """An error that fails the group tally.""" + + +def parse_args(argv: list[str] | None) -> argparse.Namespace: + """Parse the command-line arguments. + + :param argv: The command-line arguments, or None for + ``sys.argv``. + :return: The parsed arguments. + """ + parser: argparse.ArgumentParser = argparse.ArgumentParser( + description="Settle the semantic code groups by a" + " majority of the three selection runs.") + parser.add_argument( + "run_dir_1", type=Path, + help="the first selection run's archive directory") + parser.add_argument( + "run_dir_2", type=Path, + help="the second selection run's archive directory") + parser.add_argument( + "run_dir_3", type=Path, + help="the third selection run's archive directory") + parser.add_argument( + "valid_keywords", type=Path, + help="a plain text file of the allowed keywords, one per" + " line") + parser.add_argument( + "output_csv", type=Path, + help="the output CSV file, by convention" + " results/groups.csv") + return parser.parse_args(argv) + + +def load_valid_keywords(path: Path) -> set[str]: + """Load the valid keyword list. + + :param path: The plain text file of the allowed keywords, one + per line. + :return: The allowed keywords. + :raises TallyError: When the file cannot be read or holds no + keyword. + """ + text: str + try: + text = path.read_text(encoding="utf-8") + except OSError as error: + raise TallyError(str(error)) from error + keywords: set[str] = {x.strip() for x in text.split("\n") + if x.strip() != ""} + if len(keywords) == 0: + raise TallyError(f"{path}: no keywords") + return keywords + + +def load_run(run_dir: Path) -> dict[str, set[str]]: + """Load and validate the selection records of one run. + + :param run_dir: The run's archive directory, containing + ``output.jsonl``. + :return: The selected keywords of every group of the run, + keyed by the group name, the duplicates within one + record's selection collapsed. + :raises TallyError: When the file cannot be read, a line is + not a well-formed output record, a record is not a + successful result, an ID lacks the ``group-`` prefix, a + group has two records, or a "text" does not parse to a + JSON array of strings. + """ + path: Path = run_dir / "output.jsonl" + text: str + try: + text = path.read_text(encoding="utf-8") + except OSError as error: + raise TallyError(str(error)) from error + records: dict[str, set[str]] = {} + line: str + for line in text.split("\n"): + if line.strip() == "": + continue + record: Any + try: + record = json.loads(line) + except json.JSONDecodeError as error: + raise TallyError( + f"{path}: malformed JSON: {error}") from error + if not isinstance(record, dict) or "id" not in record: + raise TallyError( + f"{path}: record without \"id\": {line}") + item_id: Any = record["id"] + if "error" in record or "text" not in record: + raise TallyError( + f"{path}: id {item_id}: not a successful result") + if not isinstance(item_id, str) \ + or not item_id.startswith(GROUP_ID_PREFIX) \ + or item_id == GROUP_ID_PREFIX: + raise TallyError( + f"{path}: id {item_id}: not in the" + f" \"{GROUP_ID_PREFIX}\" form") + group: str = item_id[len(GROUP_ID_PREFIX):] + if group in records: + raise TallyError( + f"{path}: id {item_id}: duplicate record") + records[group] = _selection( + record["text"], f"{path}: id {item_id}") + if len(records) == 0: + raise TallyError(f"{path}: no records") + return records + + +def _selection(text: Any, label: str) -> set[str]: + """Parse and validate the selected keywords of one record. + + :param text: The "text" field of the record. + :param label: The location of the record, for the error + message. + :return: The selected keywords, the duplicates collapsed. + :raises TallyError: When the text does not parse to a JSON + array of strings. + """ + if not isinstance(text, str): + raise TallyError(f"{label}: \"text\" is not a string") + selected: Any + try: + selected = json.loads(text) + except json.JSONDecodeError as error: + raise TallyError( + f"{label}: \"text\" is malformed JSON:" + f" {error}") from error + if not isinstance(selected, list) \ + or not all(isinstance(x, str) for x in selected): + raise TallyError( + f"{label}: \"text\" does not parse to a JSON array" + " of strings") + return set(selected) + + +def drop_invalid(runs: list[dict[str, set[str]]], + run_dirs: list[Path], + valid: set[str]) -> None: + """Drop the out-of-vocabulary selections of every run. + + Every dropped occurrence is reported on standard error as an + observable side effect. + + :param runs: The runs' records, filtered in place. + :param run_dirs: The run directories, for the messages. + :param valid: The allowed keywords. + :return: None. + """ + records: dict[str, set[str]] + run_dir: Path + for records, run_dir in zip(runs, run_dirs): + group: str + selected: set[str] + for group, selected in records.items(): + keyword: str + for keyword in sorted(selected - valid): + print( + f"note: {run_dir.name} group-{group}:" + f" dropped out-of-vocabulary item" + f" \"{keyword}\"", file=sys.stderr) + records[group] = selected & valid + + +def tally(runs: list[dict[str, set[str]]]) \ + -> list[tuple[str, str, int]]: + """Tally the keyword votes of the runs, group by group. + + :param runs: The runs' records, all covering the same set of + groups. + :return: The settled rows, each the group name, the keyword, + and the number of votes, ordered by the group name and + then by the keyword, by Unicode code point. + :raises TallyError: When the runs do not cover the same set + of groups. + """ + groups: set[str] = set(runs[0]) + records: dict[str, set[str]] + for records in runs[1:]: + if set(records) != groups: + raise TallyError( + "the three runs do not cover the same groups: " + + ", ".join(sorted( + groups.symmetric_difference(set(records))))) + rows: list[tuple[str, str, int]] = [] + group: str + for group in sorted(groups): + votes: dict[str, int] = {} + for records in runs: + keyword: str + for keyword in records[group]: + votes[keyword] = votes.get(keyword, 0) + 1 + rows.extend( + (group, x, votes[x]) + for x in sorted(votes) if votes[x] >= MAJORITY) + return rows + + +def write_csv(output_csv: Path, + rows: list[tuple[str, str, int]]) -> None: + """Write the group table CSV file. + + Writes an RFC 4180 CSV file, UTF-8, with CRLF line endings, + carrying the header row ``Group,Keyword,Votes`` and one row + per settled (group, keyword) pair, in the row order. The + parent directory is created when it does not exist. + + :param output_csv: The output CSV file. + :param rows: The settled rows, in the output order. + :return: None. + :raises OSError: When the file cannot be written. + """ + output_csv.parent.mkdir(parents=True, exist_ok=True) + with open(output_csv, "w", encoding="utf-8", + newline="") as file: + writer: Any = csv.writer(file) + writer.writerow(HEADER) + writer.writerows(rows) + + +def main(argv: list[str] | None = None) -> int: + """Settle the code groups by a majority of the three runs. + + Writes the final group table as the given CSV file, holding + the header row ``Group,Keyword,Votes`` and one row per + (group, keyword) pair at least two of the three runs select, + ordered by the group name and then by the keyword. A + selected item that is not in the valid keyword list casts no + vote, each dropped occurrence reported on standard error. + Nothing is written when the three archives do not cover the + same groups, a record is not a successful result, an ID lacks + the ``group-`` prefix, or a record's "text" does not parse to + a JSON array of strings; the error message names what failed. + + :param argv: The command-line arguments, or None for + ``sys.argv``. + :return: The exit status: 0 on success, non-zero on failure. + """ + started: float = time.monotonic() + args: argparse.Namespace = parse_args(argv) + run_dirs: list[Path] = [ + args.run_dir_1, args.run_dir_2, args.run_dir_3] + try: + valid: set[str] = load_valid_keywords(args.valid_keywords) + runs: list[dict[str, set[str]]] = [ + load_run(x) for x in run_dirs] + drop_invalid(runs, run_dirs, valid) + rows: list[tuple[str, str, int]] = tally(runs) + write_csv(args.output_csv, rows) + except (TallyError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + elapsed: str = format_duration(time.monotonic() - started) + print( + f"Done. Settled {len(rows)} codes across" + f" {len(runs[0])} groups. {elapsed} elapsed.", + file=sys.stderr) + return 0 diff --git a/tools/tests/test_tally_groups.py b/tools/tests/test_tally_groups.py new file mode 100644 index 0000000..58b066e --- /dev/null +++ b/tools/tests/test_tally_groups.py @@ -0,0 +1,243 @@ +# Tools for A Feminist Audit of Pop Music. +# Copyright 2026 imacat. All rights reserved. +# Authors: +# imacat@mail.imacat.idv.tw (imacat), 2026/8/15 +# AI assistance: Claude Code (Anthropic) +"""Unit tests for the group tally module.""" +import csv +import io +import json +import tempfile +import unittest +from contextlib import redirect_stderr +from pathlib import Path + +from pop_fem_audit_tools.commands import tally_groups + + +class TestTallyGroups(unittest.TestCase): + """Test cases for the group tally.""" + + def setUp(self) -> None: + """Create the run directories and the input files.""" + tmp: tempfile.TemporaryDirectory[str] \ + = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.__dir: Path = Path(tmp.name) + self.__runs: list[Path] = [] + number: int + for number in (1, 2, 3): + run_dir: Path = self.__dir / f"run{number}" + run_dir.mkdir() + self.__runs.append(run_dir) + self.__valid: Path = self.__dir / "valid-keywords.txt" + self.__valid.write_text( + "alpha\nbeta\ngamma\ndelta\n", encoding="utf-8") + self.__output_csv: Path \ + = self.__dir / "results" / "groups.csv" + + def __write_runs( + self, runs: list[dict[str, list[str]]]) -> None: + """Write the three runs' output.jsonl files. + + :param runs: The selections of each run, keyed by the + group record ID. + :return: None. + """ + run_dir: Path + selections: dict[str, list[str]] + for run_dir, selections in zip(self.__runs, runs): + lines: list[str] = [ + json.dumps({"id": x, "text": json.dumps(y), + "stop_reason": "end_turn"}) + for x, y in selections.items()] + (run_dir / "output.jsonl").write_text( + "\n".join(lines) + "\n", encoding="utf-8") + + def __run_tally(self) -> tuple[int, str]: + """Run the tally command against the run directories. + + :return: The exit status and the standard error text. + """ + stderr: io.StringIO = io.StringIO() + status: int + with redirect_stderr(stderr): + status = tally_groups.main( + [str(x) for x in self.__runs] + + [str(self.__valid), str(self.__output_csv)]) + return status, stderr.getvalue() + + def __read_rows(self) -> list[list[str]]: + """Read the written CSV file back as rows. + + :return: The rows, the header row included. + """ + with open(self.__output_csv, encoding="utf-8", + newline="") as file: + return list(csv.reader(file)) + + def test_majority_vote_settles_two_of_three(self) -> None: + """Test that a pair needs at least two votes to settle.""" + self.__write_runs([ + {"group-one": ["alpha", "beta", "gamma"]}, + {"group-one": ["alpha", "beta"]}, + {"group-one": ["alpha"]}]) + status: int + status, _ = self.__run_tally() + self.assertEqual(status, 0) + self.assertEqual(self.__read_rows(), [ + ["Group", "Keyword", "Votes"], + ["one", "alpha", "3"], + ["one", "beta", "2"]]) + + def test_rows_sorted_by_group_then_keyword(self) -> None: + """Test the row order: group name, then keyword, by + Unicode code point.""" + selections: dict[str, list[str]] = { + "group-women-power": ["beta", "alpha"], + "group-masculine": ["delta", "gamma"]} + self.__write_runs([selections, selections, selections]) + status: int + status, _ = self.__run_tally() + self.assertEqual(status, 0) + self.assertEqual(self.__read_rows(), [ + ["Group", "Keyword", "Votes"], + ["masculine", "delta", "3"], + ["masculine", "gamma", "3"], + ["women-power", "alpha", "3"], + ["women-power", "beta", "3"]]) + + def test_output_is_crlf_with_header(self) -> None: + """Test the written bytes: RFC 4180, CRLF, header row.""" + selections: dict[str, list[str]] = {"group-one": ["alpha"]} + self.__write_runs([selections, selections, selections]) + status: int + status, _ = self.__run_tally() + self.assertEqual(status, 0) + self.assertEqual( + self.__output_csv.read_bytes(), + b"Group,Keyword,Votes\r\n" + b"one,alpha,3\r\n") + + def test_out_of_vocabulary_item_casts_no_vote(self) -> None: + """Test that a selected item outside the valid keyword + list is dropped, each occurrence reported on standard + error.""" + selections: dict[str, list[str]] = { + "group-one": ["alpha", "hallucinated"]} + self.__write_runs([selections, selections, selections]) + status: int + stderr: str + status, stderr = self.__run_tally() + self.assertEqual(status, 0) + self.assertEqual(self.__read_rows(), [ + ["Group", "Keyword", "Votes"], + ["one", "alpha", "3"]]) + self.assertEqual(stderr.count( + "dropped out-of-vocabulary item \"hallucinated\""), 3) + + def test_duplicate_selection_counts_once(self) -> None: + """Test that a keyword listed twice in one run's selection + casts one vote only.""" + self.__write_runs([ + {"group-one": ["alpha", "alpha"]}, + {"group-one": []}, + {"group-one": []}]) + status: int + status, _ = self.__run_tally() + self.assertEqual(status, 0) + self.assertEqual(self.__read_rows(), + [["Group", "Keyword", "Votes"]]) + + def test_empty_selections_yield_header_only(self) -> None: + """Test that no settled pair still writes the header.""" + selections: dict[str, list[str]] = {"group-one": []} + self.__write_runs([selections, selections, selections]) + status: int + status, _ = self.__run_tally() + self.assertEqual(status, 0) + self.assertEqual(self.__read_rows(), + [["Group", "Keyword", "Votes"]]) + + def test_mismatched_group_sets_fail(self) -> None: + """Test that runs covering different groups fail, nothing + written.""" + self.__write_runs([ + {"group-one": ["alpha"]}, + {"group-one": ["alpha"]}, + {"group-two": ["alpha"]}]) + status: int + stderr: str + status, stderr = self.__run_tally() + self.assertNotEqual(status, 0) + self.assertIn("do not cover the same groups", stderr) + self.assertFalse(self.__output_csv.exists()) + + def test_id_without_the_group_prefix_fails(self) -> None: + """Test that a record ID without the "group-" prefix + fails the run.""" + selections: dict[str, list[str]] = {"song-1": ["alpha"]} + self.__write_runs([selections, selections, selections]) + status: int + stderr: str + status, stderr = self.__run_tally() + self.assertNotEqual(status, 0) + self.assertIn("group-", stderr) + self.assertFalse(self.__output_csv.exists()) + + def test_duplicate_record_fails(self) -> None: + """Test that two records of one group in one run fail the + run.""" + selections: dict[str, list[str]] = {"group-one": ["alpha"]} + self.__write_runs([selections, selections, selections]) + path: Path = self.__runs[0] / "output.jsonl" + path.write_text( + path.read_text(encoding="utf-8") + + json.dumps({"id": "group-one", + "text": json.dumps(["beta"])}) + "\n", + encoding="utf-8") + status: int + stderr: str + status, stderr = self.__run_tally() + self.assertNotEqual(status, 0) + self.assertIn("duplicate record", stderr) + + def test_text_not_an_array_of_strings_fails(self) -> None: + """Test that a "text" that is not a JSON array of strings + fails the run.""" + self.__write_runs([ + {"group-one": ["alpha"]}, + {"group-one": ["alpha"]}, + {"group-one": ["alpha"]}]) + (self.__runs[2] / "output.jsonl").write_text( + json.dumps({"id": "group-one", + "text": json.dumps({"alpha": []})}) + + "\n", encoding="utf-8") + status: int + stderr: str + status, stderr = self.__run_tally() + self.assertNotEqual(status, 0) + self.assertIn("JSON array of strings", stderr) + + def test_unsuccessful_record_fails(self) -> None: + """Test that an errored record fails the run.""" + selections: dict[str, list[str]] = {"group-one": ["alpha"]} + self.__write_runs([selections, selections, selections]) + (self.__runs[1] / "output.jsonl").write_text( + json.dumps({"id": "group-one", "error": "overloaded"}) + + "\n", encoding="utf-8") + status: int + stderr: str + status, stderr = self.__run_tally() + self.assertNotEqual(status, 0) + self.assertIn("not a successful result", stderr) + + def test_missing_valid_keywords_file_fails(self) -> None: + """Test that a missing valid keyword list fails the run.""" + selections: dict[str, list[str]] = {"group-one": ["alpha"]} + self.__write_runs([selections, selections, selections]) + self.__valid.unlink() + status: int + status, _ = self.__run_tally() + self.assertNotEqual(status, 0) + self.assertFalse(self.__output_csv.exists())