diff --git a/tools/src/pop_fem_audit_tools/commands/build_db.py b/tools/src/pop_fem_audit_tools/commands/build_db.py index c8c695e..61ea48a 100644 --- a/tools/src/pop_fem_audit_tools/commands/build_db.py +++ b/tools/src/pop_fem_audit_tools/commands/build_db.py @@ -632,7 +632,7 @@ class PerformerGenderDeriver: known: set[str] = {x for x in values if x} if len(known) > 1: return cls.__MIXED - if len(known) == 1 and all(x for x in values): + if len(known) == 1 and all(values): return known.pop() return None diff --git a/tools/src/pop_fem_audit_tools/commands/fetch_artists.py b/tools/src/pop_fem_audit_tools/commands/fetch_artists.py index 68140ad..b134dcf 100644 --- a/tools/src/pop_fem_audit_tools/commands/fetch_artists.py +++ b/tools/src/pop_fem_audit_tools/commands/fetch_artists.py @@ -161,6 +161,10 @@ class RetryExhausted(Exception): """ +NOTE_NOT_FOUND: str = "not found" +"""The note marking an artist that could not be resolved.""" + + class ArtistFetcher: """A fetcher of artist metadata from Wikidata. @@ -252,9 +256,6 @@ class ArtistFetcher: = ("band", "group", "duo", "trio") """The label keywords that suggest a musical ensemble, covering labels like "boy band" and "girl group".""" - __NOTE_NOT_FOUND: ClassVar[str] = "not found" - """The note sentinel of an artist without a resolved - Wikidata item.""" __MIXED_GENDER: ClassVar[str] = "mixed" """The gender recorded for a group whose members do not share one gender.""" @@ -301,7 +302,7 @@ class ArtistFetcher: try: qid: str | None = self.__resolve_qid(name, titles) if qid is None: - snapshot.note = self.__NOTE_NOT_FOUND + snapshot.note = NOTE_NOT_FOUND return snapshot snapshot.qid = qid self.__resolve(snapshot) @@ -1039,9 +1040,9 @@ class ArtistSnapshotUpdater: artist.name, titles) self.__append_row(csv_file, snapshot) status: str = snapshot.qid - if snapshot.note == "not found": + if snapshot.note == NOTE_NOT_FOUND: not_found += 1 - status = "not found" + status = NOTE_NOT_FOUND elif snapshot.note.startswith("error: "): errors += 1 status = snapshot.note diff --git a/tools/src/pop_fem_audit_tools/commands/tally_annotations.py b/tools/src/pop_fem_audit_tools/commands/tally_annotations.py index ac5a818..3008c69 100644 --- a/tools/src/pop_fem_audit_tools/commands/tally_annotations.py +++ b/tools/src/pop_fem_audit_tools/commands/tally_annotations.py @@ -82,11 +82,11 @@ class AnnotationTallier: song's stored performer gender; a gender missing here (including None) takes every prefix.""" __NUMBERING_RE: ClassVar[re.Pattern[str]] = re.compile( - r"^(?:模式[一二三四五六七八九十]+|[一二三四五六七八九十]+)[：、]") + r"^(?:模式)?[一二三四五六七八九十]+[：、]") """The leading numbering token of a pattern heading, stripped to yield the pattern name.""" __HEADING_RE: ClassVar[re.Pattern[str]] = re.compile( - r"^#+\s*(.*)$") + r"^#++\s*+(.*)$") """A Markdown heading line, the heading text captured.""" def __init__(self, male_synthesis: Path, female_synthesis: Path, @@ -311,36 +311,7 @@ class AnnotationTallier: pooled: dict[int, list[list[str]]] = {} run_dir: Path for run_dir in self.__run_dirs: - path: Path = run_dir / "output.jsonl" - text: str - try: - text = path.read_text(encoding="utf-8") - except OSError as error: - raise TallyError(str(error)) from error - line: str - for line in text.split("\n"): - if line.strip() == "": - continue - record: Any - try: - record = json.loads(line) - except json.JSONDecodeError as error: - raise TallyError( - f"{path}: malformed JSON: {error}") \ - from error - song_id: int = self.__parse_song_id( - record["id"], run_dir) - if "text" not in record: - print( - f"warning: {run_dir}: song-{song_id}: no" - " \"text\" field, skipped", - file=sys.stderr) - continue - ballot: list[str] | None = self.__parse_ballot( - record["text"], run_dir, song_id) - if ballot is None: - continue - pooled.setdefault(song_id, []).append(ballot) + self.__load_run_ballots(run_dir, pooled) song_id: int ballots: list[list[str]] for song_id, ballots in pooled.items(): @@ -351,6 +322,74 @@ class AnnotationTallier: f" {self.__BALLOTS_PER_SONG}") return pooled + @classmethod + def __load_run_ballots( + cls, run_dir: Path, + pooled: dict[int, list[list[str]]]) -> None: + """Load one run's ballots into the pool. + + :param run_dir: The annotation run's archive directory, + containing ``output.jsonl``. + :param pooled: The pool to append the run's ballots + into, keyed by the numeric song ID; mutated in + place. + :return: None. + :raises TallyError: When the run's ``output.jsonl`` + cannot be read, or a line is malformed JSON. + """ + path: Path = run_dir / "output.jsonl" + text: str + try: + text = path.read_text(encoding="utf-8") + except OSError as error: + raise TallyError(str(error)) from error + line: str + for line in text.split("\n"): + if line.strip() == "": + continue + record: Any + try: + record = json.loads(line) + except json.JSONDecodeError as error: + raise TallyError( + f"{path}: malformed JSON: {error}") \ + from error + cls.__load_ballot_record(record, run_dir, pooled) + + @classmethod + def __load_ballot_record( + cls, record: Any, run_dir: Path, + pooled: dict[int, list[list[str]]]) -> None: + """Parse and pool one ballot record. + + A record whose "text" field is missing, or does not + parse to a JSON array of strings, is skipped, a warning + naming the run directory and song reported on standard + error, as an observable side effect. + + :param record: The parsed JSON record. + :param run_dir: The run directory the record came from, + for the warning and error messages. + :param pooled: The pool to append the record's ballot + into, keyed by the numeric song ID; mutated in + place. + :return: None. + :raises TallyError: When the record's "id" field is not + in the ``song-`` form. + """ + song_id: int = cls.__parse_song_id(record["id"], run_dir) + if "text" not in record: + print( + f"warning: {run_dir}: song-{song_id}: no" + " \"text\" field, skipped", + file=sys.stderr) + return + ballot: list[str] | None = cls.__parse_ballot( + record["text"], run_dir, song_id) + if ballot is None: + return + pooled.setdefault(song_id, []).append(ballot) + @classmethod def __parse_song_id(cls, item_id: Any, run_dir: Path) -> int: """Parse the numeric song ID out of an annotation record diff --git a/tools/tests/test_run_llm.py b/tools/tests/test_run_llm.py index fe98b7a..aff3956 100644 --- a/tools/tests/test_run_llm.py +++ b/tools/tests/test_run_llm.py @@ -21,7 +21,7 @@ from pop_fem_audit_tools.commands import run_llm class RunLLMTestCase(unittest.TestCase): """The common base test case with the shared helpers.""" - def make_temp_dir(self) -> Path: + def _make_temp_dir(self) -> Path: """Create a temporary directory removed on test cleanup. :return: The path of the temporary directory. @@ -31,8 +31,8 @@ class RunLLMTestCase(unittest.TestCase): self.addCleanup(tmp.cleanup) return Path(tmp.name) - def make_success_entry(self, custom_id: str, - text: str) -> mock.Mock: + def _make_success_entry(self, custom_id: str, + text: str) -> mock.Mock: """Create a mock succeeded batch result entry. :param custom_id: The custom ID of the entry. @@ -47,8 +47,8 @@ class RunLLMTestCase(unittest.TestCase): return entry @staticmethod - def make_error_entry(custom_id: str, - error_type: str) -> mock.Mock: + def _make_error_entry(custom_id: str, + error_type: str) -> mock.Mock: """Create a mock errored batch result entry. The error object is shaped as the SDK envelope: the outer @@ -95,7 +95,7 @@ class TestLoadItems(RunLLMTestCase): def setUp(self) -> None: """Create the prompt and input paths for a dry run.""" - directory: Path = self.make_temp_dir() + directory: Path = self._make_temp_dir() self.__prompt: Path = directory / "task.md" self.__prompt.write_text("The task.\n", encoding="utf-8") self.__input: Path = directory / "items.jsonl" @@ -171,7 +171,7 @@ class TestRequestBuilding(RunLLMTestCase): def setUp(self) -> None: """Create the prompt and input files for a dry run.""" - directory: Path = self.make_temp_dir() + directory: Path = self._make_temp_dir() self.__prompt: Path = directory / "task.md" self.__prompt.write_text( "the system prompt", encoding="utf-8") @@ -224,7 +224,7 @@ class TestMainFlow(RunLLMTestCase): def setUp(self) -> None: """Create a temporary directory with the input files.""" - directory: Path = self.make_temp_dir() + directory: Path = self._make_temp_dir() self.__runs: Path = directory / "runs" self.__archive_dir: Path = self.__runs / "task_v1" / "run1" self.__prompt: Path = directory / "task_v1.md" @@ -310,8 +310,8 @@ class TestMainFlow(RunLLMTestCase): the item order preserved and the token usage summed, the output text written unescaped.""" client: mock.Mock = self.__make_client( - [self.make_success_entry("a", "answer 中文 a"), - self.make_success_entry("b", "answer b")]) + [self._make_success_entry("a", "answer 中文 a"), + self._make_success_entry("b", "answer b")]) status: int stderr: str with mock.patch( @@ -356,8 +356,8 @@ class TestMainFlow(RunLLMTestCase): (run_dir / "stale.jsonl").write_text( "stale", encoding="utf-8") client: mock.Mock = self.__make_client( - [self.make_success_entry("a", "answer a"), - self.make_success_entry("b", "answer b")]) + [self._make_success_entry("a", "answer a"), + self._make_success_entry("b", "answer b")]) status: int = self.__run_main( self.__argv + ["--replace"], client)[0] self.assertEqual(status, 0) @@ -375,8 +375,8 @@ class TestMainFlow(RunLLMTestCase): (run2_dir / "output.jsonl").write_text( "stale run2 data", encoding="utf-8") client: mock.Mock = self.__make_client( - [self.make_success_entry("a", "answer a"), - self.make_success_entry("b", "answer b")]) + [self._make_success_entry("a", "answer a"), + self._make_success_entry("b", "answer b")]) argv: list[str] = [ str(self.__prompt), str(self.__input), str(run2_dir), "--replace"] @@ -393,8 +393,8 @@ class TestMainFlow(RunLLMTestCase): """Test that a failed item aborts with a non-zero status, the summed usage counting only the succeeded item.""" client: mock.Mock = self.__make_client( - [self.make_success_entry("a", "answer a"), - self.make_error_entry("b", "invalid_request_error")]) + [self._make_success_entry("a", "answer a"), + self._make_error_entry("b", "invalid_request_error")]) status: int stderr: str status, _, stderr = self.__run_main(self.__argv, client) @@ -417,7 +417,7 @@ class TestMainFlow(RunLLMTestCase): """Test that an item missing from the batch results is reported as a failed item.""" client: mock.Mock = self.__make_client( - [self.make_success_entry("a", "answer a")]) + [self._make_success_entry("a", "answer a")]) status: int stderr: str status, _, stderr = self.__run_main(self.__argv, client)