imacat/pop-fem-audit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Add the tally-annotations subcommand settling the pattern matrix

Browse Source 
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
imacat
2026-08-18 22:37:32 +08:00
co-authored by Claude Fable 5
parent 0db25812f2
commit 5a91f66402
8 changed files with 2005 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/README.rst
+6
View File
@@ -79,6 +79,12 @@ tally-groups
Settle the semantic code groups by a majority of the three group-selection runs' archives, and write the final group table, dropping any keyword outside the settled vocabulary. Check ``pop-fem-audit-tools tally-groups -h`` for complete instructions on its usage.
tally-annotations
-----------------
Settle the pattern annotations by a majority of the three annotation runs' archives. It extracts the patterns from the group synthesis archives, checks every ballot against the patterns applicable to that song's performer gender, and writes the settled pattern table and the settled song-by-pattern table. Check ``pop-fem-audit-tools tally-annotations -h`` for complete instructions on its usage.
Copyright
=========
tools/docs/source/pop_fem_audit_tools.commands.rst
+8
View File
@@ -52,6 +52,14 @@ pop\_fem\_audit\_tools.commands.run\_llm module
:show-inheritance:
:undoc-members:
pop\_fem\_audit\_tools.commands.tally\_annotations module
---------------------------------------------------------
.. automodule:: pop_fem_audit_tools.commands.tally_annotations
:members:
:show-inheritance:
:undoc-members:
pop\_fem\_audit\_tools.commands.tally\_codings module
-----------------------------------------------------
tools/src/pop_fem_audit_tools/__main__.py
+2
View File
@@ -22,6 +22,7 @@ from .commands import (
fetch_artists_command,
fetch_lyrics_command,
run_llm_command,
tally_annotations_command,
tally_codings_command,
tally_groups_command,
)
@@ -36,6 +37,7 @@ SUBCOMMANDS: dict[str, Callable[[list[str] | None], int]] = {
"fetch-artists": fetch_artists_command,
"fetch-lyrics": fetch_lyrics_command,
"run-llm": run_llm_command,
"tally-annotations": tally_annotations_command,
"tally-codings": tally_codings_command,
"tally-groups": tally_groups_command,
}
tools/src/pop_fem_audit_tools/commands/__init__.py
+26
View File
@@ -95,6 +95,32 @@ def run_llm_command(argv: list[str] | None = None) -> int:
return main(argv)
def tally_annotations_command(argv: list[str] | None = None) -> int:
"""Settle the step-5 pattern annotations by a majority of the
three annotation runs.
Writes the pattern table CSV file, holding the header row
``Pattern,Group,Name,Description`` and one row per pattern
extracted from the three gendered synthesis archives, and the
annotation table CSV file, holding the header row
``Song,Artist Credit,Pattern,Votes`` and one row per (song,
pattern) pair at least two of the three annotation runs'
cleaned ballots carry, the song named by its title and its
stored artist credit from the SQLite working store. Nothing
is written when a synthesis section yields an empty name or
description, a run record is malformed, a song does not
appear exactly three times in the pooled ballots, or a
settled song is not in the working store; the error message
names what failed.
:param argv: The command-line arguments, or None for
``sys.argv``.
:return: The exit status: 0 on success, non-zero on failure.
"""
from .tally_annotations import main
return main(argv)
def tally_codings_command(argv: list[str] | None = None) -> int:
"""Settle the coding by a majority of the three coding runs.
tools/src/pop_fem_audit_tools/commands/tally_annotations.py
+656
View File
@@ -0,0 +1,656 @@
# Tools for A Feminist Audit of Pop Music.
# Copyright 2026 imacat. All rights reserved.
# Authors:
# imacat@mail.imacat.idv.tw (imacat), 2026/8/15
# AI assistance: Claude Code (Anthropic)
"""The majority tally of the three step-5 annotation runs.
Settles the per-song pattern annotation step: the pattern
vocabulary comes from the three gendered synthesis archives (the
male-group, female-group, and mixed-group runs of step
5-03-synthesize), and the same annotation definition file is run
three times independently over every song to select which
patterns apply. This command extracts the pattern table from the
synthesis archives, tallies the three annotation runs' votes, and
writes the two final tables the paper cites: the pattern table
(the "patterns" CSV argument) and the per-song annotation table
(the "annotations" CSV argument).
Each synthesis archive's ``output.jsonl`` holds a single record
whose "text" field is a Markdown document, one section per
pattern, each section starting with a heading line. The pattern
name is the heading text with its leading numbering token (for
example ``模式一：`` or ``一、``) stripped; the pattern
description is the section's non-empty lines that are not example
quotes (a line starting with ``- ``), joined with a single space.
The male synthesis yields the pattern IDs ``M1``, ``M2``, ...; the
female synthesis ``F1``, ``F2``, ...; the mixed synthesis ``X1``,
``X2``, ..., every set numbered in the section order of its own
document.
Each annotation run's ``output.jsonl`` holds one record per song,
the ID ``song-<ID>`` and the "text" field a JSON array of the
pattern IDs the song was annotated with. The records of every
given run directory are pooled. A record whose "text" field is
missing, or does not parse to a JSON array of strings, is
skipped, a warning naming the run directory and song reported on
standard error; this lets a rescue archive of replacement ballots
be passed as an additional run directory when a run archive holds
a dead record. Every song must appear exactly three times in the
pool once such records are skipped. A pattern ID that is not one of
the extracted IDs, or whose gendered prefix does not apply to the
song's stored performer gender (male songs take ``M``/``X``,
female songs take ``F``/``X``, every other song takes all three),
is dropped from its ballot, as is a duplicate within the one
ballot; every drop is reported on standard error. A (song,
pattern) pair is settled when at least two of the three cleaned
ballots carry it, so three votes never tie.
The song's title and stored artist credit are looked up in the
given SQLite working store by the numeric song ID, so this
command runs after ``build-db``. Nothing is written when a
synthesis section yields an empty name or description, a run
record is malformed, a song does not appear exactly three times
in the pool, or a settled song is not in the working store; the
error message names what failed.
"""
import argparse
import csv
import json
import re
import sqlite3
import sys
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from ..utils import format_duration
MAJORITY: int = 2
"""The number of cleaned ballots that must carry a pattern for a
(song, pattern) pair to be settled."""
BALLOTS_PER_SONG: int = 3
"""The number of times a song must appear in the pooled ballots."""
SONG_ID_PREFIX: str = "song-"
"""The prefix every annotation record ID must carry; the numeric
song ID is the rest of the ID."""
PATTERNS_HEADER: tuple[str, str, str, str] = (
"Pattern", "Group", "Name", "Description")
"""The header row of the pattern table CSV file."""
ANNOTATIONS_HEADER: tuple[str, str, str, str] = (
"Song", "Artist Credit", "Pattern", "Votes")
"""The header row of the annotation table CSV file."""
_GROUP_PREFIXES: tuple[tuple[str, str], ...] = (
("male", "M"), ("female", "F"), ("mixed", "X"))
"""The synthesis group name and its pattern ID prefix, in the
order the three synthesis archives are given."""
_APPLICABLE_PREFIXES: dict[str, set[str]] = {
"male": {"M", "X"}, "female": {"F", "X"}}
"""The pattern ID prefixes applicable to a song, keyed by the
song's stored performer gender; a gender missing here (including
None) takes every prefix."""
_NUMBERING_RE: re.Pattern[str] = re.compile(
r"^(?:模式[一二三四五六七八九十]+|[一二三四五六七八九十]+)[：、]")
"""The leading numbering token of a pattern heading, stripped to
yield the pattern name."""
_HEADING_RE: re.Pattern[str] = re.compile(r"^#+\s*(.*)$")
"""A Markdown heading line, the heading text captured."""
class TallyError(Exception):
"""An error that fails the annotation tally."""
@dataclass(frozen=True)
class Pattern:
"""One extracted pattern of a gendered synthesis document."""
id: str
"""The pattern ID, the group's prefix and its 1-based section
number."""
group: str
"""The synthesis group the pattern came from: "male",
"female", or "mixed"."""
name: str
"""The pattern name, its numbering token stripped."""
description: str
"""The pattern description, its example quotes excluded."""
def parse_args(argv: list[str] | None) -> argparse.Namespace:
"""Parse the command-line arguments.
:param argv: The command-line arguments, or None for
``sys.argv``.
:return: The parsed arguments.
"""
parser: argparse.ArgumentParser = argparse.ArgumentParser(
description="Settle the step-5 pattern annotations by a"
" majority of the three annotation runs.")
parser.add_argument(
"male_synthesis", type=Path,
help="the male-group synthesis run's archive directory")
parser.add_argument(
"female_synthesis", type=Path,
help="the female-group synthesis run's archive directory")
parser.add_argument(
"mixed_synthesis", type=Path,
help="the mixed-group synthesis run's archive directory")
parser.add_argument(
"db_path", type=Path,
help="the SQLite working database")
parser.add_argument(
"patterns_csv", type=Path,
help="the output pattern table CSV file, by convention"
" results/patterns.csv")
parser.add_argument(
"annotations_csv", type=Path,
help="the output annotation table CSV file, by convention"
" results/annotations.csv")
parser.add_argument(
"run_dir", type=Path, nargs="+",
help="an annotation run's archive directory, one or more")
return parser.parse_args(argv)
def load_synthesis_text(synthesis_dir: Path) -> str:
"""Load the Markdown pattern document of one synthesis archive.
:param synthesis_dir: The synthesis run's archive directory,
containing ``output.jsonl``.
:return: The "text" field of the archive's single record.
:raises TallyError: When the file cannot be read, does not
hold exactly one JSON record, the record is not a
successful result, or its "text" is not a string.
"""
path: Path = synthesis_dir / "output.jsonl"
text: str
try:
text = path.read_text(encoding="utf-8")
except OSError as error:
raise TallyError(str(error)) from error
lines: list[str] = [x for x in text.split("\n")
if x.strip() != ""]
if len(lines) != 1:
raise TallyError(
f"{path}: expected exactly one record, found"
f" {len(lines)}")
record: Any
try:
record = json.loads(lines[0])
except json.JSONDecodeError as error:
raise TallyError(
f"{path}: malformed JSON: {error}") from error
if not isinstance(record, dict) or "error" in record \
or "text" not in record:
raise TallyError(f"{path}: not a successful result")
body: Any = record["text"]
if not isinstance(body, str):
raise TallyError(f"{path}: \"text\" is not a string")
return body
def extract_patterns(
synthesis_dir: Path, group: str, prefix: str) \
-> list[Pattern]:
"""Extract the numbered patterns of one synthesis document.
:param synthesis_dir: The synthesis run's archive directory.
:param group: The synthesis group name: "male", "female", or
"mixed".
:param prefix: The pattern ID prefix of the group.
:return: The patterns, in the document's section order, IDs
``<prefix>1``, ``<prefix>2``, ...
:raises TallyError: When the archive cannot be loaded, holds
no pattern section, or a section yields an empty name or
description.
"""
text: str = load_synthesis_text(synthesis_dir)
sections: list[tuple[str, str]] = _parse_sections(text)
if len(sections) == 0:
raise TallyError(
f"{synthesis_dir}: no pattern sections found")
patterns: list[Pattern] = []
index: int
name: str
description: str
for index, (name, description) in enumerate(sections, start=1):
if name == "" or description == "":
raise TallyError(
f"{synthesis_dir}: section {index}: empty name"
" or description")
patterns.append(Pattern(
id=f"{prefix}{index}", group=group, name=name,
description=description))
return patterns
def _parse_sections(text: str) -> list[tuple[str, str]]:
"""Split a pattern document into its numbered sections.
:param text: The synthesis document.
:return: The sections' (name, description) pairs, in document
order; the text before the first heading is discarded.
"""
lines: list[str] = text.split("\n")
headings: list[int] = [
i for i, x in enumerate(lines) if _HEADING_RE.match(x)]
sections: list[tuple[str, str]] = []
position: int
start: int
for position, start in enumerate(headings):
end: int = headings[position + 1] \
if position + 1 < len(headings) else len(lines)
heading: re.Match[str] | None = _HEADING_RE.match(
lines[start])
assert heading is not None
name: str = _NUMBERING_RE.sub(
"", heading.group(1).strip(), count=1).strip()
body: list[str] = [x.strip() for x in lines[start + 1:end]]
description: str = " ".join(
x for x in body if x != "" and not x.startswith("- "))
sections.append((name, description))
return sections
def load_ballots(run_dirs: list[Path]) -> dict[int, list[list[str]]]:
"""Load and pool the annotation ballots of the given runs.
A record whose "text" field is missing, or does not parse to
a JSON array of strings, is skipped, a warning naming the run
directory and song reported on standard error, as an
observable side effect.
:param run_dirs: The annotation runs' archive directories.
:return: The raw ballots (the selected pattern IDs, duplicates
not yet collapsed) of every song, keyed by the numeric
song ID, in the pooled record order.
:raises TallyError: When an ``output.jsonl`` cannot be read, a
line is not a well-formed record, or a song does not
appear exactly :data:`BALLOTS_PER_SONG` times in the pool
once malformed-"text" records are skipped.
"""
pooled: dict[int, list[list[str]]] = {}
run_dir: Path
for run_dir in run_dirs:
path: Path = run_dir / "output.jsonl"
text: str
try:
text = path.read_text(encoding="utf-8")
except OSError as error:
raise TallyError(str(error)) from error
line: str
for line in text.split("\n"):
if line.strip() == "":
continue
record: Any
try:
record = json.loads(line)
except json.JSONDecodeError as error:
raise TallyError(
f"{path}: malformed JSON: {error}") from error
if not isinstance(record, dict) or "id" not in record:
raise TallyError(
f"{path}: record without \"id\": {line}")
song_id: int = _parse_song_id(record["id"], run_dir)
if "text" not in record:
print(
f"warning: {run_dir}: song-{song_id}: no"
" \"text\" field, skipped", file=sys.stderr)
continue
ballot: list[str] | None = _parse_ballot(
record["text"], run_dir, song_id)
if ballot is None:
continue
pooled.setdefault(song_id, []).append(ballot)
song_id: int
ballots: list[list[str]]
for song_id, ballots in pooled.items():
if len(ballots) != BALLOTS_PER_SONG:
raise TallyError(
f"song-{song_id}: appears {len(ballots)} times"
f" in the pool, expected {BALLOTS_PER_SONG}")
return pooled
def _parse_song_id(item_id: Any, run_dir: Path) -> int:
"""Parse the numeric song ID out of an annotation record ID.
:param item_id: The record's "id" field.
:param run_dir: The run directory the record came from, for
the error message.
:return: The parsed numeric song ID.
:raises TallyError: When the ID is not in the ``song-<ID>``
form.
"""
if not isinstance(item_id, str) \
or not item_id.startswith(SONG_ID_PREFIX) \
or not item_id[len(SONG_ID_PREFIX):].isdigit():
raise TallyError(
f"{run_dir}: id \"{item_id}\": not in"
f" \"{SONG_ID_PREFIX}<ID>\" form")
return int(item_id[len(SONG_ID_PREFIX):])
def _parse_ballot(text: Any, run_dir: Path, song_id: int) \
-> list[str] | None:
"""Parse and validate one song's ballot "text" field.
A "text" field that is not a string, is not well-formed JSON,
or does not parse to a JSON array of strings is reported on
standard error as a warning naming the run directory and
song, and the record is skipped.
:param text: The record's "text" field.
:param run_dir: The run directory the record came from, for
the warning message.
:param song_id: The numeric song ID, for the warning message.
:return: The selected pattern IDs, in the given order,
duplicates not collapsed; None when the "text" field is
malformed.
"""
if not isinstance(text, str):
print(
f"warning: {run_dir}: song-{song_id}: \"text\" is not"
" a string, skipped", file=sys.stderr)
return None
selected: Any
try:
selected = json.loads(text)
except json.JSONDecodeError:
print(
f"warning: {run_dir}: song-{song_id}: \"text\" is"
" malformed JSON, skipped", file=sys.stderr)
return None
if not isinstance(selected, list) \
or not all(isinstance(x, str) for x in selected):
print(
f"warning: {run_dir}: song-{song_id}: \"text\" does"
" not parse to a JSON array of strings, skipped",
file=sys.stderr)
return None
return selected
def load_genders(db_path: Path) -> dict[int, str | None]:
"""Load the stored performer gender of every song.
:param db_path: The SQLite working database.
:return: The stored performer gender, keyed by the song ID.
:raises TallyError: When the database cannot be read.
"""
songs: dict[int, tuple[str, str, str | None]] \
= _load_songs(db_path)
return {x: y[2] for x, y in songs.items()}
def _load_songs(db_path: Path) \
-> dict[int, tuple[str, str, str | None]]:
"""Load the title, artist credit, and gender of every song.
:param db_path: The SQLite working database.
:return: The title, the stored artist credit, and the stored
performer gender of every song, keyed by the song ID.
:raises TallyError: When the database cannot be read.
"""
try:
connection: sqlite3.Connection = sqlite3.connect(
f"file:{db_path.resolve()}?mode=ro", uri=True)
except sqlite3.Error as error:
raise TallyError(str(error)) from error
try:
rows: Any = connection.execute(
"SELECT id, title, artist_credit, performer_gender"
" FROM songs")
return {x[0]: (x[1], x[2], x[3]) for x in rows}
except sqlite3.Error as error:
raise TallyError(str(error)) from error
finally:
connection.close()
def clean_ballots(
pooled: dict[int, list[list[str]]],
pattern_ids: set[str],
genders: dict[int, str | None]) \
-> tuple[dict[int, list[set[str]]], int]:
"""Drop the out-of-scope and duplicate items of every ballot.
Every dropped occurrence is reported on standard error as an
observable side effect.
:param pooled: The raw ballots of every song, keyed by the
numeric song ID.
:param pattern_ids: The extracted pattern IDs.
:param genders: The stored performer gender of every song
known to the working store, keyed by the song ID; a song
missing here takes every pattern ID prefix.
:return: The cleaned ballots (the applicable, deduplicated
pattern IDs) of every song, keyed by the numeric song ID,
and the total number of dropped occurrences.
"""
cleaned: dict[int, list[set[str]]] = {}
dropped: int = 0
song_id: int
ballots: list[list[str]]
for song_id, ballots in pooled.items():
applicable: set[str] = _APPLICABLE_PREFIXES.get(
genders.get(song_id), {"M", "F", "X"})
cleaned_ballots: list[set[str]] = []
ballot: list[str]
for ballot in ballots:
kept: set[str]
count: int
kept, count = _clean_one_ballot(
ballot, song_id, pattern_ids, applicable)
cleaned_ballots.append(kept)
dropped += count
cleaned[song_id] = cleaned_ballots
return cleaned, dropped
def _clean_one_ballot(
ballot: list[str], song_id: int, pattern_ids: set[str],
applicable: set[str]) -> tuple[set[str], int]:
"""Drop the out-of-scope and duplicate items of one ballot.
:param ballot: The raw selected pattern IDs, in the given
order.
:param song_id: The numeric song ID, for the warning messages.
:param pattern_ids: The extracted pattern IDs.
:param applicable: The pattern ID prefixes applicable to the
song.
:return: The applicable, deduplicated pattern IDs, and the
number of dropped occurrences.
"""
kept: set[str] = set()
dropped: int = 0
pattern_id: str
for pattern_id in ballot:
if pattern_id in kept:
print(
f"warning: song-{song_id}: dropped duplicate"
f" ballot item \"{pattern_id}\"", file=sys.stderr)
dropped += 1
continue
if pattern_id not in pattern_ids \
or pattern_id[:1] not in applicable:
print(
f"warning: song-{song_id}: dropped out-of-scope"
f" ballot item \"{pattern_id}\"", file=sys.stderr)
dropped += 1
continue
kept.add(pattern_id)
return kept, dropped
def tally_votes(cleaned: dict[int, list[set[str]]]) \
-> dict[int, dict[str, int]]:
"""Tally the pattern votes of the cleaned ballots, song by song.
:param cleaned: The cleaned ballots of every song, keyed by
the numeric song ID.
:return: The settled pattern votes of every song (at least
:data:`MAJORITY` of its cleaned ballots), keyed by the
numeric song ID and then by the pattern ID; a song with no
settled pattern maps to an empty mapping.
"""
tallied: dict[int, dict[str, int]] = {}
song_id: int
ballots: list[set[str]]
for song_id, ballots in cleaned.items():
counts: dict[str, int] = {}
ballot: set[str]
for ballot in ballots:
pattern_id: str
for pattern_id in ballot:
counts[pattern_id] = counts.get(pattern_id, 0) + 1
tallied[song_id] = {
x: y for x, y in counts.items() if y >= MAJORITY}
return tallied
def write_patterns_csv(
output_csv: Path, patterns: list[Pattern]) -> None:
"""Write the pattern table CSV file.
Writes an RFC 4180 CSV file, UTF-8, with CRLF line endings,
carrying the header row ``Pattern,Group,Name,Description`` and
one row per extracted pattern, in the given order. The parent
directory is created when it does not exist.
:param output_csv: The output CSV file.
:param patterns: The extracted patterns, in the output order.
:return: None.
:raises OSError: When the file cannot be written.
"""
output_csv.parent.mkdir(parents=True, exist_ok=True)
with open(output_csv, "w", encoding="utf-8",
newline="") as file:
writer: Any = csv.writer(file)
writer.writerow(PATTERNS_HEADER)
writer.writerows(
(x.id, x.group, x.name, x.description)
for x in patterns)
def build_annotation_rows(
tallied: dict[int, dict[str, int]],
songs: dict[int, tuple[str, str, str | None]],
patterns: list[Pattern]) \
-> list[tuple[str, str, str, int]]:
"""Build the ordered data rows of the annotation table.
:param tallied: The settled pattern votes of every song, keyed
by the numeric song ID and then by the pattern ID.
:param songs: The title, the stored artist credit, and the
stored performer gender of every stored song, keyed by the
song ID.
:param patterns: The extracted patterns, in the pattern table
order.
:return: The rows, each the song title, the artist credit, the
pattern ID, and the number of votes, ordered by the
numeric song ID and then by the pattern table order.
:raises TallyError: When a settled song is not in the working
store.
"""
order: dict[str, int] = {x.id: i for i, x in enumerate(patterns)}
rows: list[tuple[str, str, str, int]] = []
song_id: int
for song_id in sorted(tallied):
votes: dict[str, int] = tallied[song_id]
if len(votes) == 0:
continue
if song_id not in songs:
raise TallyError(
f"song-{song_id}: not in the working store")
title: str
artist_credit: str
title, artist_credit, _ = songs[song_id]
pattern_id: str
for pattern_id in sorted(votes, key=lambda x: order[x]):
rows.append((
title, artist_credit, pattern_id,
votes[pattern_id]))
return rows
def write_annotations_csv(
output_csv: Path, rows: list[tuple[str, str, str, int]]) \
-> None:
"""Write the annotation table CSV file.
Writes an RFC 4180 CSV file, UTF-8, with CRLF line endings,
carrying the header row ``Song,Artist Credit,Pattern,Votes``
and one row per settled (song, pattern) pair, in the given
order. The parent directory is created when it does not
exist.
:param output_csv: The output CSV file.
:param rows: The settled rows, in the output order.
:return: None.
:raises OSError: When the file cannot be written.
"""
output_csv.parent.mkdir(parents=True, exist_ok=True)
with open(output_csv, "w", encoding="utf-8",
newline="") as file:
writer: Any = csv.writer(file)
writer.writerow(ANNOTATIONS_HEADER)
writer.writerows(rows)
def main(argv: list[str] | None = None) -> int:
"""Settle the pattern annotations by a majority of the runs.
Writes the pattern table CSV file and the annotation table CSV
file described in the module docstring. Nothing is written
when a synthesis section yields an empty name or description,
a run record is malformed, a song does not appear exactly
:data:`BALLOTS_PER_SONG` times in the pool, or a settled song
is not in the working store; the error message names what
failed.
:param argv: The command-line arguments, or None for
``sys.argv``.
:return: The exit status: 0 on success, non-zero on failure.
"""
started: float = time.monotonic()
args: argparse.Namespace = parse_args(argv)
try:
patterns: list[Pattern] = []
synthesis_dirs: tuple[Path, Path, Path] = (
args.male_synthesis, args.female_synthesis,
args.mixed_synthesis)
synthesis_dir: Path
group: str
prefix: str
for synthesis_dir, (group, prefix) in zip(
synthesis_dirs, _GROUP_PREFIXES):
patterns.extend(
extract_patterns(synthesis_dir, group, prefix))
write_patterns_csv(args.patterns_csv, patterns)
pattern_ids: set[str] = {x.id for x in patterns}
pooled: dict[int, list[list[str]]] \
= load_ballots(args.run_dir)
songs: dict[int, tuple[str, str, str | None]] \
= _load_songs(args.db_path)
genders: dict[int, str | None] \
= {x: y[2] for x, y in songs.items()}
cleaned: dict[int, list[set[str]]]
dropped: int
cleaned, dropped = clean_ballots(pooled, pattern_ids,
genders)
tallied: dict[int, dict[str, int]] = tally_votes(cleaned)
rows: list[tuple[str, str, str, int]] \
= build_annotation_rows(tallied, songs, patterns)
write_annotations_csv(args.annotations_csv, rows)
except (TallyError, OSError) as error:
print(f"error: {error}", file=sys.stderr)
return 1
elapsed: str = format_duration(time.monotonic() - started)
print(
f"Done. Tallied {len(rows)} settled pairs across"
f" {len(tallied)} songs, {dropped} votes dropped."
f" {elapsed} elapsed.", file=sys.stderr)
return 0
tools/tests/test_tally_annotations.py
+525
View File
@@ -0,0 +1,525 @@
# Tools for A Feminist Audit of Pop Music.
# Copyright 2026 imacat. All rights reserved.
# Authors:
# imacat@mail.imacat.idv.tw (imacat), 2026/8/15
# AI assistance: Claude Code (Anthropic)
"""Unit tests for the step-5 annotation tally module."""
import csv
import io
import json
import tempfile
import unittest
from contextlib import redirect_stderr
from pathlib import Path
import sqlalchemy as sa
from sqlalchemy.orm import Session
from pop_fem_audit_tools.commands import tally_annotations
from pop_fem_audit_tools.database import Base
from pop_fem_audit_tools.models import Song
class TestTallyAnnotations(unittest.TestCase):
"""Test cases for the step-5 pattern annotation tally."""
def setUp(self) -> None:
"""Create the archive directories, the database, and the
output paths."""
tmp: tempfile.TemporaryDirectory[str] \
= tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.__dir: Path = Path(tmp.name)
self.__male_synthesis: Path = self.__dir / "male-synthesis"
self.__female_synthesis: Path \
= self.__dir / "female-synthesis"
self.__mixed_synthesis: Path = self.__dir / "mixed-synthesis"
self.__male_synthesis.mkdir()
self.__female_synthesis.mkdir()
self.__mixed_synthesis.mkdir()
self.__db_path: Path = self.__dir / "working.sqlite3"
self.__patterns_csv: Path \
= self.__dir / "results" / "patterns.csv"
self.__annotations_csv: Path \
= self.__dir / "results" / "annotations.csv"
self.__runs: list[Path] = []
number: int
for number in (1, 2, 3):
run_dir: Path = self.__dir / f"run{number}"
run_dir.mkdir()
self.__runs.append(run_dir)
def __write_default_synthesis_archives(self) -> None:
"""Write the male, female, and mixed synthesis archives.
The male archive holds two patterns (``M1``, ``M2``), and
the female and mixed archives hold one pattern each
(``F1``, ``X1``).
:return: None.
"""
self.__write_synthesis(self.__male_synthesis, [
("模式一：厭女語彙的常態化", ["描述一。"], ["「引文一」"]),
("模式二：陰陽權力階序", ["描述二。"], ["「引文二」"])])
self.__write_synthesis(self.__female_synthesis, [
("一、女女敵對", ["描述三。"], ["「引文三」"])])
self.__write_synthesis(self.__mixed_synthesis, [
("模式一：佔有語法", ["描述四。"], ["「引文四」"])])
@staticmethod
def __write_synthesis(
synthesis_dir: Path,
sections: list[tuple[str, list[str], list[str]]]) \
-> None:
"""Write one synthesis archive's ``output.jsonl``.
:param synthesis_dir: The synthesis run's archive
directory.
:param sections: Each pattern's heading text, description
lines, and example quote lines (written with the
``- `` prefix).
:return: None.
"""
parts: list[str] = []
heading: str
body: list[str]
quotes: list[str]
for heading, body, quotes in sections:
parts.append(f"## {heading}")
parts.append("")
parts.extend(body)
parts.append("")
parts.extend(f"- {x}" for x in quotes)
parts.append("")
text: str = "\n".join(parts)
record: dict[str, str] = {
"id": "synthesis", "text": text,
"stop_reason": "end_turn"}
(synthesis_dir / "output.jsonl").write_text(
json.dumps(record, ensure_ascii=False) + "\n",
encoding="utf-8")
def __seed_songs(
self,
songs: list[tuple[int, str, str, str | None]]) -> None:
"""Create the working store schema and the fixture songs.
:param songs: The song ID, title, artist credit, and
stored performer gender of every fixture song.
:return: None.
"""
engine: sa.Engine = sa.create_engine(
f"sqlite:///{self.__db_path}")
Base.metadata.create_all(engine)
session: Session
with Session(engine) as session:
song_id: int
title: str
artist_credit: str
gender: str | None
for song_id, title, artist_credit, gender in songs:
session.add(Song(
id=song_id, title=title,
artist_credit=artist_credit,
performer_gender=gender))
session.commit()
engine.dispose()
def __write_run(
self, run_dir: Path,
ballots: dict[int, list[str]]) -> None:
"""Write one annotation run's ``output.jsonl``.
:param run_dir: The run's archive directory.
:param ballots: The selected pattern IDs of every song,
keyed by the numeric song ID.
:return: None.
"""
lines: list[str] = [
json.dumps({
"id": f"song-{song_id}",
"text": json.dumps(pattern_ids, ensure_ascii=False),
"stop_reason": "end_turn"}, ensure_ascii=False)
for song_id, pattern_ids in ballots.items()]
(run_dir / "output.jsonl").write_text(
"\n".join(lines) + "\n", encoding="utf-8")
def __write_same_ballots_to_all_runs(
self, ballots: dict[int, list[str]]) -> None:
"""Write the same ballots to all three run directories.
:param ballots: The selected pattern IDs of every song,
keyed by the numeric song ID.
:return: None.
"""
run_dir: Path
for run_dir in self.__runs:
self.__write_run(run_dir, ballots)
def __run_tally(self) -> tuple[int, str]:
"""Run the tally command against the fixture archives.
:return: The exit status and the standard error text.
"""
stderr: io.StringIO = io.StringIO()
status: int
with redirect_stderr(stderr):
status = tally_annotations.main([
str(self.__male_synthesis),
str(self.__female_synthesis),
str(self.__mixed_synthesis), str(self.__db_path),
str(self.__patterns_csv),
str(self.__annotations_csv)]
+ [str(x) for x in self.__runs])
return status, stderr.getvalue()
@staticmethod
def __read_rows(path: Path) -> list[list[str]]:
"""Read a written CSV file back as rows.
:param path: The CSV file.
:return: The rows, the header row included.
"""
with open(path, encoding="utf-8", newline="") as file:
return list(csv.reader(file))
def test_patterns_extracted_in_group_and_section_order(
self) -> None:
"""Test that the pattern table lists the male, then
female, then mixed patterns, in section order, the
numbering token stripped and the example quotes
excluded from the description."""
self.__write_default_synthesis_archives()
self.__seed_songs([(1, "Song A", "Artist A", "male")])
self.__write_same_ballots_to_all_runs({1: []})
status: int
status, _ = self.__run_tally()
self.assertEqual(status, 0)
self.assertEqual(self.__read_rows(self.__patterns_csv), [
["Pattern", "Group", "Name", "Description"],
["M1", "male", "厭女語彙的常態化", "描述一。"],
["M2", "male", "陰陽權力階序", "描述二。"],
["F1", "female", "女女敵對", "描述三。"],
["X1", "mixed", "佔有語法", "描述四。"]])
def test_empty_pattern_name_fails(self) -> None:
"""Test that a heading that is only a numbering token
fails the tally, nothing written."""
self.__write_synthesis(self.__male_synthesis, [
("模式一：", ["描述。"], [])])
self.__write_synthesis(self.__female_synthesis, [
("一、女女敵對", ["描述。"], [])])
self.__write_synthesis(self.__mixed_synthesis, [
("模式一：佔有語法", ["描述。"], [])])
self.__seed_songs([(1, "Song A", "Artist A", "male")])
self.__write_same_ballots_to_all_runs({1: []})
status: int
stderr: str
status, stderr = self.__run_tally()
self.assertNotEqual(status, 0)
self.assertIn("empty name or description", stderr)
self.assertFalse(self.__patterns_csv.exists())
def test_empty_pattern_description_fails(self) -> None:
"""Test that a section with no description line (only
example quotes) fails the tally, nothing written."""
self.__write_synthesis(self.__male_synthesis, [
("模式一：厭女語彙", [], ["「引文」"])])
self.__write_synthesis(self.__female_synthesis, [
("一、女女敵對", ["描述。"], [])])
self.__write_synthesis(self.__mixed_synthesis, [
("模式一：佔有語法", ["描述。"], [])])
self.__seed_songs([(1, "Song A", "Artist A", "male")])
self.__write_same_ballots_to_all_runs({1: []})
status: int
stderr: str
status, stderr = self.__run_tally()
self.assertNotEqual(status, 0)
self.assertIn("empty name or description", stderr)
self.assertFalse(self.__patterns_csv.exists())
def test_song_not_appearing_three_times_fails(self) -> None:
"""Test that a song missing from one run's ballots fails
the tally."""
self.__write_default_synthesis_archives()
self.__seed_songs([(1, "Song A", "Artist A", "male")])
self.__write_run(self.__runs[0], {1: ["M1"]})
self.__write_run(self.__runs[1], {1: ["M1"]})
self.__write_run(self.__runs[2], {})
status: int
stderr: str
status, stderr = self.__run_tally()
self.assertNotEqual(status, 0)
self.assertIn("song-1", stderr)
self.assertIn("expected", stderr)
self.assertFalse(self.__annotations_csv.exists())
def test_ballot_missing_text_field_skipped_then_fails(
self) -> None:
"""Test that a ballot record without "text" is skipped
with a warning, leaving the song short of the three
ballots the completeness check requires."""
self.__write_default_synthesis_archives()
self.__seed_songs([(1, "Song A", "Artist A", "male")])
self.__write_same_ballots_to_all_runs({1: ["M1"]})
(self.__runs[0] / "output.jsonl").write_text(
json.dumps({"id": "song-1"}) + "\n", encoding="utf-8")
status: int
stderr: str
status, stderr = self.__run_tally()
self.assertNotEqual(status, 0)
self.assertIn("warning:", stderr)
self.assertIn(str(self.__runs[0]), stderr)
self.assertIn("no \"text\" field", stderr)
self.assertIn("song-1", stderr)
self.assertIn("expected", stderr)
def test_ballot_text_not_array_of_strings_skipped_then_fails(
self) -> None:
"""Test that a "text" that is not a JSON array of strings
is skipped with a warning, leaving the song short of the
three ballots the completeness check requires."""
self.__write_default_synthesis_archives()
self.__seed_songs([(1, "Song A", "Artist A", "male")])
self.__write_same_ballots_to_all_runs({1: ["M1"]})
(self.__runs[0] / "output.jsonl").write_text(
json.dumps({"id": "song-1",
"text": json.dumps({"M1": []})}) + "\n",
encoding="utf-8")
status: int
stderr: str
status, stderr = self.__run_tally()
self.assertNotEqual(status, 0)
self.assertIn("warning:", stderr)
self.assertIn(str(self.__runs[0]), stderr)
self.assertIn("JSON array of strings", stderr)
self.assertIn("song-1", stderr)
self.assertIn("expected", stderr)
def test_dead_record_rescued_from_extra_run_dir(self) -> None:
"""Test that a dead record (malformed "text") in one run
directory is skipped with a warning, and its ballot is
rescued from an extra run directory, tallying
successfully."""
self.__write_default_synthesis_archives()
self.__seed_songs([(1, "Song A", "Artist A", "male")])
self.__write_same_ballots_to_all_runs({1: ["M1"]})
(self.__runs[0] / "output.jsonl").write_text(
json.dumps({"id": "song-1",
"text": json.dumps({"M1": []})}) + "\n",
encoding="utf-8")
rescue_dir: Path = self.__dir / "rescue"
rescue_dir.mkdir()
self.__write_run(rescue_dir, {1: ["M1"]})
stderr: io.StringIO = io.StringIO()
status: int
with redirect_stderr(stderr):
status = tally_annotations.main([
str(self.__male_synthesis),
str(self.__female_synthesis),
str(self.__mixed_synthesis), str(self.__db_path),
str(self.__patterns_csv),
str(self.__annotations_csv)]
+ [str(x) for x in self.__runs]
+ [str(rescue_dir)])
self.assertEqual(status, 0)
self.assertIn("warning:", stderr.getvalue())
self.assertIn("JSON array of strings", stderr.getvalue())
self.assertEqual(self.__read_rows(self.__annotations_csv), [
["Song", "Artist Credit", "Pattern", "Votes"],
["Song A", "Artist A", "M1", "3"]])
def test_unknown_pattern_id_dropped_with_warning(self) -> None:
"""Test that a selected ID outside the extracted patterns
is dropped, the occurrence reported on standard error."""
self.__write_default_synthesis_archives()
self.__seed_songs([(1, "Song A", "Artist A", "male")])
self.__write_same_ballots_to_all_runs(
{1: ["M1", "hallucinated"]})
status: int
stderr: str
status, stderr = self.__run_tally()
self.assertEqual(status, 0)
self.assertEqual(self.__read_rows(self.__annotations_csv), [
["Song", "Artist Credit", "Pattern", "Votes"],
["Song A", "Artist A", "M1", "3"]])
self.assertEqual(
stderr.count("dropped out-of-scope ballot item"
" \"hallucinated\""), 3)
def test_out_of_scope_pattern_dropped_with_warning(
self) -> None:
"""Test that a pattern outside the song's gendered scope
is dropped, the occurrence reported on standard error."""
self.__write_default_synthesis_archives()
self.__seed_songs([(1, "Song A", "Artist A", "male")])
self.__write_same_ballots_to_all_runs({1: ["M1", "F1"]})
status: int
stderr: str
status, stderr = self.__run_tally()
self.assertEqual(status, 0)
self.assertEqual(self.__read_rows(self.__annotations_csv), [
["Song", "Artist Credit", "Pattern", "Votes"],
["Song A", "Artist A", "M1", "3"]])
self.assertEqual(
stderr.count(
"dropped out-of-scope ballot item \"F1\""), 3)
def test_mixed_pattern_applies_to_every_song(self) -> None:
"""Test that a mixed-group pattern ("X" prefix) is in
scope for a male-credited song."""
self.__write_default_synthesis_archives()
self.__seed_songs([(1, "Song A", "Artist A", "male")])
self.__write_same_ballots_to_all_runs({1: ["X1"]})
status: int
stderr: str
status, stderr = self.__run_tally()
self.assertEqual(status, 0)
self.assertEqual(self.__read_rows(self.__annotations_csv), [
["Song", "Artist Credit", "Pattern", "Votes"],
["Song A", "Artist A", "X1", "3"]])
def test_unknown_gender_takes_every_pattern_prefix(
self) -> None:
"""Test that a song with no stored performer gender
accepts patterns of every group."""
self.__write_default_synthesis_archives()
self.__seed_songs([(1, "Song A", "Artist A", None)])
self.__write_same_ballots_to_all_runs({1: ["M1", "F1"]})
status: int
status, _ = self.__run_tally()
self.assertEqual(status, 0)
self.assertEqual(self.__read_rows(self.__annotations_csv), [
["Song", "Artist Credit", "Pattern", "Votes"],
["Song A", "Artist A", "M1", "3"],
["Song A", "Artist A", "F1", "3"]])
def test_duplicate_ballot_item_dropped_with_warning(
self) -> None:
"""Test that a pattern ID listed twice in one ballot is
collapsed, the extra occurrence reported on standard
error."""
self.__write_default_synthesis_archives()
self.__seed_songs([(1, "Song A", "Artist A", "male")])
self.__write_run(self.__runs[0], {1: ["M1", "M1"]})
self.__write_run(self.__runs[1], {1: []})
self.__write_run(self.__runs[2], {1: []})
status: int
stderr: str
status, stderr = self.__run_tally()
self.assertEqual(status, 0)
self.assertEqual(self.__read_rows(self.__annotations_csv),
[["Song", "Artist Credit", "Pattern",
"Votes"]])
self.assertEqual(
stderr.count(
"dropped duplicate ballot item \"M1\""), 1)
def test_majority_vote_settles_two_of_three(self) -> None:
"""Test that a (song, pattern) pair needs at least two of
the three cleaned ballots to settle."""
self.__write_default_synthesis_archives()
self.__seed_songs([(1, "Song A", "Artist A", "male")])
self.__write_run(self.__runs[0], {1: ["M1", "M2"]})
self.__write_run(self.__runs[1], {1: ["M1"]})
self.__write_run(self.__runs[2], {1: []})
status: int
status, _ = self.__run_tally()
self.assertEqual(status, 0)
self.assertEqual(self.__read_rows(self.__annotations_csv), [
["Song", "Artist Credit", "Pattern", "Votes"],
["Song A", "Artist A", "M1", "2"]])
def test_annotations_sorted_by_song_then_pattern_order(
self) -> None:
"""Test the row order: numeric song ID, then pattern in
the male-then-female-then-mixed extraction order."""
self.__write_default_synthesis_archives()
self.__seed_songs([
(2, "Song B", "Artist B", "female"),
(10, "Song C", "Artist C", None)])
self.__write_same_ballots_to_all_runs({
10: ["X1", "M1"], 2: ["F1"]})
status: int
status, _ = self.__run_tally()
self.assertEqual(status, 0)
self.assertEqual(self.__read_rows(self.__annotations_csv), [
["Song", "Artist Credit", "Pattern", "Votes"],
["Song B", "Artist B", "F1", "3"],
["Song C", "Artist C", "M1", "3"],
["Song C", "Artist C", "X1", "3"]])
def test_output_is_crlf_with_header(self) -> None:
"""Test the written bytes of both tables: RFC 4180, CRLF,
header row."""
self.__write_synthesis(self.__male_synthesis, [
("模式一：厭女語彙", ["描述一。"], [])])
self.__write_synthesis(self.__female_synthesis, [
("一、女女敵對", ["描述二。"], [])])
self.__write_synthesis(self.__mixed_synthesis, [
("模式一：佔有語法", ["描述三。"], [])])
self.__seed_songs([(1, "Song A", "Artist A", "male")])
self.__write_same_ballots_to_all_runs({1: ["M1"]})
status: int
status, _ = self.__run_tally()
self.assertEqual(status, 0)
self.assertEqual(
self.__patterns_csv.read_bytes(),
(
"Pattern,Group,Name,Description\r\n"
"M1,male,厭女語彙,描述一。\r\n"
"F1,female,女女敵對,描述二。\r\n"
"X1,mixed,佔有語法,描述三。\r\n"
).encode("utf-8"))
self.assertEqual(
self.__annotations_csv.read_bytes(),
b"Song,Artist Credit,Pattern,Votes\r\n"
b"Song A,Artist A,M1,3\r\n")
def test_song_missing_from_working_store_fails(self) -> None:
"""Test that a settled song not stored in the working
store fails the tally, nothing written."""
self.__write_default_synthesis_archives()
self.__seed_songs([(2, "Song B", "Artist B", "male")])
self.__write_same_ballots_to_all_runs({1: ["M1"]})
status: int
stderr: str
status, stderr = self.__run_tally()
self.assertNotEqual(status, 0)
self.assertIn("song-1", stderr)
self.assertIn("not in the working store", stderr)
self.assertFalse(self.__annotations_csv.exists())
def test_no_pattern_sections_fails(self) -> None:
"""Test that a synthesis document with no heading fails
the tally, nothing written."""
(self.__male_synthesis / "output.jsonl").write_text(
json.dumps({"id": "synthesis", "text": "no headings"})
+ "\n", encoding="utf-8")
self.__write_synthesis(self.__female_synthesis, [
("一、女女敵對", ["描述。"], [])])
self.__write_synthesis(self.__mixed_synthesis, [
("模式一：佔有語法", ["描述。"], [])])
self.__seed_songs([(1, "Song A", "Artist A", "male")])
self.__write_same_ballots_to_all_runs({1: []})
status: int
stderr: str
status, stderr = self.__run_tally()
self.assertNotEqual(status, 0)
self.assertIn("no pattern sections", stderr)
self.assertFalse(self.__patterns_csv.exists())
def test_summary_line_reports_counts(self) -> None:
"""Test that the closing summary reports the settled
pair, song, and dropped vote counts."""
self.__write_default_synthesis_archives()
self.__seed_songs([(1, "Song A", "Artist A", "male")])
self.__write_same_ballots_to_all_runs(
{1: ["M1", "hallucinated"]})
status: int
stderr: str
status, stderr = self.__run_tally()
self.assertEqual(status, 0)
self.assertIn("Tallied 1 settled pairs across 1 songs",
stderr)
self.assertIn("3 votes dropped", stderr)