From 1714257c89c44436725d6c3bb8aafaf1068ac900 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BE=9D=E7=91=AA=E8=B2=93?= Date: Thu, 6 Aug 2026 09:06:23 +0800 Subject: [PATCH] Add the compare-codings subcommand for the arbitration handoff Co-Authored-By: Claude Opus 5 (1M context) --- docs/decision-log.md | 9 + docs/methodology.md | 3 +- tools/README.rst | 6 + .../source/pop_fem_audit_tools.commands.rst | 8 + tools/src/pop_fem_audit_tools/__main__.py | 2 + .../pop_fem_audit_tools/commands/__init__.py | 1 + .../commands/compare_codings.py | 280 ++++++++++++ tools/tests/test_compare_codings.py | 398 ++++++++++++++++++ 8 files changed, 705 insertions(+), 2 deletions(-) create mode 100644 tools/src/pop_fem_audit_tools/commands/compare_codings.py create mode 100644 tools/tests/test_compare_codings.py diff --git a/docs/decision-log.md b/docs/decision-log.md index 59583ed..a040e67 100644 --- a/docs/decision-log.md +++ b/docs/decision-log.md @@ -495,6 +495,15 @@ 代價是少數組名不副實，依前條據實報告。實驗歸檔不入 repo，費用 $0.09（sonnet）與 $0.43（fable）記入 `run-costs.md`。 +- **定案編碼表只存歌與碼，不附引述與來源層**：引述已完整 + 存在於三份執行歸檔（3-1 兩次、3-2 仲裁），定案表再抄 + 一份即庫內重複。且同一個碼在三處的引述可能不同——兩次 + 執行各自的引述、仲裁者重新判讀後自己的引述——抄一份 + 等於替後續分析先做了選擇，並抹掉「兩次執行是否抓住 + 同一句」這個可分析的不穩定性。另一考量：`results/` 是 + 論文引用的目錄，一萬四千餘行引述抄進去，會把步驟 3-1 + 才從 64% 降到 23% 的歌詞重製量堆回去。來源層（共識或 + 仲裁）同理由三份歸檔可導出，一併不存。 - **刪除 `source-provenance.csv`，`cluster-keywords` 的產物 由六份減為五份**：該檔是兩份標註執行歸檔 `output.jsonl` 的攤平視圖，14,035 列 Keyword,Run,Song，未帶入任何庫裏 diff --git a/docs/methodology.md b/docs/methodology.md index a559a51..9b4356c 100644 --- a/docs/methodology.md +++ b/docs/methodology.md @@ -148,8 +148,7 @@ LLM。設計原則見 `research-plan.md`；本檔記載可重現的 鍵按字典序。 - **步驟 3 定案**：每首歌的最終標籤為共識標籤加上仲裁 保留的標籤，寫入逐首紀錄檔（歌依 ID 升序、標籤按 - 字典序，各標籤附其定案時的引述與來源層——共識或 - 仲裁）。 + 字典序）。 - **序列化通則**：所有中間檔為 UTF-8，欄序、鍵序與元素 序皆依上列規則明定，無時間戳、無隨機成分；JSON 解析 一律偵測重複鍵，違規即失敗。人讀為主的產物採純文字或 diff --git a/tools/README.rst b/tools/README.rst index e78a4f2..1eab23c 100644 --- a/tools/README.rst +++ b/tools/README.rst @@ -67,6 +67,12 @@ cluster-keywords Deterministically build the coding vocabulary from the two tagging runs' archives, by pooling their keywords per the project's handoff contract and then sentence-embedding and clustering them. Requires the optional ``cluster`` dependency group. Check ``pop-fem-audit-tools cluster-keywords -h`` for complete instructions on its usage. +compare-codings +--------------- + +Compare the two coding runs' archives and export the per-song disagreements, for the arbitration step to rule on. The keywords both runs assigned are settled by the comparison itself; only the keywords assigned by exactly one run are exported, each with the quotes the assigning run gave as evidence. Check ``pop-fem-audit-tools compare-codings -h`` for complete instructions on its usage. + + Copyright ========= diff --git a/tools/docs/source/pop_fem_audit_tools.commands.rst b/tools/docs/source/pop_fem_audit_tools.commands.rst index 369b03f..07d97d9 100644 --- a/tools/docs/source/pop_fem_audit_tools.commands.rst +++ b/tools/docs/source/pop_fem_audit_tools.commands.rst @@ -20,6 +20,14 @@ pop\_fem\_audit\_tools.commands.cluster\_keywords module :show-inheritance: :undoc-members: +pop\_fem\_audit\_tools.commands.compare\_codings module +------------------------------------------------------- + +.. automodule:: pop_fem_audit_tools.commands.compare_codings + :members: + :show-inheritance: + :undoc-members: + pop\_fem\_audit\_tools.commands.export\_llm\_input module --------------------------------------------------------- diff --git a/tools/src/pop_fem_audit_tools/__main__.py b/tools/src/pop_fem_audit_tools/__main__.py index f021ef1..f9f42b7 100644 --- a/tools/src/pop_fem_audit_tools/__main__.py +++ b/tools/src/pop_fem_audit_tools/__main__.py @@ -18,6 +18,7 @@ from types import ModuleType from .commands import ( build_db_command, cluster_keywords_command, + compare_codings_command, export_llm_input_command, fetch_artists_command, fetch_lyrics_command, @@ -30,6 +31,7 @@ MODULE_PROG: str = "python -m pop_fem_audit_tools" SUBCOMMANDS: dict[str, Callable[[list[str] | None], int]] = { "build-db": build_db_command, "cluster-keywords": cluster_keywords_command, + "compare-codings": compare_codings_command, "export-llm-input": export_llm_input_command, "fetch-artists": fetch_artists_command, "fetch-lyrics": fetch_lyrics_command, diff --git a/tools/src/pop_fem_audit_tools/commands/__init__.py b/tools/src/pop_fem_audit_tools/commands/__init__.py index ed11b0c..eca3d70 100644 --- a/tools/src/pop_fem_audit_tools/commands/__init__.py +++ b/tools/src/pop_fem_audit_tools/commands/__init__.py @@ -5,6 +5,7 @@ """The registry of the CLI subcommands.""" from .build_db import main as build_db_command from .cluster_keywords import main as cluster_keywords_command +from .compare_codings import main as compare_codings_command from .export_llm_input import main as export_llm_input_command from .fetch_artists import main as fetch_artists_command from .fetch_lyrics import main as fetch_lyrics_command diff --git a/tools/src/pop_fem_audit_tools/commands/compare_codings.py b/tools/src/pop_fem_audit_tools/commands/compare_codings.py new file mode 100644 index 0000000..926af11 --- /dev/null +++ b/tools/src/pop_fem_audit_tools/commands/compare_codings.py @@ -0,0 +1,280 @@ +# Tools for A Feminist Audit of Pop Music. +# Copyright 2026 imacat. All rights reserved. +# Authors: +# imacat@mail.imacat.idv.tw (imacat), 2026/8/6 +"""The coding run comparison step. + +Compares the two independent coding runs of the same songs and +exports their per-song disagreements, writing one fixed-named +artifact under the output directory given as the third positional +command-line argument. The two runs are compared by their keyword +key sets alone: a keyword assigned to a song by exactly one of the +two runs is a disagreement, and a keyword assigned by both is +agreed. The quotes supporting a keyword never take part in the +comparison; they are carried along as the evidence of the disagreed +keyword they support. The disagreements are written as a JSON file, +as :data:`DISAGREEMENTS_JSON`, holding only the songs the two runs +disagree on; it is what the arbitration step of the coding +procedure settles. The agreed half is not written; it is returned +by :func:`compare_codings`, as the keyword names alone, for later +steps to consume. The step is fully deterministic; no LLM call is +made. +""" +import argparse +import json +import sys +import time +from pathlib import Path +from typing import Any + +from ..utils import format_duration + +DISAGREEMENTS_JSON: str = "disagreements.json" +"""The disagreement JSON file's fixed name under the output +directory.""" + +type Coding = dict[str, list[str]] +"""The keywords assigned to one song, each with its lyric quotes.""" + +type Codings = dict[str, Coding] +"""The coding of every song of one run, keyed by the song ID.""" + +type AgreedKeywords = dict[str, list[str]] +"""The keywords both runs assigned to a song, keyed by the song +ID.""" + + +def parse_args(argv: list[str] | None) -> argparse.Namespace: + """Parse the command-line arguments. + + :param argv: The command-line arguments, or None for + ``sys.argv``. + :return: The parsed arguments. + """ + parser: argparse.ArgumentParser = argparse.ArgumentParser( + description="Compare the two coding runs of the same songs" + " and export their per-song disagreements for" + " the arbitration step.") + parser.add_argument( + "run_dir_1", type=Path, + help="the first coding run's archive directory") + parser.add_argument( + "run_dir_2", type=Path, + help="the second coding run's archive directory") + parser.add_argument( + "output_dir", type=Path, + help="the output directory, created if missing, that" + f" receives {DISAGREEMENTS_JSON}") + return parser.parse_args(argv) + + +def reject_duplicate_keys( + pairs: list[tuple[str, Any]]) -> dict[str, Any]: + """Build a mapping from key-value pairs, rejecting duplicates. + + :param pairs: The key-value pairs, in document order. + :return: The mapping built from the pairs. + :raises ValueError: When a key appears more than once. + """ + result: dict[str, Any] = {} + key: str + value: Any + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate key \"{key}\"") + result[key] = value + return result + + +def song_number(song_id: str) -> int: + """Return the song number carried by a song ID. + + :param song_id: The song ID, expected as ``song-``. + :return: The song number. + :raises ValueError: When the song ID is not ``song-``. + """ + prefix: str = "song-" + if not song_id.startswith(prefix) \ + or not song_id[len(prefix):].isdigit(): + raise ValueError( + f"id \"{song_id}\": not in \"song-\" form") + return int(song_id[len(prefix):]) + + +def load_run(run_dir: Path) -> Codings: + """Load and validate the coding of one coding run. + + Every record must carry a "text" field parsing to a JSON + object; a record that does not fails the run. Lines are split + on the newline character alone, as the quoted lyrics may carry + other control characters that are not line breaks here. + + :param run_dir: The run's archive directory, containing + ``output.jsonl``. + :return: The keyword assignments of every song, keyed by the + song ID, in file order. + :raises OSError: When ``output.jsonl`` cannot be read. + :raises ValueError: When a line is not a well-formed output + record, a "text" field does not parse to a JSON object, a + JSON document holds a duplicate key, or a song ID appears + more than once. + """ + path: Path = run_dir / "output.jsonl" + text: str = path.read_text(encoding="utf-8") + codings: Codings = {} + line: str + for line in text.split("\n"): + if line.strip() == "": + continue + record: Any = json.loads( + line, object_pairs_hook=reject_duplicate_keys) + if not isinstance(record, dict) or "id" not in record: + raise ValueError( + f"{path}: record without \"id\": {line}") + song_id: str = record["id"] + try: + song_number(song_id) + except ValueError as error: + raise ValueError(f"{path}: {error}") from error + if "text" not in record: + raise ValueError( + f"{path}: id {song_id}: record without \"text\"") + if song_id in codings: + raise ValueError( + f"{path}: id {song_id}: duplicate song ID") + try: + coding: Any = json.loads( + record["text"], + object_pairs_hook=reject_duplicate_keys) + except json.JSONDecodeError as error: + raise ValueError( + f"{path}: id {song_id}: \"text\" does not parse as" + f" JSON: {error}") from error + if not isinstance(coding, dict): + raise ValueError( + f"{path}: id {song_id}: \"text\" does not parse to" + " a JSON object") + codings[song_id] = coding + return codings + + +def compare_codings( + codings1: Codings, + codings2: Codings) -> tuple[AgreedKeywords, Codings]: + """Compare the codings of two runs of the same songs. + + The comparison is by keyword key set alone; the quotes never + take part in it. A keyword assigned by both runs is agreed, + and is returned by its name alone, without quotes. A keyword + assigned by exactly one run is a disagreement, and carries the + quotes of the run that assigned it. A song with no agreed + keyword has no entry in the agreed half, and a song the two + runs fully agree on has no entry in the disagreement half. + Both halves are ordered by ascending song number, and every + song's keywords lexicographically. + + :param codings1: The first run's coding of every song, keyed + by the song ID. + :param codings2: The second run's coding of every song, keyed + by the song ID. + :return: The per-song agreed keyword names and the per-song + disagreed keywords with their quotes. + :raises ValueError: When the two runs do not cover exactly the + same set of song IDs, or a song ID is not ``song-``. + """ + only1: set[str] = set(codings1) - set(codings2) + only2: set[str] = set(codings2) - set(codings1) + if len(only1) > 0 or len(only2) > 0: + raise ValueError( + "the two runs do not cover the same songs:" + f" {len(only1)} only in the first run" + f" ({', '.join(sorted(only1)[:5])})," + f" {len(only2)} only in the second run" + f" ({', '.join(sorted(only2)[:5])})") + agreed: AgreedKeywords = {} + disagreed: Codings = {} + song_id: str + for song_id in sorted(codings1, key=song_number): + coding1: Coding = codings1[song_id] + coding2: Coding = codings2[song_id] + song_agreed: list[str] = [] + song_disagreed: Coding = {} + keyword: str + for keyword in sorted(set(coding1) | set(coding2)): + if keyword in coding1 and keyword in coding2: + song_agreed.append(keyword) + elif keyword in coding1: + song_disagreed[keyword] = coding1[keyword] + else: + song_disagreed[keyword] = coding2[keyword] + if len(song_agreed) > 0: + agreed[song_id] = song_agreed + if len(song_disagreed) > 0: + disagreed[song_id] = song_disagreed + return agreed, disagreed + + +def count_keywords(codings: Codings) -> int: + """Count the keywords of every song of a coding. + + :param codings: The keyword assignments of every song, keyed + by the song ID. + :return: The total number of keyword assignments. + """ + return sum(len(x) for x in codings.values()) + + +def write_disagreements(path: Path, + disagreements: Codings) -> None: + """Write the per-song disagreement JSON file. + + Writes a JSON file holding a single object mapping the song ID + to the disagreed keywords of that song, each with the quotes of + the run that assigned it, in the given order, UTF-8, with a + trailing newline. Only the songs the two runs disagree on are + written. + + :param path: The path of the disagreement JSON file to write. + :param disagreements: The disagreed keywords of every song, + keyed by the song ID. + :return: None. + :raises OSError: When the file cannot be written. + """ + path.write_text( + json.dumps(disagreements, ensure_ascii=False, indent=2) + + "\n", + encoding="utf-8") + + +def main(argv: list[str] | None = None) -> int: + """Compare the two coding runs and export the disagreements. + + Writes the disagreement JSON file under the output directory, + creating it (with parents) if it does not exist. When the + input is rejected, the file is not written. + + :param argv: The command-line arguments, or None for + ``sys.argv``. + :return: The exit status: 0 on success, non-zero on failure. + """ + started: float = time.monotonic() + args: argparse.Namespace = parse_args(argv) + codings1: Codings + disagreements: Codings + try: + codings1 = load_run(args.run_dir_1) + codings2: Codings = load_run(args.run_dir_2) + disagreements = compare_codings(codings1, codings2)[1] + except (OSError, ValueError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + args.output_dir.mkdir(parents=True, exist_ok=True) + write_disagreements( + args.output_dir / DISAGREEMENTS_JSON, disagreements) + elapsed: str = format_duration(time.monotonic() - started) + print( + f"Done. {len(disagreements)} of {len(codings1)} songs" + f" disagree on {count_keywords(disagreements)} keywords." + f" {elapsed} elapsed.", + file=sys.stderr) + return 0 diff --git a/tools/tests/test_compare_codings.py b/tools/tests/test_compare_codings.py new file mode 100644 index 0000000..e08afce --- /dev/null +++ b/tools/tests/test_compare_codings.py @@ -0,0 +1,398 @@ +# Tools for A Feminist Audit of Pop Music. +# Copyright 2026 imacat. All rights reserved. +# Authors: +# imacat@mail.imacat.idv.tw (imacat), 2026/8/6 +"""Unit tests for the coding run comparison module.""" +import io +import json +import tempfile +import unittest +from contextlib import redirect_stderr +from pathlib import Path +from typing import Any + +from pop_fem_audit_tools.commands import compare_codings + + +class TestCompareCodings(unittest.TestCase): + """Test cases for the coding run comparison.""" + + def setUp(self) -> None: + """Create a temporary directory with two run directories.""" + tmp: tempfile.TemporaryDirectory[str] \ + = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.__dir: Path = Path(tmp.name) + self.__run1: Path = self.__dir / "run1" + self.__run2: Path = self.__dir / "run2" + self.__run1.mkdir() + self.__run2.mkdir() + self.__output_dir: Path = self.__dir / "output" + self.__disagreements_json: Path \ + = self.__output_dir \ + / compare_codings.DISAGREEMENTS_JSON + + @staticmethod + def __write_output( + run_dir: Path, records: list[dict[str, Any]]) -> None: + """Write the ``output.jsonl`` file of one run. + + :param run_dir: The run's archive directory. + :param records: The envelope records, in file order. + :return: None. + """ + lines: list[str] = [ + json.dumps(x, ensure_ascii=False) for x in records] + (run_dir / "output.jsonl").write_text( + "\n".join(lines) + "\n", encoding="utf-8") + + def __write_codings( + self, run_dir: Path, + codings: dict[str, dict[str, list[str]]]) -> None: + """Write the coding of every song of one run. + + :param run_dir: The run's archive directory. + :param codings: The keyword assignments of every song, + keyed by the song ID, in file order. + :return: None. + """ + self.__write_output(run_dir, [ + {"id": song_id, "text": json.dumps( + coding, ensure_ascii=False), + "stop_reason": "end_turn", "usage": {}} + for song_id, coding in codings.items()]) + + def __run_compare(self) -> tuple[int, str]: + """Run the comparison with captured standard error. + + :return: A tuple of the exit status and the standard + error. + """ + argv: list[str] = [ + str(self.__run1), str(self.__run2), + str(self.__output_dir)] + stderr: io.StringIO = io.StringIO() + with redirect_stderr(stderr): + status: int = compare_codings.main(argv) + return status, stderr.getvalue() + + def __read_disagreements(self) -> dict[str, Any]: + """Read the disagreement JSON file. + + :return: The parsed disagreements. + """ + return json.loads( + self.__disagreements_json.read_text(encoding="utf-8")) + + def __read_disagreement_text(self) -> str: + """Read the disagreement JSON file verbatim. + + :return: The file content, as written. + """ + return self.__disagreements_json.read_text( + encoding="utf-8") + + def test_keyword_in_one_run_only_disagrees(self) -> None: + """Test that a keyword assigned by exactly one run is a + disagreement carrying that run's quotes.""" + self.__write_codings(self.__run1, { + "song-1": {"shared": ["q1"], "only-1": ["q2"]}}) + self.__write_codings(self.__run2, { + "song-1": {"shared": ["q3"], "only-2": ["q4"]}}) + status: int + status, _ = self.__run_compare() + self.assertEqual(status, 0) + self.assertEqual(self.__read_disagreements(), { + "song-1": {"only-1": ["q2"], "only-2": ["q4"]}}) + + def test_quotes_do_not_take_part_in_comparison(self) -> None: + """Test that identical key sets with different quotes + yield no disagreement at all.""" + self.__write_codings(self.__run1, { + "song-1": {"shared": ["one quote"]}}) + self.__write_codings(self.__run2, { + "song-1": {"shared": ["another", "quote"]}}) + status: int + stderr: str + status, stderr = self.__run_compare() + self.assertEqual(status, 0) + self.assertEqual(self.__read_disagreements(), {}) + self.assertIn("0 of 1 songs disagree on 0 keywords.", + stderr) + + def test_agreeing_songs_omitted(self) -> None: + """Test that only the songs with at least one + disagreement are written.""" + self.__write_codings(self.__run1, { + "song-1": {"shared": ["q"]}, + "song-2": {"shared": ["q"], "only-1": ["q"]}}) + self.__write_codings(self.__run2, { + "song-1": {"shared": ["q"]}, + "song-2": {"shared": ["q"]}}) + status: int + status, _ = self.__run_compare() + self.assertEqual(status, 0) + self.assertEqual( + list(self.__read_disagreements().keys()), ["song-2"]) + + def test_songs_in_ascending_song_number_order(self) -> None: + """Test that the songs are ordered by ascending song + number, not by the song ID as text.""" + self.__write_codings(self.__run1, { + "song-10": {"only-1": ["q"]}, + "song-2": {"only-1": ["q"]}, + "song-1": {"only-1": ["q"]}}) + self.__write_codings(self.__run2, { + "song-1": {}, "song-2": {}, "song-10": {}}) + status: int + status, _ = self.__run_compare() + self.assertEqual(status, 0) + self.assertEqual( + list(self.__read_disagreements().keys()), + ["song-1", "song-2", "song-10"]) + + def test_keywords_in_lexicographic_order(self) -> None: + """Test that a song's disagreed keywords are ordered + lexicographically, whichever run assigned them.""" + self.__write_codings(self.__run1, { + "song-1": {"zeta": ["q"], "alpha": ["q"]}}) + self.__write_codings(self.__run2, { + "song-1": {"mu": ["q"], "beta": ["q"]}}) + status: int + status, _ = self.__run_compare() + self.assertEqual(status, 0) + self.assertEqual( + list(self.__read_disagreements()["song-1"].keys()), + ["alpha", "beta", "mu", "zeta"]) + + def test_summary_line_counts(self) -> None: + """Test the closing summary line's song and keyword + counts.""" + self.__write_codings(self.__run1, { + "song-1": {"only-1": ["q"], "also-1": ["q"]}, + "song-2": {"shared": ["q"]}, + "song-3": {}}) + self.__write_codings(self.__run2, { + "song-1": {}, + "song-2": {"shared": ["q"]}, + "song-3": {"only-2": ["q"]}}) + status: int + stderr: str + status, stderr = self.__run_compare() + self.assertEqual(status, 0) + self.assertIn("Done. 2 of 3 songs disagree on 3" + " keywords.", stderr) + self.assertIn("elapsed.", stderr) + + def test_control_character_in_quote_kept(self) -> None: + """Test that a quote holding a line-separating control + character neither truncates the record nor is lost.""" + quote: str = "first line\u0085second line" + self.__write_codings(self.__run1, { + "song-1": {"only-1": [quote]}}) + self.__write_codings(self.__run2, {"song-1": {}}) + status: int + status, _ = self.__run_compare() + self.assertEqual(status, 0) + self.assertEqual( + self.__read_disagreements(), + {"song-1": {"only-1": [quote]}}) + + def test_non_ascii_written_verbatim(self) -> None: + """Test that the output is UTF-8 with the non-ASCII text + unescaped, indented by two spaces, and newline + terminated.""" + self.__write_codings(self.__run1, { + "song-1": {"only-1": ["女性力量"]}}) + self.__write_codings(self.__run2, {"song-1": {}}) + status: int + status, _ = self.__run_compare() + self.assertEqual(status, 0) + text: str = self.__read_disagreement_text() + self.assertIn("女性力量", text) + self.assertIn("\n \"song-1\": {", text) + self.assertTrue(text.endswith("}\n")) + + def test_existing_output_dir_reused(self) -> None: + """Test that an already-existing output directory is + written into rather than rejected.""" + self.__output_dir.mkdir(parents=True) + (self.__output_dir / "keep.txt").write_text( + "kept", encoding="utf-8") + self.__write_codings(self.__run1, { + "song-1": {"only-1": ["q"]}}) + self.__write_codings(self.__run2, {"song-1": {}}) + status: int + status, _ = self.__run_compare() + self.assertEqual(status, 0) + self.assertTrue(self.__disagreements_json.exists()) + self.assertEqual( + (self.__output_dir / "keep.txt").read_text( + encoding="utf-8"), + "kept") + + def test_mismatched_song_sets_rejected(self) -> None: + """Test that runs covering different songs fail without + writing the disagreement file.""" + self.__write_codings(self.__run1, { + "song-1": {"only-1": ["q"]}}) + self.__write_codings(self.__run2, { + "song-2": {"only-2": ["q"]}}) + status: int + stderr: str + status, stderr = self.__run_compare() + self.assertEqual(status, 1) + self.assertIn("song-1", stderr) + self.assertIn("song-2", stderr) + self.assertFalse(self.__disagreements_json.exists()) + + def test_extra_song_in_one_run_rejected(self) -> None: + """Test that one run covering an extra song fails.""" + self.__write_codings(self.__run1, { + "song-1": {}, "song-2": {}}) + self.__write_codings(self.__run2, {"song-1": {}}) + status: int + stderr: str + status, stderr = self.__run_compare() + self.assertEqual(status, 1) + self.assertIn("song-2", stderr) + self.assertFalse(self.__disagreements_json.exists()) + + def test_non_object_text_rejected(self) -> None: + """Test that a "text" field parsing to something other + than a JSON object fails the run.""" + self.__write_output(self.__run1, [ + {"id": "song-1", "text": json.dumps(["only-1"])}]) + self.__write_codings(self.__run2, {"song-1": {}}) + status: int + stderr: str + status, stderr = self.__run_compare() + self.assertEqual(status, 1) + self.assertIn("song-1", stderr) + self.assertFalse(self.__disagreements_json.exists()) + + def test_unparsable_text_rejected(self) -> None: + """Test that a "text" field that is not JSON at all, such + as a refusal, fails the run.""" + self.__write_output(self.__run1, [ + {"id": "song-1", "text": "I cannot help with that."}]) + self.__write_codings(self.__run2, {"song-1": {}}) + status: int + stderr: str + status, stderr = self.__run_compare() + self.assertEqual(status, 1) + self.assertIn("song-1", stderr) + self.assertFalse(self.__disagreements_json.exists()) + + def test_duplicate_keyword_in_text_rejected(self) -> None: + """Test that a "text" JSON object with a duplicate keyword + fails the run.""" + self.__write_output(self.__run1, [ + {"id": "song-1", + "text": '{"only-1": ["q"], "only-1": ["r"]}'}]) + self.__write_codings(self.__run2, {"song-1": {}}) + status: int + stderr: str + status, stderr = self.__run_compare() + self.assertEqual(status, 1) + self.assertIn("duplicate key", stderr) + self.assertFalse(self.__disagreements_json.exists()) + + def test_duplicate_key_in_envelope_rejected(self) -> None: + """Test that an envelope record with a duplicate key fails + the run.""" + (self.__run1 / "output.jsonl").write_text( + '{"id": "song-1", "id": "song-2", "text": "{}"}\n', + encoding="utf-8") + self.__write_codings(self.__run2, {"song-1": {}}) + status: int + stderr: str + status, stderr = self.__run_compare() + self.assertEqual(status, 1) + self.assertIn("duplicate key", stderr) + self.assertFalse(self.__disagreements_json.exists()) + + def test_duplicate_song_id_rejected(self) -> None: + """Test that the same song ID appearing twice in one run + fails the run.""" + self.__write_output(self.__run1, [ + {"id": "song-1", "text": "{}"}, + {"id": "song-1", "text": "{}"}]) + self.__write_codings(self.__run2, {"song-1": {}}) + status: int + stderr: str + status, stderr = self.__run_compare() + self.assertEqual(status, 1) + self.assertIn("duplicate song ID", stderr) + self.assertFalse(self.__disagreements_json.exists()) + + def test_malformed_song_id_rejected(self) -> None: + """Test that a song ID not in the ``song-`` form fails + the run.""" + self.__write_output(self.__run1, [ + {"id": "track-1", "text": "{}"}]) + self.__write_codings(self.__run2, {"song-1": {}}) + status: int + stderr: str + status, stderr = self.__run_compare() + self.assertEqual(status, 1) + self.assertIn("track-1", stderr) + self.assertFalse(self.__disagreements_json.exists()) + + def test_missing_output_file_reported(self) -> None: + """Test that a run directory without ``output.jsonl`` + fails with the path in the message.""" + self.__write_codings(self.__run2, {"song-1": {}}) + status: int + stderr: str + status, stderr = self.__run_compare() + self.assertEqual(status, 1) + self.assertIn("output.jsonl", stderr) + self.assertFalse(self.__disagreements_json.exists()) + + def test_blank_lines_skipped(self) -> None: + """Test that blank lines in ``output.jsonl`` are + ignored.""" + (self.__run1 / "output.jsonl").write_text( + '\n{"id": "song-1", "text": "{\\"only-1\\": [\\"q\\"]}"}' + "\n\n", + encoding="utf-8") + self.__write_codings(self.__run2, {"song-1": {}}) + status: int + status, _ = self.__run_compare() + self.assertEqual(status, 0) + self.assertEqual( + self.__read_disagreements(), + {"song-1": {"only-1": ["q"]}}) + + def test_compare_returns_agreed_keyword_names(self) -> None: + """Test that the comparison function returns the agreed + keyword names alone, sorted, without their quotes.""" + codings1: dict[str, dict[str, list[str]]] = { + "song-1": {"zeta": ["a", "b"], "alpha": ["c"], + "only-1": ["d"]}} + codings2: dict[str, dict[str, list[str]]] = { + "song-1": {"zeta": ["b", "e"], "alpha": ["f"]}} + agreed: dict[str, list[str]] + disagreed: dict[str, dict[str, list[str]]] + agreed, disagreed = compare_codings.compare_codings( + codings1, codings2) + self.assertEqual(agreed, {"song-1": ["alpha", "zeta"]}) + self.assertEqual( + disagreed, {"song-1": {"only-1": ["d"]}}) + + def test_compare_omits_songs_without_agreement(self) -> None: + """Test that a song with no agreed keyword has no entry in + the agreed half.""" + agreed: dict[str, list[str]] + agreed = compare_codings.compare_codings( + {"song-1": {"only-1": ["q"]}, "song-2": {"both": ["q"]}}, + {"song-1": {}, "song-2": {"both": ["q"]}})[0] + self.assertEqual(list(agreed.keys()), ["song-2"]) + + def test_compare_rejects_different_song_sets(self) -> None: + """Test that the comparison function rejects runs that do + not cover the same songs.""" + with self.assertRaises(ValueError): + compare_codings.compare_codings( + {"song-1": {}}, {"song-2": {}})