Add the compare-codings subcommand for the arbitration handoff

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

docs/decision-log.md

@@ -495,6 +495,15 @@
   代價是少數組名不副實，依前條據實報告。實驗歸檔不入
   repo，費用 $0.09（sonnet）與 $0.43（fable）記入
   `run-costs.md`。
+- **定案編碼表只存歌與碼，不附引述與來源層**：引述已完整
+  存在於三份執行歸檔（3-1 兩次、3-2 仲裁），定案表再抄
+  一份即庫內重複。且同一個碼在三處的引述可能不同——兩次
+  執行各自的引述、仲裁者重新判讀後自己的引述——抄一份
+  等於替後續分析先做了選擇，並抹掉「兩次執行是否抓住
+  同一句」這個可分析的不穩定性。另一考量：`results/` 是
+  論文引用的目錄，一萬四千餘行引述抄進去，會把步驟 3-1
+  才從 64% 降到 23% 的歌詞重製量堆回去。來源層（共識或
+  仲裁）同理由三份歸檔可導出，一併不存。
 - **刪除 `source-provenance.csv`，`cluster-keywords` 的產物
   由六份減為五份**：該檔是兩份標註執行歸檔 `output.jsonl`
   的攤平視圖，14,035 列 Keyword,Run,Song，未帶入任何庫裏

docs/methodology.md

@@ -148,8 +148,7 @@ LLM。設計原則見 `research-plan.md`；本檔記載可重現的
   鍵按字典序。
 - **步驟 3 定案**：每首歌的最終標籤為共識標籤加上仲裁
   保留的標籤，寫入逐首紀錄檔（歌依 ID 升序、標籤按
-  字典序，各標籤附其定案時的引述與來源層——共識或
-  仲裁）。
+  字典序）。
 - **序列化通則**：所有中間檔為 UTF-8，欄序、鍵序與元素
   序皆依上列規則明定，無時間戳、無隨機成分；JSON 解析
   一律偵測重複鍵，違規即失敗。人讀為主的產物採純文字或

tools/README.rst

@@ -67,6 +67,12 @@ cluster-keywords
 Deterministically build the coding vocabulary from the two tagging runs' archives, by pooling their keywords per the project's handoff contract and then sentence-embedding and clustering them.  Requires the optional ``cluster`` dependency group.  Check ``pop-fem-audit-tools cluster-keywords -h`` for complete instructions on its usage.
 
 
+compare-codings
+---------------
+
+Compare the two coding runs' archives and export the per-song disagreements, for the arbitration step to rule on.  The keywords both runs assigned are settled by the comparison itself; only the keywords assigned by exactly one run are exported, each with the quotes the assigning run gave as evidence.  Check ``pop-fem-audit-tools compare-codings -h`` for complete instructions on its usage.
+
+
 Copyright
 =========
 

tools/docs/source/pop_fem_audit_tools.commands.rst

@@ -20,6 +20,14 @@ pop\_fem\_audit\_tools.commands.cluster\_keywords module
    :show-inheritance:
    :undoc-members:
 
+pop\_fem\_audit\_tools.commands.compare\_codings module
+-------------------------------------------------------
+
+.. automodule:: pop_fem_audit_tools.commands.compare_codings
+   :members:
+   :show-inheritance:
+   :undoc-members:
+
 pop\_fem\_audit\_tools.commands.export\_llm\_input module
 ---------------------------------------------------------
 

tools/src/pop_fem_audit_tools/__main__.py

@@ -18,6 +18,7 @@ from types import ModuleType
 from .commands import (
     build_db_command,
     cluster_keywords_command,
+    compare_codings_command,
     export_llm_input_command,
     fetch_artists_command,
     fetch_lyrics_command,
@@ -30,6 +31,7 @@ MODULE_PROG: str = "python -m pop_fem_audit_tools"
 SUBCOMMANDS: dict[str, Callable[[list[str] | None], int]] = {
     "build-db": build_db_command,
     "cluster-keywords": cluster_keywords_command,
+    "compare-codings": compare_codings_command,
     "export-llm-input": export_llm_input_command,
     "fetch-artists": fetch_artists_command,
     "fetch-lyrics": fetch_lyrics_command,

tools/src/pop_fem_audit_tools/commands/__init__.py

@@ -5,6 +5,7 @@
 """The registry of the CLI subcommands."""
 from .build_db import main as build_db_command
 from .cluster_keywords import main as cluster_keywords_command
+from .compare_codings import main as compare_codings_command
 from .export_llm_input import main as export_llm_input_command
 from .fetch_artists import main as fetch_artists_command
 from .fetch_lyrics import main as fetch_lyrics_command

tools/src/pop_fem_audit_tools/commands/compare_codings.py

@@ -0,0 +1,280 @@
+# Tools for A Feminist Audit of Pop Music.
+# Copyright 2026 imacat.  All rights reserved.
+# Authors:
+#   imacat@mail.imacat.idv.tw (imacat), 2026/8/6
+"""The coding run comparison step.
+
+Compares the two independent coding runs of the same songs and
+exports their per-song disagreements, writing one fixed-named
+artifact under the output directory given as the third positional
+command-line argument.  The two runs are compared by their keyword
+key sets alone: a keyword assigned to a song by exactly one of the
+two runs is a disagreement, and a keyword assigned by both is
+agreed.  The quotes supporting a keyword never take part in the
+comparison; they are carried along as the evidence of the disagreed
+keyword they support.  The disagreements are written as a JSON file,
+as :data:`DISAGREEMENTS_JSON`, holding only the songs the two runs
+disagree on; it is what the arbitration step of the coding
+procedure settles.  The agreed half is not written; it is returned
+by :func:`compare_codings`, as the keyword names alone, for later
+steps to consume.  The step is fully deterministic; no LLM call is
+made.
+"""
+import argparse
+import json
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+from ..utils import format_duration
+
+DISAGREEMENTS_JSON: str = "disagreements.json"
+"""The disagreement JSON file's fixed name under the output
+directory."""
+
+type Coding = dict[str, list[str]]
+"""The keywords assigned to one song, each with its lyric quotes."""
+
+type Codings = dict[str, Coding]
+"""The coding of every song of one run, keyed by the song ID."""
+
+type AgreedKeywords = dict[str, list[str]]
+"""The keywords both runs assigned to a song, keyed by the song
+ID."""
+
+
+def parse_args(argv: list[str] | None) -> argparse.Namespace:
+    """Parse the command-line arguments.
+
+    :param argv: The command-line arguments, or None for
+        ``sys.argv``.
+    :return: The parsed arguments.
+    """
+    parser: argparse.ArgumentParser = argparse.ArgumentParser(
+        description="Compare the two coding runs of the same songs"
+                    " and export their per-song disagreements for"
+                    " the arbitration step.")
+    parser.add_argument(
+        "run_dir_1", type=Path,
+        help="the first coding run's archive directory")
+    parser.add_argument(
+        "run_dir_2", type=Path,
+        help="the second coding run's archive directory")
+    parser.add_argument(
+        "output_dir", type=Path,
+        help="the output directory, created if missing, that"
+             f" receives {DISAGREEMENTS_JSON}")
+    return parser.parse_args(argv)
+
+
+def reject_duplicate_keys(
+        pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+    """Build a mapping from key-value pairs, rejecting duplicates.
+
+    :param pairs: The key-value pairs, in document order.
+    :return: The mapping built from the pairs.
+    :raises ValueError: When a key appears more than once.
+    """
+    result: dict[str, Any] = {}
+    key: str
+    value: Any
+    for key, value in pairs:
+        if key in result:
+            raise ValueError(f"duplicate key \"{key}\"")
+        result[key] = value
+    return result
+
+
+def song_number(song_id: str) -> int:
+    """Return the song number carried by a song ID.
+
+    :param song_id: The song ID, expected as ``song-<N>``.
+    :return: The song number.
+    :raises ValueError: When the song ID is not ``song-<N>``.
+    """
+    prefix: str = "song-"
+    if not song_id.startswith(prefix) \
+            or not song_id[len(prefix):].isdigit():
+        raise ValueError(
+            f"id \"{song_id}\": not in \"song-<N>\" form")
+    return int(song_id[len(prefix):])
+
+
+def load_run(run_dir: Path) -> Codings:
+    """Load and validate the coding of one coding run.
+
+    Every record must carry a "text" field parsing to a JSON
+    object; a record that does not fails the run.  Lines are split
+    on the newline character alone, as the quoted lyrics may carry
+    other control characters that are not line breaks here.
+
+    :param run_dir: The run's archive directory, containing
+        ``output.jsonl``.
+    :return: The keyword assignments of every song, keyed by the
+        song ID, in file order.
+    :raises OSError: When ``output.jsonl`` cannot be read.
+    :raises ValueError: When a line is not a well-formed output
+        record, a "text" field does not parse to a JSON object, a
+        JSON document holds a duplicate key, or a song ID appears
+        more than once.
+    """
+    path: Path = run_dir / "output.jsonl"
+    text: str = path.read_text(encoding="utf-8")
+    codings: Codings = {}
+    line: str
+    for line in text.split("\n"):
+        if line.strip() == "":
+            continue
+        record: Any = json.loads(
+            line, object_pairs_hook=reject_duplicate_keys)
+        if not isinstance(record, dict) or "id" not in record:
+            raise ValueError(
+                f"{path}: record without \"id\": {line}")
+        song_id: str = record["id"]
+        try:
+            song_number(song_id)
+        except ValueError as error:
+            raise ValueError(f"{path}: {error}") from error
+        if "text" not in record:
+            raise ValueError(
+                f"{path}: id {song_id}: record without \"text\"")
+        if song_id in codings:
+            raise ValueError(
+                f"{path}: id {song_id}: duplicate song ID")
+        try:
+            coding: Any = json.loads(
+                record["text"],
+                object_pairs_hook=reject_duplicate_keys)
+        except json.JSONDecodeError as error:
+            raise ValueError(
+                f"{path}: id {song_id}: \"text\" does not parse as"
+                f" JSON: {error}") from error
+        if not isinstance(coding, dict):
+            raise ValueError(
+                f"{path}: id {song_id}: \"text\" does not parse to"
+                " a JSON object")
+        codings[song_id] = coding
+    return codings
+
+
+def compare_codings(
+        codings1: Codings,
+        codings2: Codings) -> tuple[AgreedKeywords, Codings]:
+    """Compare the codings of two runs of the same songs.
+
+    The comparison is by keyword key set alone; the quotes never
+    take part in it.  A keyword assigned by both runs is agreed,
+    and is returned by its name alone, without quotes.  A keyword
+    assigned by exactly one run is a disagreement, and carries the
+    quotes of the run that assigned it.  A song with no agreed
+    keyword has no entry in the agreed half, and a song the two
+    runs fully agree on has no entry in the disagreement half.
+    Both halves are ordered by ascending song number, and every
+    song's keywords lexicographically.
+
+    :param codings1: The first run's coding of every song, keyed
+        by the song ID.
+    :param codings2: The second run's coding of every song, keyed
+        by the song ID.
+    :return: The per-song agreed keyword names and the per-song
+        disagreed keywords with their quotes.
+    :raises ValueError: When the two runs do not cover exactly the
+        same set of song IDs, or a song ID is not ``song-<N>``.
+    """
+    only1: set[str] = set(codings1) - set(codings2)
+    only2: set[str] = set(codings2) - set(codings1)
+    if len(only1) > 0 or len(only2) > 0:
+        raise ValueError(
+            "the two runs do not cover the same songs:"
+            f" {len(only1)} only in the first run"
+            f" ({', '.join(sorted(only1)[:5])}),"
+            f" {len(only2)} only in the second run"
+            f" ({', '.join(sorted(only2)[:5])})")
+    agreed: AgreedKeywords = {}
+    disagreed: Codings = {}
+    song_id: str
+    for song_id in sorted(codings1, key=song_number):
+        coding1: Coding = codings1[song_id]
+        coding2: Coding = codings2[song_id]
+        song_agreed: list[str] = []
+        song_disagreed: Coding = {}
+        keyword: str
+        for keyword in sorted(set(coding1) | set(coding2)):
+            if keyword in coding1 and keyword in coding2:
+                song_agreed.append(keyword)
+            elif keyword in coding1:
+                song_disagreed[keyword] = coding1[keyword]
+            else:
+                song_disagreed[keyword] = coding2[keyword]
+        if len(song_agreed) > 0:
+            agreed[song_id] = song_agreed
+        if len(song_disagreed) > 0:
+            disagreed[song_id] = song_disagreed
+    return agreed, disagreed
+
+
+def count_keywords(codings: Codings) -> int:
+    """Count the keywords of every song of a coding.
+
+    :param codings: The keyword assignments of every song, keyed
+        by the song ID.
+    :return: The total number of keyword assignments.
+    """
+    return sum(len(x) for x in codings.values())
+
+
+def write_disagreements(path: Path,
+                        disagreements: Codings) -> None:
+    """Write the per-song disagreement JSON file.
+
+    Writes a JSON file holding a single object mapping the song ID
+    to the disagreed keywords of that song, each with the quotes of
+    the run that assigned it, in the given order, UTF-8, with a
+    trailing newline.  Only the songs the two runs disagree on are
+    written.
+
+    :param path: The path of the disagreement JSON file to write.
+    :param disagreements: The disagreed keywords of every song,
+        keyed by the song ID.
+    :return: None.
+    :raises OSError: When the file cannot be written.
+    """
+    path.write_text(
+        json.dumps(disagreements, ensure_ascii=False, indent=2)
+        + "\n",
+        encoding="utf-8")
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Compare the two coding runs and export the disagreements.
+
+    Writes the disagreement JSON file under the output directory,
+    creating it (with parents) if it does not exist.  When the
+    input is rejected, the file is not written.
+
+    :param argv: The command-line arguments, or None for
+        ``sys.argv``.
+    :return: The exit status: 0 on success, non-zero on failure.
+    """
+    started: float = time.monotonic()
+    args: argparse.Namespace = parse_args(argv)
+    codings1: Codings
+    disagreements: Codings
+    try:
+        codings1 = load_run(args.run_dir_1)
+        codings2: Codings = load_run(args.run_dir_2)
+        disagreements = compare_codings(codings1, codings2)[1]
+    except (OSError, ValueError) as error:
+        print(f"error: {error}", file=sys.stderr)
+        return 1
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    write_disagreements(
+        args.output_dir / DISAGREEMENTS_JSON, disagreements)
+    elapsed: str = format_duration(time.monotonic() - started)
+    print(
+        f"Done.  {len(disagreements)} of {len(codings1)} songs"
+        f" disagree on {count_keywords(disagreements)} keywords."
+        f"  {elapsed} elapsed.",
+        file=sys.stderr)
+    return 0

tools/tests/test_compare_codings.py

@@ -0,0 +1,398 @@
+# Tools for A Feminist Audit of Pop Music.
+# Copyright 2026 imacat.  All rights reserved.
+# Authors:
+#   imacat@mail.imacat.idv.tw (imacat), 2026/8/6
+"""Unit tests for the coding run comparison module."""
+import io
+import json
+import tempfile
+import unittest
+from contextlib import redirect_stderr
+from pathlib import Path
+from typing import Any
+
+from pop_fem_audit_tools.commands import compare_codings
+
+
+class TestCompareCodings(unittest.TestCase):
+    """Test cases for the coding run comparison."""
+
+    def setUp(self) -> None:
+        """Create a temporary directory with two run directories."""
+        tmp: tempfile.TemporaryDirectory[str] \
+            = tempfile.TemporaryDirectory()
+        self.addCleanup(tmp.cleanup)
+        self.__dir: Path = Path(tmp.name)
+        self.__run1: Path = self.__dir / "run1"
+        self.__run2: Path = self.__dir / "run2"
+        self.__run1.mkdir()
+        self.__run2.mkdir()
+        self.__output_dir: Path = self.__dir / "output"
+        self.__disagreements_json: Path \
+            = self.__output_dir \
+            / compare_codings.DISAGREEMENTS_JSON
+
+    @staticmethod
+    def __write_output(
+            run_dir: Path, records: list[dict[str, Any]]) -> None:
+        """Write the ``output.jsonl`` file of one run.
+
+        :param run_dir: The run's archive directory.
+        :param records: The envelope records, in file order.
+        :return: None.
+        """
+        lines: list[str] = [
+            json.dumps(x, ensure_ascii=False) for x in records]
+        (run_dir / "output.jsonl").write_text(
+            "\n".join(lines) + "\n", encoding="utf-8")
+
+    def __write_codings(
+            self, run_dir: Path,
+            codings: dict[str, dict[str, list[str]]]) -> None:
+        """Write the coding of every song of one run.
+
+        :param run_dir: The run's archive directory.
+        :param codings: The keyword assignments of every song,
+            keyed by the song ID, in file order.
+        :return: None.
+        """
+        self.__write_output(run_dir, [
+            {"id": song_id, "text": json.dumps(
+                coding, ensure_ascii=False),
+             "stop_reason": "end_turn", "usage": {}}
+            for song_id, coding in codings.items()])
+
+    def __run_compare(self) -> tuple[int, str]:
+        """Run the comparison with captured standard error.
+
+        :return: A tuple of the exit status and the standard
+            error.
+        """
+        argv: list[str] = [
+            str(self.__run1), str(self.__run2),
+            str(self.__output_dir)]
+        stderr: io.StringIO = io.StringIO()
+        with redirect_stderr(stderr):
+            status: int = compare_codings.main(argv)
+        return status, stderr.getvalue()
+
+    def __read_disagreements(self) -> dict[str, Any]:
+        """Read the disagreement JSON file.
+
+        :return: The parsed disagreements.
+        """
+        return json.loads(
+            self.__disagreements_json.read_text(encoding="utf-8"))
+
+    def __read_disagreement_text(self) -> str:
+        """Read the disagreement JSON file verbatim.
+
+        :return: The file content, as written.
+        """
+        return self.__disagreements_json.read_text(
+            encoding="utf-8")
+
+    def test_keyword_in_one_run_only_disagrees(self) -> None:
+        """Test that a keyword assigned by exactly one run is a
+        disagreement carrying that run's quotes."""
+        self.__write_codings(self.__run1, {
+            "song-1": {"shared": ["q1"], "only-1": ["q2"]}})
+        self.__write_codings(self.__run2, {
+            "song-1": {"shared": ["q3"], "only-2": ["q4"]}})
+        status: int
+        status, _ = self.__run_compare()
+        self.assertEqual(status, 0)
+        self.assertEqual(self.__read_disagreements(), {
+            "song-1": {"only-1": ["q2"], "only-2": ["q4"]}})
+
+    def test_quotes_do_not_take_part_in_comparison(self) -> None:
+        """Test that identical key sets with different quotes
+        yield no disagreement at all."""
+        self.__write_codings(self.__run1, {
+            "song-1": {"shared": ["one quote"]}})
+        self.__write_codings(self.__run2, {
+            "song-1": {"shared": ["another", "quote"]}})
+        status: int
+        stderr: str
+        status, stderr = self.__run_compare()
+        self.assertEqual(status, 0)
+        self.assertEqual(self.__read_disagreements(), {})
+        self.assertIn("0 of 1 songs disagree on 0 keywords.",
+                      stderr)
+
+    def test_agreeing_songs_omitted(self) -> None:
+        """Test that only the songs with at least one
+        disagreement are written."""
+        self.__write_codings(self.__run1, {
+            "song-1": {"shared": ["q"]},
+            "song-2": {"shared": ["q"], "only-1": ["q"]}})
+        self.__write_codings(self.__run2, {
+            "song-1": {"shared": ["q"]},
+            "song-2": {"shared": ["q"]}})
+        status: int
+        status, _ = self.__run_compare()
+        self.assertEqual(status, 0)
+        self.assertEqual(
+            list(self.__read_disagreements().keys()), ["song-2"])
+
+    def test_songs_in_ascending_song_number_order(self) -> None:
+        """Test that the songs are ordered by ascending song
+        number, not by the song ID as text."""
+        self.__write_codings(self.__run1, {
+            "song-10": {"only-1": ["q"]},
+            "song-2": {"only-1": ["q"]},
+            "song-1": {"only-1": ["q"]}})
+        self.__write_codings(self.__run2, {
+            "song-1": {}, "song-2": {}, "song-10": {}})
+        status: int
+        status, _ = self.__run_compare()
+        self.assertEqual(status, 0)
+        self.assertEqual(
+            list(self.__read_disagreements().keys()),
+            ["song-1", "song-2", "song-10"])
+
+    def test_keywords_in_lexicographic_order(self) -> None:
+        """Test that a song's disagreed keywords are ordered
+        lexicographically, whichever run assigned them."""
+        self.__write_codings(self.__run1, {
+            "song-1": {"zeta": ["q"], "alpha": ["q"]}})
+        self.__write_codings(self.__run2, {
+            "song-1": {"mu": ["q"], "beta": ["q"]}})
+        status: int
+        status, _ = self.__run_compare()
+        self.assertEqual(status, 0)
+        self.assertEqual(
+            list(self.__read_disagreements()["song-1"].keys()),
+            ["alpha", "beta", "mu", "zeta"])
+
+    def test_summary_line_counts(self) -> None:
+        """Test the closing summary line's song and keyword
+        counts."""
+        self.__write_codings(self.__run1, {
+            "song-1": {"only-1": ["q"], "also-1": ["q"]},
+            "song-2": {"shared": ["q"]},
+            "song-3": {}})
+        self.__write_codings(self.__run2, {
+            "song-1": {},
+            "song-2": {"shared": ["q"]},
+            "song-3": {"only-2": ["q"]}})
+        status: int
+        stderr: str
+        status, stderr = self.__run_compare()
+        self.assertEqual(status, 0)
+        self.assertIn("Done.  2 of 3 songs disagree on 3"
+                      " keywords.", stderr)
+        self.assertIn("elapsed.", stderr)
+
+    def test_control_character_in_quote_kept(self) -> None:
+        """Test that a quote holding a line-separating control
+        character neither truncates the record nor is lost."""
+        quote: str = "first line\u0085second line"
+        self.__write_codings(self.__run1, {
+            "song-1": {"only-1": [quote]}})
+        self.__write_codings(self.__run2, {"song-1": {}})
+        status: int
+        status, _ = self.__run_compare()
+        self.assertEqual(status, 0)
+        self.assertEqual(
+            self.__read_disagreements(),
+            {"song-1": {"only-1": [quote]}})
+
+    def test_non_ascii_written_verbatim(self) -> None:
+        """Test that the output is UTF-8 with the non-ASCII text
+        unescaped, indented by two spaces, and newline
+        terminated."""
+        self.__write_codings(self.__run1, {
+            "song-1": {"only-1": ["女性力量"]}})
+        self.__write_codings(self.__run2, {"song-1": {}})
+        status: int
+        status, _ = self.__run_compare()
+        self.assertEqual(status, 0)
+        text: str = self.__read_disagreement_text()
+        self.assertIn("女性力量", text)
+        self.assertIn("\n  \"song-1\": {", text)
+        self.assertTrue(text.endswith("}\n"))
+
+    def test_existing_output_dir_reused(self) -> None:
+        """Test that an already-existing output directory is
+        written into rather than rejected."""
+        self.__output_dir.mkdir(parents=True)
+        (self.__output_dir / "keep.txt").write_text(
+            "kept", encoding="utf-8")
+        self.__write_codings(self.__run1, {
+            "song-1": {"only-1": ["q"]}})
+        self.__write_codings(self.__run2, {"song-1": {}})
+        status: int
+        status, _ = self.__run_compare()
+        self.assertEqual(status, 0)
+        self.assertTrue(self.__disagreements_json.exists())
+        self.assertEqual(
+            (self.__output_dir / "keep.txt").read_text(
+                encoding="utf-8"),
+            "kept")
+
+    def test_mismatched_song_sets_rejected(self) -> None:
+        """Test that runs covering different songs fail without
+        writing the disagreement file."""
+        self.__write_codings(self.__run1, {
+            "song-1": {"only-1": ["q"]}})
+        self.__write_codings(self.__run2, {
+            "song-2": {"only-2": ["q"]}})
+        status: int
+        stderr: str
+        status, stderr = self.__run_compare()
+        self.assertEqual(status, 1)
+        self.assertIn("song-1", stderr)
+        self.assertIn("song-2", stderr)
+        self.assertFalse(self.__disagreements_json.exists())
+
+    def test_extra_song_in_one_run_rejected(self) -> None:
+        """Test that one run covering an extra song fails."""
+        self.__write_codings(self.__run1, {
+            "song-1": {}, "song-2": {}})
+        self.__write_codings(self.__run2, {"song-1": {}})
+        status: int
+        stderr: str
+        status, stderr = self.__run_compare()
+        self.assertEqual(status, 1)
+        self.assertIn("song-2", stderr)
+        self.assertFalse(self.__disagreements_json.exists())
+
+    def test_non_object_text_rejected(self) -> None:
+        """Test that a "text" field parsing to something other
+        than a JSON object fails the run."""
+        self.__write_output(self.__run1, [
+            {"id": "song-1", "text": json.dumps(["only-1"])}])
+        self.__write_codings(self.__run2, {"song-1": {}})
+        status: int
+        stderr: str
+        status, stderr = self.__run_compare()
+        self.assertEqual(status, 1)
+        self.assertIn("song-1", stderr)
+        self.assertFalse(self.__disagreements_json.exists())
+
+    def test_unparsable_text_rejected(self) -> None:
+        """Test that a "text" field that is not JSON at all, such
+        as a refusal, fails the run."""
+        self.__write_output(self.__run1, [
+            {"id": "song-1", "text": "I cannot help with that."}])
+        self.__write_codings(self.__run2, {"song-1": {}})
+        status: int
+        stderr: str
+        status, stderr = self.__run_compare()
+        self.assertEqual(status, 1)
+        self.assertIn("song-1", stderr)
+        self.assertFalse(self.__disagreements_json.exists())
+
+    def test_duplicate_keyword_in_text_rejected(self) -> None:
+        """Test that a "text" JSON object with a duplicate keyword
+        fails the run."""
+        self.__write_output(self.__run1, [
+            {"id": "song-1",
+             "text": '{"only-1": ["q"], "only-1": ["r"]}'}])
+        self.__write_codings(self.__run2, {"song-1": {}})
+        status: int
+        stderr: str
+        status, stderr = self.__run_compare()
+        self.assertEqual(status, 1)
+        self.assertIn("duplicate key", stderr)
+        self.assertFalse(self.__disagreements_json.exists())
+
+    def test_duplicate_key_in_envelope_rejected(self) -> None:
+        """Test that an envelope record with a duplicate key fails
+        the run."""
+        (self.__run1 / "output.jsonl").write_text(
+            '{"id": "song-1", "id": "song-2", "text": "{}"}\n',
+            encoding="utf-8")
+        self.__write_codings(self.__run2, {"song-1": {}})
+        status: int
+        stderr: str
+        status, stderr = self.__run_compare()
+        self.assertEqual(status, 1)
+        self.assertIn("duplicate key", stderr)
+        self.assertFalse(self.__disagreements_json.exists())
+
+    def test_duplicate_song_id_rejected(self) -> None:
+        """Test that the same song ID appearing twice in one run
+        fails the run."""
+        self.__write_output(self.__run1, [
+            {"id": "song-1", "text": "{}"},
+            {"id": "song-1", "text": "{}"}])
+        self.__write_codings(self.__run2, {"song-1": {}})
+        status: int
+        stderr: str
+        status, stderr = self.__run_compare()
+        self.assertEqual(status, 1)
+        self.assertIn("duplicate song ID", stderr)
+        self.assertFalse(self.__disagreements_json.exists())
+
+    def test_malformed_song_id_rejected(self) -> None:
+        """Test that a song ID not in the ``song-<N>`` form fails
+        the run."""
+        self.__write_output(self.__run1, [
+            {"id": "track-1", "text": "{}"}])
+        self.__write_codings(self.__run2, {"song-1": {}})
+        status: int
+        stderr: str
+        status, stderr = self.__run_compare()
+        self.assertEqual(status, 1)
+        self.assertIn("track-1", stderr)
+        self.assertFalse(self.__disagreements_json.exists())
+
+    def test_missing_output_file_reported(self) -> None:
+        """Test that a run directory without ``output.jsonl``
+        fails with the path in the message."""
+        self.__write_codings(self.__run2, {"song-1": {}})
+        status: int
+        stderr: str
+        status, stderr = self.__run_compare()
+        self.assertEqual(status, 1)
+        self.assertIn("output.jsonl", stderr)
+        self.assertFalse(self.__disagreements_json.exists())
+
+    def test_blank_lines_skipped(self) -> None:
+        """Test that blank lines in ``output.jsonl`` are
+        ignored."""
+        (self.__run1 / "output.jsonl").write_text(
+            '\n{"id": "song-1", "text": "{\\"only-1\\": [\\"q\\"]}"}'
+            "\n\n",
+            encoding="utf-8")
+        self.__write_codings(self.__run2, {"song-1": {}})
+        status: int
+        status, _ = self.__run_compare()
+        self.assertEqual(status, 0)
+        self.assertEqual(
+            self.__read_disagreements(),
+            {"song-1": {"only-1": ["q"]}})
+
+    def test_compare_returns_agreed_keyword_names(self) -> None:
+        """Test that the comparison function returns the agreed
+        keyword names alone, sorted, without their quotes."""
+        codings1: dict[str, dict[str, list[str]]] = {
+            "song-1": {"zeta": ["a", "b"], "alpha": ["c"],
+                       "only-1": ["d"]}}
+        codings2: dict[str, dict[str, list[str]]] = {
+            "song-1": {"zeta": ["b", "e"], "alpha": ["f"]}}
+        agreed: dict[str, list[str]]
+        disagreed: dict[str, dict[str, list[str]]]
+        agreed, disagreed = compare_codings.compare_codings(
+            codings1, codings2)
+        self.assertEqual(agreed, {"song-1": ["alpha", "zeta"]})
+        self.assertEqual(
+            disagreed, {"song-1": {"only-1": ["d"]}})
+
+    def test_compare_omits_songs_without_agreement(self) -> None:
+        """Test that a song with no agreed keyword has no entry in
+        the agreed half."""
+        agreed: dict[str, list[str]]
+        agreed = compare_codings.compare_codings(
+            {"song-1": {"only-1": ["q"]}, "song-2": {"both": ["q"]}},
+            {"song-1": {}, "song-2": {"both": ["q"]}})[0]
+        self.assertEqual(list(agreed.keys()), ["song-2"])
+
+    def test_compare_rejects_different_song_sets(self) -> None:
+        """Test that the comparison function rejects runs that do
+        not cover the same songs."""
+        with self.assertRaises(ValueError):
+            compare_codings.compare_codings(
+                {"song-1": {}}, {"song-2": {}})