Settle the coding by a majority of three runs instead of an arbitration
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -67,12 +67,6 @@ cluster-keywords
|
||||
Deterministically build the coding vocabulary from the two tagging runs' archives, by pooling their keywords per the project's handoff contract and then sentence-embedding and clustering them. Check ``pop-fem-audit-tools cluster-keywords -h`` for complete instructions on its usage.
|
||||
|
||||
|
||||
compare-codings
|
||||
---------------
|
||||
|
||||
Compare the two coding runs' archives and export the per-song disagreements, for the arbitration step to rule on. The keywords both runs assigned are settled by the comparison itself; only the keywords assigned by exactly one run are exported, each with the quotes the assigning run gave as evidence. Check ``pop-fem-audit-tools compare-codings -h`` for complete instructions on its usage.
|
||||
|
||||
|
||||
Copyright
|
||||
=========
|
||||
|
||||
|
||||
@@ -20,14 +20,6 @@ pop\_fem\_audit\_tools.commands.cluster\_keywords module
|
||||
:show-inheritance:
|
||||
:undoc-members:
|
||||
|
||||
pop\_fem\_audit\_tools.commands.compare\_codings module
|
||||
-------------------------------------------------------
|
||||
|
||||
.. automodule:: pop_fem_audit_tools.commands.compare_codings
|
||||
:members:
|
||||
:show-inheritance:
|
||||
:undoc-members:
|
||||
|
||||
pop\_fem\_audit\_tools.commands.export\_llm\_input module
|
||||
---------------------------------------------------------
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@ from types import ModuleType
|
||||
from .commands import (
|
||||
build_db_command,
|
||||
cluster_keywords_command,
|
||||
compare_codings_command,
|
||||
export_llm_input_command,
|
||||
fetch_artists_command,
|
||||
fetch_lyrics_command,
|
||||
@@ -31,7 +30,6 @@ MODULE_PROG: str = "python -m pop_fem_audit_tools"
|
||||
SUBCOMMANDS: dict[str, Callable[[list[str] | None], int]] = {
|
||||
"build-db": build_db_command,
|
||||
"cluster-keywords": cluster_keywords_command,
|
||||
"compare-codings": compare_codings_command,
|
||||
"export-llm-input": export_llm_input_command,
|
||||
"fetch-artists": fetch_artists_command,
|
||||
"fetch-lyrics": fetch_lyrics_command,
|
||||
|
||||
@@ -47,21 +47,6 @@ def cluster_keywords_command(argv: list[str] | None = None) -> int:
|
||||
return main(argv)
|
||||
|
||||
|
||||
def compare_codings_command(argv: list[str] | None = None) -> int:
|
||||
"""Compare the two coding runs and export the disagreements.
|
||||
|
||||
Writes the disagreement JSON file under the output directory,
|
||||
creating it (with parents) if it does not exist. When the
|
||||
input is rejected, the file is not written.
|
||||
|
||||
:param argv: The command-line arguments, or None for
|
||||
``sys.argv``.
|
||||
:return: The exit status: 0 on success, non-zero on failure.
|
||||
"""
|
||||
from .compare_codings import main
|
||||
return main(argv)
|
||||
|
||||
|
||||
def export_llm_input_command(argv: list[str] | None = None) -> int:
|
||||
"""Export the LLM input JSONL file from the working store.
|
||||
|
||||
|
||||
@@ -1,288 +0,0 @@
|
||||
# Tools for A Feminist Audit of Pop Music.
|
||||
# Copyright 2026 imacat. All rights reserved.
|
||||
# Authors:
|
||||
# imacat@mail.imacat.idv.tw (imacat), 2026/8/6
|
||||
"""The coding run comparison step.
|
||||
|
||||
Compares the two independent coding runs of the same songs and
|
||||
exports their per-song disagreements, writing one fixed-named
|
||||
artifact under the output directory given as the third positional
|
||||
command-line argument. The two runs are compared by their keyword
|
||||
key sets alone: a keyword assigned to a song by exactly one of the
|
||||
two runs is a disagreement, and a keyword assigned by both is
|
||||
agreed. The quotes supporting a keyword never take part in the
|
||||
comparison; they are carried along as the evidence of the disagreed
|
||||
keyword they support. The disagreements are written as a JSON file,
|
||||
as :data:`DISAGREEMENTS_JSON`, holding only the songs the two runs
|
||||
disagree on, each song's disagreed keywords wrapped in a
|
||||
"disagreements" object so that the file merges into the arbitration
|
||||
input as the per-song extra parameters; it is what the arbitration
|
||||
step of the coding procedure settles. The agreed half is not
|
||||
written; it is returned
|
||||
by :func:`compare_codings`, as the keyword names alone, for later
|
||||
steps to consume. The step is fully deterministic; no LLM call is
|
||||
made.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..utils import format_duration
|
||||
|
||||
DISAGREEMENTS_JSON: str = "disagreements.json"
|
||||
"""The disagreement JSON file's fixed name under the output
|
||||
directory."""
|
||||
|
||||
type Coding = dict[str, list[str]]
|
||||
"""The keywords assigned to one song, each with its lyric quotes."""
|
||||
|
||||
type Codings = dict[str, Coding]
|
||||
"""The coding of every song of one run, keyed by the song ID."""
|
||||
|
||||
type AgreedKeywords = dict[str, list[str]]
|
||||
"""The keywords both runs assigned to a song, keyed by the song
|
||||
ID."""
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None) -> argparse.Namespace:
|
||||
"""Parse the command-line arguments.
|
||||
|
||||
:param argv: The command-line arguments, or None for
|
||||
``sys.argv``.
|
||||
:return: The parsed arguments.
|
||||
"""
|
||||
parser: argparse.ArgumentParser = argparse.ArgumentParser(
|
||||
description="Compare the two coding runs of the same songs"
|
||||
" and export their per-song disagreements for"
|
||||
" the arbitration step.")
|
||||
parser.add_argument(
|
||||
"run_dir_1", type=Path,
|
||||
help="the first coding run's archive directory")
|
||||
parser.add_argument(
|
||||
"run_dir_2", type=Path,
|
||||
help="the second coding run's archive directory")
|
||||
parser.add_argument(
|
||||
"output_dir", type=Path,
|
||||
help="the output directory, created if missing, that"
|
||||
f" receives {DISAGREEMENTS_JSON}")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def reject_duplicate_keys(
|
||||
pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
"""Build a mapping from key-value pairs, rejecting duplicates.
|
||||
|
||||
:param pairs: The key-value pairs, in document order.
|
||||
:return: The mapping built from the pairs.
|
||||
:raises ValueError: When a key appears more than once.
|
||||
"""
|
||||
result: dict[str, Any] = {}
|
||||
key: str
|
||||
value: Any
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise ValueError(f"duplicate key \"{key}\"")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def song_number(song_id: str) -> int:
|
||||
"""Return the song number carried by a song ID.
|
||||
|
||||
:param song_id: The song ID, expected as ``song-<N>``.
|
||||
:return: The song number.
|
||||
:raises ValueError: When the song ID is not ``song-<N>``.
|
||||
"""
|
||||
prefix: str = "song-"
|
||||
if not song_id.startswith(prefix) \
|
||||
or not song_id[len(prefix):].isdigit():
|
||||
raise ValueError(
|
||||
f"id \"{song_id}\": not in \"song-<N>\" form")
|
||||
return int(song_id[len(prefix):])
|
||||
|
||||
|
||||
def load_run(run_dir: Path) -> Codings:
|
||||
"""Load and validate the coding of one coding run.
|
||||
|
||||
Every record must carry a "text" field parsing to a JSON
|
||||
object; a record that does not fails the run. Lines are split
|
||||
on the newline character alone, as the quoted lyrics may carry
|
||||
other control characters that are not line breaks here.
|
||||
|
||||
:param run_dir: The run's archive directory, containing
|
||||
``output.jsonl``.
|
||||
:return: The keyword assignments of every song, keyed by the
|
||||
song ID, in file order.
|
||||
:raises OSError: When ``output.jsonl`` cannot be read.
|
||||
:raises ValueError: When a line is not a well-formed output
|
||||
record, a "text" field does not parse to a JSON object, a
|
||||
JSON document holds a duplicate key, or a song ID appears
|
||||
more than once.
|
||||
"""
|
||||
path: Path = run_dir / "output.jsonl"
|
||||
text: str = path.read_text(encoding="utf-8")
|
||||
codings: Codings = {}
|
||||
line: str
|
||||
for line in text.split("\n"):
|
||||
if line.strip() == "":
|
||||
continue
|
||||
record: Any = json.loads(
|
||||
line, object_pairs_hook=reject_duplicate_keys)
|
||||
if not isinstance(record, dict) or "id" not in record:
|
||||
raise ValueError(
|
||||
f"{path}: record without \"id\": {line}")
|
||||
song_id: str = record["id"]
|
||||
try:
|
||||
song_number(song_id)
|
||||
except ValueError as error:
|
||||
raise ValueError(f"{path}: {error}") from error
|
||||
if "text" not in record:
|
||||
raise ValueError(
|
||||
f"{path}: id {song_id}: record without \"text\"")
|
||||
if song_id in codings:
|
||||
raise ValueError(
|
||||
f"{path}: id {song_id}: duplicate song ID")
|
||||
try:
|
||||
coding: Any = json.loads(
|
||||
record["text"],
|
||||
object_pairs_hook=reject_duplicate_keys)
|
||||
except json.JSONDecodeError as error:
|
||||
raise ValueError(
|
||||
f"{path}: id {song_id}: \"text\" does not parse as"
|
||||
f" JSON: {error}") from error
|
||||
if not isinstance(coding, dict):
|
||||
raise ValueError(
|
||||
f"{path}: id {song_id}: \"text\" does not parse to"
|
||||
" a JSON object")
|
||||
codings[song_id] = coding
|
||||
return codings
|
||||
|
||||
|
||||
def compare_codings(
|
||||
codings1: Codings,
|
||||
codings2: Codings) -> tuple[AgreedKeywords, Codings]:
|
||||
"""Compare the codings of two runs of the same songs.
|
||||
|
||||
The comparison is by keyword key set alone; the quotes never
|
||||
take part in it. A keyword assigned by both runs is agreed,
|
||||
and is returned by its name alone, without quotes. A keyword
|
||||
assigned by exactly one run is a disagreement, and carries the
|
||||
quotes of the run that assigned it. A song with no agreed
|
||||
keyword has no entry in the agreed half, and a song the two
|
||||
runs fully agree on has no entry in the disagreement half.
|
||||
Both halves are ordered by ascending song number, and every
|
||||
song's keywords lexicographically.
|
||||
|
||||
:param codings1: The first run's coding of every song, keyed
|
||||
by the song ID.
|
||||
:param codings2: The second run's coding of every song, keyed
|
||||
by the song ID.
|
||||
:return: The per-song agreed keyword names and the per-song
|
||||
disagreed keywords with their quotes.
|
||||
:raises ValueError: When the two runs do not cover exactly the
|
||||
same set of song IDs, or a song ID is not ``song-<N>``.
|
||||
"""
|
||||
only1: set[str] = set(codings1) - set(codings2)
|
||||
only2: set[str] = set(codings2) - set(codings1)
|
||||
if len(only1) > 0 or len(only2) > 0:
|
||||
raise ValueError(
|
||||
"the two runs do not cover the same songs:"
|
||||
f" {len(only1)} only in the first run"
|
||||
f" ({', '.join(sorted(only1)[:5])}),"
|
||||
f" {len(only2)} only in the second run"
|
||||
f" ({', '.join(sorted(only2)[:5])})")
|
||||
agreed: AgreedKeywords = {}
|
||||
disagreed: Codings = {}
|
||||
song_id: str
|
||||
for song_id in sorted(codings1, key=song_number):
|
||||
coding1: Coding = codings1[song_id]
|
||||
coding2: Coding = codings2[song_id]
|
||||
song_agreed: list[str] = []
|
||||
song_disagreed: Coding = {}
|
||||
keyword: str
|
||||
for keyword in sorted(set(coding1) | set(coding2)):
|
||||
if keyword in coding1 and keyword in coding2:
|
||||
song_agreed.append(keyword)
|
||||
elif keyword in coding1:
|
||||
song_disagreed[keyword] = coding1[keyword]
|
||||
else:
|
||||
song_disagreed[keyword] = coding2[keyword]
|
||||
if len(song_agreed) > 0:
|
||||
agreed[song_id] = song_agreed
|
||||
if len(song_disagreed) > 0:
|
||||
disagreed[song_id] = song_disagreed
|
||||
return agreed, disagreed
|
||||
|
||||
|
||||
def count_keywords(codings: Codings) -> int:
|
||||
"""Count the keywords of every song of a coding.
|
||||
|
||||
:param codings: The keyword assignments of every song, keyed
|
||||
by the song ID.
|
||||
:return: The total number of keyword assignments.
|
||||
"""
|
||||
return sum(len(x) for x in codings.values())
|
||||
|
||||
|
||||
def write_disagreements(path: Path,
|
||||
disagreements: Codings) -> None:
|
||||
"""Write the per-song disagreement JSON file.
|
||||
|
||||
Writes a JSON file holding a single object mapping the song ID
|
||||
to a ``{"disagreements": {...}}`` object holding the disagreed
|
||||
keywords of that song, each with the quotes of the run that
|
||||
assigned it, in the given order, UTF-8, with a trailing newline.
|
||||
The wrapper is what makes the file merge into the arbitration
|
||||
input as the per-song extra parameters, giving a "disagreements"
|
||||
sibling of the lyrics rather than loose keywords. Only the
|
||||
songs the two runs disagree on are written.
|
||||
|
||||
:param path: The path of the disagreement JSON file to write.
|
||||
:param disagreements: The disagreed keywords of every song,
|
||||
keyed by the song ID.
|
||||
:return: None.
|
||||
:raises OSError: When the file cannot be written.
|
||||
"""
|
||||
wrapped: dict[str, dict[str, Coding]] = {
|
||||
song_id: {"disagreements": coding}
|
||||
for song_id, coding in disagreements.items()}
|
||||
path.write_text(
|
||||
json.dumps(wrapped, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8")
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""Compare the two coding runs and export the disagreements.
|
||||
|
||||
Writes the disagreement JSON file under the output directory,
|
||||
creating it (with parents) if it does not exist. When the
|
||||
input is rejected, the file is not written.
|
||||
|
||||
:param argv: The command-line arguments, or None for
|
||||
``sys.argv``.
|
||||
:return: The exit status: 0 on success, non-zero on failure.
|
||||
"""
|
||||
started: float = time.monotonic()
|
||||
args: argparse.Namespace = parse_args(argv)
|
||||
codings1: Codings
|
||||
disagreements: Codings
|
||||
try:
|
||||
codings1 = load_run(args.run_dir_1)
|
||||
codings2: Codings = load_run(args.run_dir_2)
|
||||
disagreements = compare_codings(codings1, codings2)[1]
|
||||
except (OSError, ValueError) as error:
|
||||
print(f"error: {error}", file=sys.stderr)
|
||||
return 1
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
write_disagreements(
|
||||
args.output_dir / DISAGREEMENTS_JSON, disagreements)
|
||||
elapsed: str = format_duration(time.monotonic() - started)
|
||||
print(
|
||||
f"Done. {len(disagreements)} of {len(codings1)} songs"
|
||||
f" disagree on {count_keywords(disagreements)} keywords."
|
||||
f" {elapsed} elapsed.",
|
||||
file=sys.stderr)
|
||||
return 0
|
||||
@@ -11,13 +11,13 @@ Anthropic Messages Batch API exactly once, archived self-contained
|
||||
under the destination directory given by the three positional
|
||||
command-line arguments: prompt, input, archive_dir. The tool
|
||||
knows nothing about run counts or protocols: run identity --
|
||||
run1/run2, arbitration -- lives entirely in the caller's command
|
||||
list, per the research plan. A rerun of an already existing
|
||||
run1, run2, run3 -- lives entirely in the caller's command list,
|
||||
per the research plan. A rerun of an already existing
|
||||
destination requires ``--replace``; any other directory is never
|
||||
touched.
|
||||
|
||||
Comparing runs and reconciling disagreements are the responsibility
|
||||
of separate subcommands, not this one.
|
||||
Counting the runs' votes into the final table is the
|
||||
responsibility of a separate subcommand, not this one.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
|
||||
@@ -1,421 +0,0 @@
|
||||
# Tools for A Feminist Audit of Pop Music.
|
||||
# Copyright 2026 imacat. All rights reserved.
|
||||
# Authors:
|
||||
# imacat@mail.imacat.idv.tw (imacat), 2026/8/6
|
||||
"""Unit tests for the coding run comparison module."""
|
||||
import io
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import redirect_stderr
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pop_fem_audit_tools.commands import compare_codings
|
||||
|
||||
|
||||
class TestCompareCodings(unittest.TestCase):
|
||||
"""Test cases for the coding run comparison."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
"""Create a temporary directory with two run directories."""
|
||||
tmp: tempfile.TemporaryDirectory[str] \
|
||||
= tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.__dir: Path = Path(tmp.name)
|
||||
self.__run1: Path = self.__dir / "run1"
|
||||
self.__run2: Path = self.__dir / "run2"
|
||||
self.__run1.mkdir()
|
||||
self.__run2.mkdir()
|
||||
self.__output_dir: Path = self.__dir / "output"
|
||||
self.__disagreements_json: Path \
|
||||
= self.__output_dir \
|
||||
/ compare_codings.DISAGREEMENTS_JSON
|
||||
|
||||
@staticmethod
|
||||
def __write_output(
|
||||
run_dir: Path, records: list[dict[str, Any]]) -> None:
|
||||
"""Write the ``output.jsonl`` file of one run.
|
||||
|
||||
:param run_dir: The run's archive directory.
|
||||
:param records: The envelope records, in file order.
|
||||
:return: None.
|
||||
"""
|
||||
lines: list[str] = [
|
||||
json.dumps(x, ensure_ascii=False) for x in records]
|
||||
(run_dir / "output.jsonl").write_text(
|
||||
"\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
def __write_codings(
|
||||
self, run_dir: Path,
|
||||
codings: dict[str, dict[str, list[str]]]) -> None:
|
||||
"""Write the coding of every song of one run.
|
||||
|
||||
:param run_dir: The run's archive directory.
|
||||
:param codings: The keyword assignments of every song,
|
||||
keyed by the song ID, in file order.
|
||||
:return: None.
|
||||
"""
|
||||
self.__write_output(run_dir, [
|
||||
{"id": song_id, "text": json.dumps(
|
||||
coding, ensure_ascii=False),
|
||||
"stop_reason": "end_turn", "usage": {}}
|
||||
for song_id, coding in codings.items()])
|
||||
|
||||
def __run_compare(self) -> tuple[int, str]:
|
||||
"""Run the comparison with captured standard error.
|
||||
|
||||
:return: A tuple of the exit status and the standard
|
||||
error.
|
||||
"""
|
||||
argv: list[str] = [
|
||||
str(self.__run1), str(self.__run2),
|
||||
str(self.__output_dir)]
|
||||
stderr: io.StringIO = io.StringIO()
|
||||
with redirect_stderr(stderr):
|
||||
status: int = compare_codings.main(argv)
|
||||
return status, stderr.getvalue()
|
||||
|
||||
def __read_disagreements(self) -> dict[str, Any]:
|
||||
"""Read the disagreement JSON file.
|
||||
|
||||
:return: The parsed disagreements, as written, with every
|
||||
song's keywords in its "disagreements" wrapper.
|
||||
"""
|
||||
return json.loads(
|
||||
self.__disagreements_json.read_text(encoding="utf-8"))
|
||||
|
||||
def __read_keywords(self, song_id: str) -> dict[str, Any]:
|
||||
"""Read one song's disagreed keywords from the file.
|
||||
|
||||
:param song_id: The song ID.
|
||||
:return: The keywords of that song, unwrapped.
|
||||
"""
|
||||
return self.__read_disagreements()[song_id]["disagreements"]
|
||||
|
||||
def __read_disagreement_text(self) -> str:
|
||||
"""Read the disagreement JSON file verbatim.
|
||||
|
||||
:return: The file content, as written.
|
||||
"""
|
||||
return self.__disagreements_json.read_text(
|
||||
encoding="utf-8")
|
||||
|
||||
def test_keyword_in_one_run_only_disagrees(self) -> None:
|
||||
"""Test that a keyword assigned by exactly one run is a
|
||||
disagreement carrying that run's quotes."""
|
||||
self.__write_codings(self.__run1, {
|
||||
"song-1": {"shared": ["q1"], "only-1": ["q2"]}})
|
||||
self.__write_codings(self.__run2, {
|
||||
"song-1": {"shared": ["q3"], "only-2": ["q4"]}})
|
||||
status: int
|
||||
status, _ = self.__run_compare()
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(self.__read_disagreements(), {
|
||||
"song-1": {"disagreements": {
|
||||
"only-1": ["q2"], "only-2": ["q4"]}}})
|
||||
|
||||
def test_keywords_wrapped_for_merging(self) -> None:
|
||||
"""Test that each song's value is the object merged into
|
||||
the arbitration input, holding "disagreements" alone."""
|
||||
self.__write_codings(self.__run1, {
|
||||
"song-1": {"only-1": ["q"]}})
|
||||
self.__write_codings(self.__run2, {"song-1": {}})
|
||||
status: int
|
||||
status, _ = self.__run_compare()
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(
|
||||
list(self.__read_disagreements()["song-1"].keys()),
|
||||
["disagreements"])
|
||||
self.assertEqual(
|
||||
self.__read_keywords("song-1"), {"only-1": ["q"]})
|
||||
|
||||
def test_quotes_do_not_take_part_in_comparison(self) -> None:
|
||||
"""Test that identical key sets with different quotes
|
||||
yield no disagreement at all."""
|
||||
self.__write_codings(self.__run1, {
|
||||
"song-1": {"shared": ["one quote"]}})
|
||||
self.__write_codings(self.__run2, {
|
||||
"song-1": {"shared": ["another", "quote"]}})
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_compare()
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(self.__read_disagreements(), {})
|
||||
self.assertIn("0 of 1 songs disagree on 0 keywords.",
|
||||
stderr)
|
||||
|
||||
def test_agreeing_songs_omitted(self) -> None:
|
||||
"""Test that only the songs with at least one
|
||||
disagreement are written."""
|
||||
self.__write_codings(self.__run1, {
|
||||
"song-1": {"shared": ["q"]},
|
||||
"song-2": {"shared": ["q"], "only-1": ["q"]}})
|
||||
self.__write_codings(self.__run2, {
|
||||
"song-1": {"shared": ["q"]},
|
||||
"song-2": {"shared": ["q"]}})
|
||||
status: int
|
||||
status, _ = self.__run_compare()
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(
|
||||
list(self.__read_disagreements().keys()), ["song-2"])
|
||||
|
||||
def test_songs_in_ascending_song_number_order(self) -> None:
|
||||
"""Test that the songs are ordered by ascending song
|
||||
number, not by the song ID as text."""
|
||||
self.__write_codings(self.__run1, {
|
||||
"song-10": {"only-1": ["q"]},
|
||||
"song-2": {"only-1": ["q"]},
|
||||
"song-1": {"only-1": ["q"]}})
|
||||
self.__write_codings(self.__run2, {
|
||||
"song-1": {}, "song-2": {}, "song-10": {}})
|
||||
status: int
|
||||
status, _ = self.__run_compare()
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(
|
||||
list(self.__read_disagreements().keys()),
|
||||
["song-1", "song-2", "song-10"])
|
||||
|
||||
def test_keywords_in_lexicographic_order(self) -> None:
|
||||
"""Test that a song's disagreed keywords are ordered
|
||||
lexicographically, whichever run assigned them."""
|
||||
self.__write_codings(self.__run1, {
|
||||
"song-1": {"zeta": ["q"], "alpha": ["q"]}})
|
||||
self.__write_codings(self.__run2, {
|
||||
"song-1": {"mu": ["q"], "beta": ["q"]}})
|
||||
status: int
|
||||
status, _ = self.__run_compare()
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(
|
||||
list(self.__read_keywords("song-1").keys()),
|
||||
["alpha", "beta", "mu", "zeta"])
|
||||
|
||||
def test_summary_line_counts(self) -> None:
|
||||
"""Test the closing summary line's song and keyword
|
||||
counts."""
|
||||
self.__write_codings(self.__run1, {
|
||||
"song-1": {"only-1": ["q"], "also-1": ["q"]},
|
||||
"song-2": {"shared": ["q"]},
|
||||
"song-3": {}})
|
||||
self.__write_codings(self.__run2, {
|
||||
"song-1": {},
|
||||
"song-2": {"shared": ["q"]},
|
||||
"song-3": {"only-2": ["q"]}})
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_compare()
|
||||
self.assertEqual(status, 0)
|
||||
self.assertIn("Done. 2 of 3 songs disagree on 3"
|
||||
" keywords.", stderr)
|
||||
self.assertIn("elapsed.", stderr)
|
||||
|
||||
def test_control_character_in_quote_kept(self) -> None:
|
||||
"""Test that a quote holding a line-separating control
|
||||
character neither truncates the record nor is lost."""
|
||||
quote: str = "first line\u0085second line"
|
||||
self.__write_codings(self.__run1, {
|
||||
"song-1": {"only-1": [quote]}})
|
||||
self.__write_codings(self.__run2, {"song-1": {}})
|
||||
status: int
|
||||
status, _ = self.__run_compare()
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(
|
||||
self.__read_keywords("song-1"), {"only-1": [quote]})
|
||||
|
||||
def test_non_ascii_written_verbatim(self) -> None:
|
||||
"""Test that the output is UTF-8 with the non-ASCII text
|
||||
unescaped, indented by two spaces, and newline
|
||||
terminated."""
|
||||
self.__write_codings(self.__run1, {
|
||||
"song-1": {"only-1": ["女性力量"]}})
|
||||
self.__write_codings(self.__run2, {"song-1": {}})
|
||||
status: int
|
||||
status, _ = self.__run_compare()
|
||||
self.assertEqual(status, 0)
|
||||
text: str = self.__read_disagreement_text()
|
||||
self.assertIn("女性力量", text)
|
||||
self.assertIn("\n \"song-1\": {", text)
|
||||
self.assertTrue(text.endswith("}\n"))
|
||||
|
||||
def test_existing_output_dir_reused(self) -> None:
|
||||
"""Test that an already-existing output directory is
|
||||
written into rather than rejected."""
|
||||
self.__output_dir.mkdir(parents=True)
|
||||
(self.__output_dir / "keep.txt").write_text(
|
||||
"kept", encoding="utf-8")
|
||||
self.__write_codings(self.__run1, {
|
||||
"song-1": {"only-1": ["q"]}})
|
||||
self.__write_codings(self.__run2, {"song-1": {}})
|
||||
status: int
|
||||
status, _ = self.__run_compare()
|
||||
self.assertEqual(status, 0)
|
||||
self.assertTrue(self.__disagreements_json.exists())
|
||||
self.assertEqual(
|
||||
(self.__output_dir / "keep.txt").read_text(
|
||||
encoding="utf-8"),
|
||||
"kept")
|
||||
|
||||
def test_mismatched_song_sets_rejected(self) -> None:
|
||||
"""Test that runs covering different songs fail without
|
||||
writing the disagreement file."""
|
||||
self.__write_codings(self.__run1, {
|
||||
"song-1": {"only-1": ["q"]}})
|
||||
self.__write_codings(self.__run2, {
|
||||
"song-2": {"only-2": ["q"]}})
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_compare()
|
||||
self.assertEqual(status, 1)
|
||||
self.assertIn("song-1", stderr)
|
||||
self.assertIn("song-2", stderr)
|
||||
self.assertFalse(self.__disagreements_json.exists())
|
||||
|
||||
def test_extra_song_in_one_run_rejected(self) -> None:
|
||||
"""Test that one run covering an extra song fails."""
|
||||
self.__write_codings(self.__run1, {
|
||||
"song-1": {}, "song-2": {}})
|
||||
self.__write_codings(self.__run2, {"song-1": {}})
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_compare()
|
||||
self.assertEqual(status, 1)
|
||||
self.assertIn("song-2", stderr)
|
||||
self.assertFalse(self.__disagreements_json.exists())
|
||||
|
||||
def test_non_object_text_rejected(self) -> None:
|
||||
"""Test that a "text" field parsing to something other
|
||||
than a JSON object fails the run."""
|
||||
self.__write_output(self.__run1, [
|
||||
{"id": "song-1", "text": json.dumps(["only-1"])}])
|
||||
self.__write_codings(self.__run2, {"song-1": {}})
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_compare()
|
||||
self.assertEqual(status, 1)
|
||||
self.assertIn("song-1", stderr)
|
||||
self.assertFalse(self.__disagreements_json.exists())
|
||||
|
||||
def test_unparsable_text_rejected(self) -> None:
|
||||
"""Test that a "text" field that is not JSON at all, such
|
||||
as a refusal, fails the run."""
|
||||
self.__write_output(self.__run1, [
|
||||
{"id": "song-1", "text": "I cannot help with that."}])
|
||||
self.__write_codings(self.__run2, {"song-1": {}})
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_compare()
|
||||
self.assertEqual(status, 1)
|
||||
self.assertIn("song-1", stderr)
|
||||
self.assertFalse(self.__disagreements_json.exists())
|
||||
|
||||
def test_duplicate_keyword_in_text_rejected(self) -> None:
|
||||
"""Test that a "text" JSON object with a duplicate keyword
|
||||
fails the run."""
|
||||
self.__write_output(self.__run1, [
|
||||
{"id": "song-1",
|
||||
"text": '{"only-1": ["q"], "only-1": ["r"]}'}])
|
||||
self.__write_codings(self.__run2, {"song-1": {}})
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_compare()
|
||||
self.assertEqual(status, 1)
|
||||
self.assertIn("duplicate key", stderr)
|
||||
self.assertFalse(self.__disagreements_json.exists())
|
||||
|
||||
def test_duplicate_key_in_envelope_rejected(self) -> None:
|
||||
"""Test that an envelope record with a duplicate key fails
|
||||
the run."""
|
||||
(self.__run1 / "output.jsonl").write_text(
|
||||
'{"id": "song-1", "id": "song-2", "text": "{}"}\n',
|
||||
encoding="utf-8")
|
||||
self.__write_codings(self.__run2, {"song-1": {}})
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_compare()
|
||||
self.assertEqual(status, 1)
|
||||
self.assertIn("duplicate key", stderr)
|
||||
self.assertFalse(self.__disagreements_json.exists())
|
||||
|
||||
def test_duplicate_song_id_rejected(self) -> None:
|
||||
"""Test that the same song ID appearing twice in one run
|
||||
fails the run."""
|
||||
self.__write_output(self.__run1, [
|
||||
{"id": "song-1", "text": "{}"},
|
||||
{"id": "song-1", "text": "{}"}])
|
||||
self.__write_codings(self.__run2, {"song-1": {}})
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_compare()
|
||||
self.assertEqual(status, 1)
|
||||
self.assertIn("duplicate song ID", stderr)
|
||||
self.assertFalse(self.__disagreements_json.exists())
|
||||
|
||||
def test_malformed_song_id_rejected(self) -> None:
|
||||
"""Test that a song ID not in the ``song-<N>`` form fails
|
||||
the run."""
|
||||
self.__write_output(self.__run1, [
|
||||
{"id": "track-1", "text": "{}"}])
|
||||
self.__write_codings(self.__run2, {"song-1": {}})
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_compare()
|
||||
self.assertEqual(status, 1)
|
||||
self.assertIn("track-1", stderr)
|
||||
self.assertFalse(self.__disagreements_json.exists())
|
||||
|
||||
def test_missing_output_file_reported(self) -> None:
|
||||
"""Test that a run directory without ``output.jsonl``
|
||||
fails with the path in the message."""
|
||||
self.__write_codings(self.__run2, {"song-1": {}})
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_compare()
|
||||
self.assertEqual(status, 1)
|
||||
self.assertIn("output.jsonl", stderr)
|
||||
self.assertFalse(self.__disagreements_json.exists())
|
||||
|
||||
def test_blank_lines_skipped(self) -> None:
|
||||
"""Test that blank lines in ``output.jsonl`` are
|
||||
ignored."""
|
||||
(self.__run1 / "output.jsonl").write_text(
|
||||
'\n{"id": "song-1", "text": "{\\"only-1\\": [\\"q\\"]}"}'
|
||||
"\n\n",
|
||||
encoding="utf-8")
|
||||
self.__write_codings(self.__run2, {"song-1": {}})
|
||||
status: int
|
||||
status, _ = self.__run_compare()
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(
|
||||
self.__read_keywords("song-1"), {"only-1": ["q"]})
|
||||
|
||||
def test_compare_returns_agreed_keyword_names(self) -> None:
|
||||
"""Test that the comparison function returns the agreed
|
||||
keyword names alone, sorted, without their quotes."""
|
||||
codings1: dict[str, dict[str, list[str]]] = {
|
||||
"song-1": {"zeta": ["a", "b"], "alpha": ["c"],
|
||||
"only-1": ["d"]}}
|
||||
codings2: dict[str, dict[str, list[str]]] = {
|
||||
"song-1": {"zeta": ["b", "e"], "alpha": ["f"]}}
|
||||
agreed: dict[str, list[str]]
|
||||
disagreed: dict[str, dict[str, list[str]]]
|
||||
agreed, disagreed = compare_codings.compare_codings(
|
||||
codings1, codings2)
|
||||
self.assertEqual(agreed, {"song-1": ["alpha", "zeta"]})
|
||||
self.assertEqual(
|
||||
disagreed, {"song-1": {"only-1": ["d"]}})
|
||||
|
||||
def test_compare_omits_songs_without_agreement(self) -> None:
|
||||
"""Test that a song with no agreed keyword has no entry in
|
||||
the agreed half."""
|
||||
agreed: dict[str, list[str]]
|
||||
agreed = compare_codings.compare_codings(
|
||||
{"song-1": {"only-1": ["q"]}, "song-2": {"both": ["q"]}},
|
||||
{"song-1": {}, "song-2": {"both": ["q"]}})[0]
|
||||
self.assertEqual(list(agreed.keys()), ["song-2"])
|
||||
|
||||
def test_compare_rejects_different_song_sets(self) -> None:
|
||||
"""Test that the comparison function rejects runs that do
|
||||
not cover the same songs."""
|
||||
with self.assertRaises(ValueError):
|
||||
compare_codings.compare_codings(
|
||||
{"song-1": {}}, {"song-2": {}})
|
||||
Reference in New Issue
Block a user