diff --git a/docs/methodology.md b/docs/methodology.md index 6f08461..cbd7923 100644 --- a/docs/methodology.md +++ b/docs/methodology.md @@ -26,10 +26,11 @@ LLM。設計原則見 `research-plan.md`;本檔記載可重現的 ### 步驟 2-1 進池 兩次標註執行的全部關鍵字取聯集、逐字串精確去重、字典序 -排列。失敗與拒答的記錄跳過(其歌曲不貢獻關鍵字);解析 -時偵測重複鍵,違規即失敗。同時寫出處記錄(關鍵字 → -(執行別,歌曲 ID)清單),供收斂軌跡分析;出處記錄不 -進任何下游輸入。 +排列,寫成純文字檔(一行一個關鍵字)。失敗與拒答的記錄 +跳過(其歌曲不貢獻關鍵字);解析時偵測重複鍵,違規即 +失敗。同時寫出處 CSV(欄位 Keyword、Run、Song,一列一筆 +出現,依三欄排序),供收斂軌跡分析;出處記錄不進任何 +下游輸入。 ### 步驟 2-2 分群 @@ -103,9 +104,9 @@ LLM。設計原則見 `research-plan.md`;本檔記載可重現的 - **步驟 1 → 2-1**:`pool-keywords` 讀兩份執行歸檔的 `output.jsonl`(一律以換行字元 `\n` 切行——歌詞含 U+0085 等控制字元時,`str.splitlines()` 類的通用切行 - 會截斷 JSON 字串,實測踩中),輸出關鍵字 JSON 陣列 - 與出處記錄。 -- **步驟 2-1 → 2-2**:`cluster-keywords` 讀關鍵字陣列, + 會截斷 JSON 字串,實測踩中),輸出關鍵字純文字檔與 + 出處 CSV。 +- **步驟 2-1 → 2-2**:`cluster-keywords` 讀關鍵字純文字檔, 輸出具名分組(組名 → 成員詞,組名與成員皆字典序) 與記錄嵌入模型、revision、演算法參數、輸入 SHA-256 的 meta。 @@ -123,10 +124,12 @@ LLM。設計原則見 `research-plan.md`;本檔記載可重現的 保留的標籤,寫入逐首紀錄檔(歌依 ID 升序、標籤按 字典序,各標籤附其定案時的引述與來源層——共識或 仲裁)。 -- **序列化通則**:所有中間檔為 UTF-8 JSON,鍵序與元素 - 序皆依上列規則明定,無時間戳、無隨機成分;解析一律 - 偵測重複鍵,違規即失敗。給定相同的 LLM 執行輸出, - 全部交接產物逐位元組可再生。 +- **序列化通則**:所有中間檔為 UTF-8,欄序、鍵序與元素 + 序皆依上列規則明定,無時間戳、無隨機成分;JSON 解析 + 一律偵測重複鍵,違規即失敗。人讀為主的產物採純文字或 + CSV(CSV 依 RFC 4180,標題列字首大寫),機器交接檔採 + JSON。給定相同的 LLM 執行輸出,全部交接產物逐位元組 + 可再生。 ## 執行與稽核 diff --git a/tools/README.rst b/tools/README.rst index c7d1ae1..bbc0e80 100644 --- a/tools/README.rst +++ b/tools/README.rst @@ -61,6 +61,12 @@ run-llm A general command that runs specific LLM instructions with the Anthropic API. The API key must be present in the ``.env`` file in the working directory. Check ``pop-fem-audit-tools run-llm -h`` for complete instructions on its usage. +pool-keywords +------------- + +Deterministically pool the keywords of the two tagging runs into the clustering step's input, per the project's handoff contract. Check ``pop-fem-audit-tools pool-keywords -h`` for complete instructions on its usage. + + Copyright ========= diff --git a/tools/docs/source/pop_fem_audit_tools.commands.rst b/tools/docs/source/pop_fem_audit_tools.commands.rst index d6f7471..0a13fcf 100644 --- a/tools/docs/source/pop_fem_audit_tools.commands.rst +++ b/tools/docs/source/pop_fem_audit_tools.commands.rst @@ -36,6 +36,14 @@ pop\_fem\_audit\_tools.commands.fetch\_lyrics module :show-inheritance: :undoc-members: +pop\_fem\_audit\_tools.commands.pool\_keywords module +----------------------------------------------------- + +.. automodule:: pop_fem_audit_tools.commands.pool_keywords + :members: + :show-inheritance: + :undoc-members: + pop\_fem\_audit\_tools.commands.run\_llm module ----------------------------------------------- diff --git a/tools/src/pop_fem_audit_tools/__main__.py b/tools/src/pop_fem_audit_tools/__main__.py index 5c1ae36..828c788 100644 --- a/tools/src/pop_fem_audit_tools/__main__.py +++ b/tools/src/pop_fem_audit_tools/__main__.py @@ -20,6 +20,7 @@ from .commands import ( export_llm_input_command, fetch_artists_command, fetch_lyrics_command, + pool_keywords_command, run_llm_command, ) @@ -31,6 +32,7 @@ SUBCOMMANDS: dict[str, Callable[[list[str] | None], int]] = { "export-llm-input": export_llm_input_command, "fetch-artists": fetch_artists_command, "fetch-lyrics": fetch_lyrics_command, + "pool-keywords": pool_keywords_command, "run-llm": run_llm_command, } """The dispatch table from the subcommand name to the tool main.""" diff --git a/tools/src/pop_fem_audit_tools/commands/__init__.py b/tools/src/pop_fem_audit_tools/commands/__init__.py index 5a3065d..4f39df8 100644 --- a/tools/src/pop_fem_audit_tools/commands/__init__.py +++ b/tools/src/pop_fem_audit_tools/commands/__init__.py @@ -7,4 +7,5 @@ from .build_db import main as build_db_command from .export_llm_input import main as export_llm_input_command from .fetch_artists import main as fetch_artists_command from .fetch_lyrics import main as fetch_lyrics_command +from .pool_keywords import main as pool_keywords_command from .run_llm import main as run_llm_command diff --git a/tools/src/pop_fem_audit_tools/commands/pool_keywords.py b/tools/src/pop_fem_audit_tools/commands/pool_keywords.py new file mode 100644 index 0000000..5cc8d33 --- /dev/null +++ b/tools/src/pop_fem_audit_tools/commands/pool_keywords.py @@ -0,0 +1,233 @@ +# Tools for A Feminist Audit of Pop Music. +# Copyright 2026 imacat. All rights reserved. +# Authors: +# imacat@mail.imacat.idv.tw (imacat), 2026/8/5 +"""The deterministic pooler of the tagging runs' keywords. + +Pools the keywords produced by the two runs of the tagging step +into the clustering step's input, given as the third positional +command-line argument, per the project's handoff contract: the +pool is the plain union of every keyword key observed across both +runs' valid records, exact-string deduplicated and sorted, written +as a plain text file with one keyword per line. The provenance +mapping, given as the fourth positional argument, records where +every keyword came from for audit purposes as a CSV file; it never +enters any LLM input. +""" +import argparse +import csv +import json +import sys +from pathlib import Path +from typing import Any + +type Records = list[tuple[int, dict[str, Any]]] +"""The valid records of one run: (song ID, keyword mapping) pairs.""" + +type Provenance = dict[str, list[tuple[str, int]]] +"""The occurrences of every keyword, keyed by the keyword.""" + + +def parse_args(argv: list[str] | None) -> argparse.Namespace: + """Parse the command-line arguments. + + :param argv: The command-line arguments, or None for + ``sys.argv``. + :return: The parsed arguments. + """ + parser: argparse.ArgumentParser = argparse.ArgumentParser( + description="Pool the keywords of the two tagging runs" + " into the clustering step's input.") + parser.add_argument( + "run_dir_1", type=Path, + help="the first tagging run's archive directory") + parser.add_argument( + "run_dir_2", type=Path, + help="the second tagging run's archive directory") + parser.add_argument( + "pool_txt", type=Path, + help="the pooled keyword text output file, one keyword" + " per line") + parser.add_argument( + "provenance_csv", type=Path, + help="the keyword provenance CSV output file") + return parser.parse_args(argv) + + +def reject_duplicate_keys( + pairs: list[tuple[str, Any]]) -> dict[str, Any]: + """Build a mapping from key-value pairs, rejecting duplicates. + + :param pairs: The key-value pairs, in document order. + :return: The mapping built from the pairs. + :raises ValueError: When a key appears more than once. + """ + result: dict[str, Any] = {} + key: str + value: Any + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate key \"{key}\"") + result[key] = value + return result + + +def parse_song_id(item_id: str, path: Path) -> int: + """Parse the integer song ID out of an item ID. + + :param item_id: The item ID, expected as ``song-``. + :param path: The output file the ID came from, for the error + message. + :return: The parsed song ID. + :raises ValueError: When the item ID is not ``song-``. + """ + prefix: str = "song-" + if not item_id.startswith(prefix) \ + or not item_id[len(prefix):].isdigit(): + raise ValueError( + f"{path}: id \"{item_id}\": not in \"song-\" form") + return int(item_id[len(prefix):]) + + +def load_run(run_dir: Path) -> tuple[str, Records]: + """Load and validate the keyword records of one tagging run. + + Records carrying an "error" field are skipped. A "text" + field that fails to parse as JSON is a refusal and is + skipped; a "text" field that parses to anything other than a + JSON object, or whose keys are not unique, fails the run. + + :param run_dir: The run's archive directory, containing + ``output.jsonl``. + :return: The run label (the directory's basename) and its + valid records, each the song ID and the parsed keyword + mapping, in file order. + :raises OSError: When ``output.jsonl`` cannot be read. + :raises ValueError: When a line is not a well-formed output + record, or a "text" field is invalid per the rules above. + """ + path: Path = run_dir / "output.jsonl" + text: str = path.read_text(encoding="utf-8") + records: Records = [] + line: str + for line in text.split("\n"): + if line.strip() == "": + continue + record: Any = json.loads(line) + if not isinstance(record, dict) or "id" not in record: + raise ValueError( + f"{path}: record without \"id\": {line}") + if "error" in record: + continue + if "text" not in record: + raise ValueError( + f"{path}: id {record['id']}: record without" + " \"text\" or \"error\"") + song_id: int = parse_song_id(record["id"], path) + try: + keywords: Any = json.loads( + record["text"], + object_pairs_hook=reject_duplicate_keys) + except json.JSONDecodeError: + continue + if not isinstance(keywords, dict): + raise ValueError( + f"{path}: id {record['id']}: \"text\" does not" + " parse to a JSON object") + records.append((song_id, keywords)) + return run_dir.name, records + + +def pool_keywords(runs: list[tuple[str, Records]], + ) -> tuple[list[str], Provenance]: + """Pool the keywords of the given tagging runs. + + :param runs: The runs, each the run label and its valid + records (song ID, keyword mapping). + :return: The sorted, exact-string-deduplicated keyword list + and the provenance mapping from each keyword to its + occurrences, sorted by (run label, song ID). + """ + provenance: Provenance = {} + label: str + records: Records + for label, records in runs: + song_id: int + keywords: dict[str, Any] + for song_id, keywords in records: + keyword: str + for keyword in keywords: + provenance.setdefault(keyword, []).append( + (label, song_id)) + for occurrences in provenance.values(): + occurrences.sort() + return sorted(provenance.keys()), provenance + + +def write_pool(path: Path, keywords: list[str]) -> None: + """Write the pooled keyword list as the clustering input. + + Writes a plain text file, one keyword per line, in the given + order, UTF-8, LF line endings, with a trailing newline. + + :param path: The path of the pool text file to write. + :param keywords: The sorted, deduplicated keyword list. + :return: None. + """ + path.write_text( + "".join(f"{keyword}\n" for keyword in keywords), + encoding="utf-8") + + +def write_provenance(path: Path, provenance: Provenance) -> None: + """Write the keyword provenance mapping. + + Writes a CSV file with the header row + ``Keyword,Run,Song``, one row per occurrence, long format. + Rows are sorted by keyword lexicographically, then by run + label, then by song ID. + + :param path: The path of the provenance CSV file to write. + :param provenance: The provenance mapping from each keyword + to its occurrences (run label, song ID). + :return: None. + :raises OSError: When the file cannot be written. + """ + keyword: str + with open(path, "w", encoding="utf-8", newline="") as file: + writer: Any = csv.writer(file) + writer.writerow(["Keyword", "Run", "Song"]) + for keyword in sorted(provenance.keys()): + label: str + song_id: int + for label, song_id in provenance[keyword]: + writer.writerow([keyword, label, song_id]) + + +def main(argv: list[str] | None = None) -> int: + """Pool the two tagging runs' keywords for clustering. + + :param argv: The command-line arguments, or None for + ``sys.argv``. + :return: The exit status: 0 on success, non-zero on failure. + """ + args: argparse.Namespace = parse_args(argv) + run1: tuple[str, Records] + run2: tuple[str, Records] + try: + run1 = load_run(args.run_dir_1) + run2 = load_run(args.run_dir_2) + except (OSError, ValueError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + keywords: list[str] + provenance: Provenance + keywords, provenance = pool_keywords([run1, run2]) + args.pool_txt.parent.mkdir(parents=True, exist_ok=True) + args.provenance_csv.parent.mkdir(parents=True, exist_ok=True) + write_pool(args.pool_txt, keywords) + write_provenance(args.provenance_csv, provenance) + print( + f"done: {len(keywords)} keywords pooled from" + f" {len(run1[1])}+{len(run2[1])} records", file=sys.stderr) + return 0 diff --git a/tools/tests/test_pool_keywords.py b/tools/tests/test_pool_keywords.py new file mode 100644 index 0000000..43cf2a7 --- /dev/null +++ b/tools/tests/test_pool_keywords.py @@ -0,0 +1,217 @@ +# Tools for A Feminist Audit of Pop Music. +# Copyright 2026 imacat. All rights reserved. +# Authors: +# imacat@mail.imacat.idv.tw (imacat), 2026/8/5 +"""Unit tests for the keyword pooler module.""" +import csv +import io +import json +import tempfile +import unittest +from contextlib import redirect_stderr +from pathlib import Path +from typing import Any + +from pop_fem_audit_tools.commands import pool_keywords + + +class TestPoolKeywords(unittest.TestCase): + """Test cases for the keyword pooler.""" + + def setUp(self) -> None: + """Create a temporary directory with two run directories.""" + tmp: tempfile.TemporaryDirectory[str] \ + = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.__dir: Path = Path(tmp.name) + self.__run1: Path = self.__dir / "run1" + self.__run2: Path = self.__dir / "run2" + self.__run1.mkdir() + self.__run2.mkdir() + self.__pool: Path = self.__dir / "pool.txt" + self.__provenance: Path = self.__dir / "provenance.csv" + + @staticmethod + def __write_output( + run_dir: Path, records: list[dict[str, Any]]) -> None: + """Write the ``output.jsonl`` file of one run. + + :param run_dir: The run's archive directory. + :param records: The envelope records, in file order. + :return: None. + """ + lines: list[str] = [ + json.dumps(x, ensure_ascii=False) for x in records] + (run_dir / "output.jsonl").write_text( + "\n".join(lines) + "\n", encoding="utf-8") + + def __run_pool(self) -> tuple[int, str]: + """Run the pooler with the standard error captured. + + :return: A tuple of the exit status and the standard + error. + """ + stderr: io.StringIO = io.StringIO() + with redirect_stderr(stderr): + status: int = pool_keywords.main([ + str(self.__run1), str(self.__run2), + str(self.__pool), str(self.__provenance)]) + return status, stderr.getvalue() + + def __read_pool(self) -> list[str]: + """Read the pool text file. + + :return: The keyword list, one keyword per line, with the + trailing empty line from the final newline removed. + """ + lines: list[str] = self.__pool.read_text( + encoding="utf-8").split("\n") + self.assertEqual(lines[-1], "") + return lines[:-1] + + def __read_provenance(self) -> list[list[str]]: + """Read the provenance CSV file. + + :return: All rows, including the header row, in file + order. + """ + with open(self.__provenance, encoding="utf-8", + newline="") as file: + return list(csv.reader(file)) + + def test_pools_union_dedup_sorted(self) -> None: + """Test the union, dedup, and lexicographic ordering, and + the plain one-keyword-per-line pool file shape.""" + self.__write_output(self.__run1, [ + {"id": "song-1", + "text": json.dumps({"strength": 1, "shared": 1})}, + ]) + self.__write_output(self.__run2, [ + {"id": "song-3", + "text": json.dumps({"warrior": 1, "shared": 1})}, + ]) + status: int + stderr: str + status, stderr = self.__run_pool() + self.assertEqual(status, 0) + self.assertEqual( + self.__read_pool(), ["shared", "strength", "warrior"]) + self.assertIn( + "done: 3 keywords pooled from 1+1 records", stderr) + + def test_skips_error_records(self) -> None: + """Test that records carrying an "error" field are + excluded from the pool and the record count.""" + self.__write_output(self.__run1, [ + {"id": "song-1", + "text": json.dumps({"strength": 1})}, + {"id": "song-2", "error": "invalid_request_error"}, + ]) + self.__write_output(self.__run2, [ + {"id": "song-3", "text": json.dumps({"warrior": 1})}, + ]) + status: int + stderr: str + status, stderr = self.__run_pool() + self.assertEqual(status, 0) + self.assertEqual( + self.__read_pool(), ["strength", "warrior"]) + self.assertIn( + "done: 2 keywords pooled from 1+1 records", stderr) + + def test_skips_non_json_text_records(self) -> None: + """Test that a refusal, whose "text" does not parse as + JSON, is skipped rather than failing the run.""" + self.__write_output(self.__run1, [ + {"id": "song-1", + "text": json.dumps({"strength": 1})}, + {"id": "song-2", "text": "I cannot help with that."}, + ]) + self.__write_output(self.__run2, [ + {"id": "song-3", "text": json.dumps({"warrior": 1})}, + ]) + status: int + stderr: str + status, stderr = self.__run_pool() + self.assertEqual(status, 0) + self.assertEqual( + self.__read_pool(), ["strength", "warrior"]) + self.assertIn( + "done: 2 keywords pooled from 1+1 records", stderr) + + def test_duplicate_key_in_text_rejected(self) -> None: + """Test that a "text" JSON object with a duplicate key + fails the run without writing any output file.""" + self.__write_output(self.__run1, [ + {"id": "song-1", + "text": '{"strength": 1, "strength": 2}'}, + ]) + self.__write_output(self.__run2, [ + {"id": "song-3", "text": json.dumps({"warrior": 1})}, + ]) + status: int + stderr: str + status, stderr = self.__run_pool() + self.assertEqual(status, 1) + self.assertIn("duplicate key", stderr) + self.assertFalse(self.__pool.exists()) + self.assertFalse(self.__provenance.exists()) + + def test_non_object_text_rejected(self) -> None: + """Test that a "text" JSON value that is not an object + fails the run without writing any output file.""" + self.__write_output(self.__run1, [ + {"id": "song-1", "text": json.dumps(["strength"])}, + ]) + self.__write_output(self.__run2, [ + {"id": "song-3", "text": json.dumps({"warrior": 1})}, + ]) + status: int + stderr: str + status, stderr = self.__run_pool() + self.assertEqual(status, 1) + self.assertIn("song-1", stderr) + self.assertFalse(self.__pool.exists()) + self.assertFalse(self.__provenance.exists()) + + def test_provenance_content_and_ordering(self) -> None: + """Test the provenance content and its ordering: rows + sorted by keyword lexicographically, then by run label, + then by song ID.""" + self.__write_output(self.__run1, [ + {"id": "song-2", "text": json.dumps({"shared": 1})}, + {"id": "song-1", "text": json.dumps({"shared": 1})}, + ]) + self.__write_output(self.__run2, [ + {"id": "song-5", + "text": json.dumps({"shared": 1, "warrior": 1})}, + ]) + status: int + status, _ = self.__run_pool() + self.assertEqual(status, 0) + rows: list[list[str]] = self.__read_provenance() + self.assertEqual(rows[1:], [ + ["shared", "run1", "1"], + ["shared", "run1", "2"], + ["shared", "run2", "5"], + ["warrior", "run2", "5"], + ]) + + def test_provenance_file_header_and_row_count(self) -> None: + """Test that the provenance CSV file starts with the + ``Keyword,Run,Song`` header row and has exactly one row + per keyword occurrence.""" + self.__write_output(self.__run1, [ + {"id": "song-2", "text": json.dumps({"shared": 1})}, + {"id": "song-1", "text": json.dumps({"shared": 1})}, + ]) + self.__write_output(self.__run2, [ + {"id": "song-5", + "text": json.dumps({"shared": 1, "warrior": 1})}, + ]) + status: int + status, _ = self.__run_pool() + self.assertEqual(status, 0) + rows: list[list[str]] = self.__read_provenance() + self.assertEqual(rows[0], ["Keyword", "Run", "Song"]) + self.assertEqual(len(rows), 1 + 4)