From 4dc526c4b0997be6674142ae42ee3a846c604d85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BE=9D=E7=91=AA=E8=B2=93?= Date: Thu, 6 Aug 2026 06:18:09 +0800 Subject: [PATCH] Record the clustering invocation in a meta file Co-Authored-By: Claude Opus 5 (1M context) --- docs/methodology.md | 12 +- .../commands/cluster_keywords.py | 136 ++++++++++++--- tools/tests/test_cluster_keywords.py | 155 ++++++++++++++++-- 3 files changed, 267 insertions(+), 36 deletions(-) diff --git a/docs/methodology.md b/docs/methodology.md index 9800ba4..2f7b2c8 100644 --- a/docs/methodology.md +++ b/docs/methodology.md @@ -59,14 +59,20 @@ Run、Song，一列一筆出現，依三欄排序），供收斂軌跡分析； 實作本步，四種模型六次執行全部無法維持完整分割， 已棄用（詳見 `decision-log.md` 2026-08-05；棄用的 定義檔止於 git 歷史，見 `git log -- prompts/`）。 -- **產物**：五份，前綴分別標示來源與結果。 +- **產物**：六份，前綴分別標示來源與結果。 `source-keywords.txt`（進池後的關鍵字，一行一個）與 `source-provenance.csv`（出處）記錄進來的是什麼； `result-keywords.txt`（組名，一行一個）與 `groups.csv`（欄位 Group、Keyword，一列一個成員） 記錄算出來的分割；`keywords-to-merge.json` （`{"keywords": [...]}`）是實際交給模型的碼，即組名 - 加上先驗主題詞。只有最後一份含研究者的介入。 + 加上先驗主題詞——六份中只有這一份含研究者的介入。 + `meta.json` 記錄執行本身：進池的兩份執行歸檔與其有效 + 筆數、嵌入模型與釘定 revision、分群參數與組數、外加 + 的先驗詞、關鍵字總數，以及產生數字的套件版本。凡命令 + 列上的選擇與環境事實皆在此，不記時間戳與輸入雜湊 + ——前者使同環境重跑逐位元組可再生，後者只會重述 git + 已保證的事。 - **可重現性**：同一輸入、同一釘定模型、同一參數逐次 重現。不同 CPU／BLAS 實作的浮點尾數差異可能使邊界 詞的歸屬翻動，屬已揭露的限制；論文所用碼表逐字 @@ -130,7 +136,7 @@ Run、Song，一列一筆出現，依三欄排序），供收斂軌跡分析； `output.jsonl`（一律以換行字元 `\n` 切行——歌詞含 U+0085 等控制字元時，`str.splitlines()` 類的通用切行 會截斷 JSON 字串，實測踩中），進池後直接分群，一次 - 產出上列五份檔案。 + 產出上列六份檔案。 - **步驟 2 → 3 輸入檔**：`export-llm-input --extras <定案碼表>` 自工作儲存產出步驟 3 的輸入，每筆 `{"id": "song-", "content": <字串>}`，`content` 為 diff --git a/tools/src/pop_fem_audit_tools/commands/cluster_keywords.py b/tools/src/pop_fem_audit_tools/commands/cluster_keywords.py index 5005ae9..5c2eb6d 100644 --- a/tools/src/pop_fem_audit_tools/commands/cluster_keywords.py +++ b/tools/src/pop_fem_audit_tools/commands/cluster_keywords.py @@ -5,7 +5,7 @@ """The deterministic vocabulary-building step. Goes from the two tagging runs' archives straight to the coding -vocabulary, writing five fixed-named artifacts under the output +vocabulary, writing six fixed-named artifacts under the output directory given as the third positional command-line argument. First, the keywords produced by the two runs of the tagging step are pooled into the pooled keyword list, per the project's handoff @@ -16,18 +16,22 @@ sorted, written as a plain text file with one keyword per line, as every keyword came from for audit purposes as a CSV file, as :data:`SOURCE_PROVENANCE_CSV`; it never enters any LLM input. Then the coding groups are built from the pooled keyword list by -sentence-embedding every keyword and clustering the embeddings: the -group membership is written as a CSV file holding the clustering -result alone, as :data:`RESULT_GROUPS_CSV`. The group name -keywords alone are written as a text file, one per line, as -:data:`RESULT_KEYWORDS_TXT`. The coding keyword set for +sentence-embedding every keyword and clustering the embeddings into +the number of groups given by the required ``--clusters`` +command-line option: the group membership is written as a CSV file +holding the clustering result alone, as :data:`RESULT_GROUPS_CSV`. +The group name keywords alone are written as a text file, one per +line, as :data:`RESULT_KEYWORDS_TXT`. The coding keyword set for ``export-llm-input --extras`` is written as a JSON file holding the group name keywords plus every extra a-priori keyword the caller gives with the repeatable ``--extra-keyword`` command-line option, as :data:`KEYWORDS_TO_MERGE_JSON`; with no ``--extra-keyword``, it holds the group names alone. No default extra keyword is ever -injected; the caller supplies each one consciously. The step is -fully deterministic; no LLM call is made. +injected; the caller supplies each one consciously. Finally, the +command-line choices and the environment that produced the numbers +-- neither recoverable from the committed inputs and outputs -- are +written as a JSON file, as :data:`META_JSON`. The step is fully +deterministic; no LLM call is made. """ import argparse import csv @@ -40,6 +44,8 @@ from typing import Any from ..utils import format_duration MODEL: str = "sentence-transformers/all-mpnet-base-v2" +SCRIPT_VERSION: str = "cluster_keywords.py 1.0.0" +"""The script version recorded into :data:`META_JSON`.""" CLUSTER_EXTRA_MESSAGE: str = ( "cluster-keywords requires the optional \"cluster\"" " dependency group; install it with" @@ -61,6 +67,9 @@ directory.""" KEYWORDS_TO_MERGE_JSON: str = "keywords-to-merge.json" """The coding keyword set JSON file's fixed name under the output directory.""" +META_JSON: str = "meta.json" +"""The run metadata JSON file's fixed name under the output +directory.""" type Records = list[tuple[int, dict[str, Any]]] """The valid records of one run: (song ID, keyword mapping) pairs.""" @@ -91,8 +100,8 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: help="the output directory, created if missing, that" f" receives {SOURCE_KEYWORDS_TXT}," f" {SOURCE_PROVENANCE_CSV}, {RESULT_KEYWORDS_TXT}," - f" {RESULT_GROUPS_CSV}, and" - f" {KEYWORDS_TO_MERGE_JSON}") + f" {RESULT_GROUPS_CSV}, {KEYWORDS_TO_MERGE_JSON}," + f" and {META_JSON}") parser.add_argument( "--model", default=MODEL, help=f"the sentence embedding model (default \"{MODEL}\")") @@ -101,8 +110,9 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: help="the model revision to pin (default: unpinned)") parser.add_argument( "--clusters", type=int, required=True, - help="the number of clusters; required, so that the\n" - "group count is stated on every invocation") + help="the number of clusters; required, as the study's" + " chosen cluster count must be stated on every" + " invocation") parser.add_argument( "--extra-keyword", dest="extra_keywords", action="append", default=None, @@ -364,6 +374,35 @@ def build_groups(keywords: list[str], embeddings: Any, return groups +def collect_versions() -> dict[str, str]: + """Collect the versions of the running environment. + + :return: The version strings, keyed by "python", "torch", + "transformers", "sentence-transformers", + "scikit-learn", and "numpy". + :raises RuntimeError: When the optional clustering + dependencies are not installed. + """ + try: + import numpy + import sentence_transformers + import sklearn + import torch + import transformers + except ImportError as error: + raise RuntimeError(CLUSTER_EXTRA_MESSAGE) from error + import platform + return { + "python": platform.python_version(), + "torch": str(torch.__version__), + "transformers": str(transformers.__version__), + "sentence-transformers": str( + sentence_transformers.__version__), + "scikit-learn": str(sklearn.__version__), + "numpy": str(numpy.__version__), + } + + def write_groups(path: Path, groups: dict[str, list[str]]) -> None: """Write the group membership CSV file. @@ -467,20 +506,75 @@ def write_keywords_to_merge(path: Path, encoding="utf-8") +def build_meta( + run1: tuple[str, Records], run2: tuple[str, Records], + args: argparse.Namespace, keyword_count: int, + extra_keywords: list[str], + versions: dict[str, str]) -> dict[str, Any]: + """Build the run metadata recorded into :data:`META_JSON`. + + :param run1: The first run's label and valid records. + :param run2: The second run's label and valid records. + :param args: The parsed command-line arguments. + :param keyword_count: The number of pooled keywords. + :param extra_keywords: The extra a-priori keywords given via + ``--extra-keyword``, in the given order. + :param versions: The version strings of the running + environment, as returned by :func:`collect_versions`. + :return: The metadata, in the documented key order. + """ + return { + "script_version": SCRIPT_VERSION, + "source_runs": [str(args.run_dir_1), str(args.run_dir_2)], + "source_records": [len(run1[1]), len(run2[1])], + "embedding": { + "model": args.model, "revision": args.revision}, + "clustering": { + "algorithm": "AgglomerativeClustering", + "linkage": "ward", "metric": "euclidean", + "clusters": args.clusters}, + "extra_keywords": extra_keywords, + "keyword_count": keyword_count, + "versions": versions, + } + + +def write_meta(path: Path, meta: dict[str, Any]) -> None: + """Write the run metadata JSON file. + + Writes a JSON file holding the researcher's command-line + choices and the environment that produced the numbers -- + neither recoverable from the committed inputs and outputs -- + UTF-8, with a trailing newline. No timestamp or input digest + is recorded, so re-running in the same environment reproduces + the file byte for byte. + + :param path: The path of the metadata JSON file to write. + :param meta: The metadata to write, as built by + :func:`build_meta`. + :return: None. + :raises OSError: When the file cannot be written. + """ + path.write_text( + json.dumps(meta, ensure_ascii=False, indent=1) + "\n", + encoding="utf-8") + + def main(argv: list[str] | None = None) -> int: """Pool the two tagging runs' keywords and cluster them. - Writes the five fixed-named artifacts under the output + Writes the six fixed-named artifacts under the output directory, creating it (with parents) if it does not exist: the pooled keyword text file and the keyword provenance CSV file; then the group membership CSV file, holding the clustering result alone; the group name keyword text file, - holding the same group names as a readable list; and the - coding keyword set JSON file, holding the group names plus - every extra keyword given via ``--extra-keyword``. When the - input is rejected, or an extra keyword duplicates a group - name or another extra keyword, none of the five files is - written. + holding the same group names as a readable list; the coding + keyword set JSON file, holding the group names plus every + extra keyword given via ``--extra-keyword``; and the run + metadata JSON file, recording the command-line choices and + the environment. When the input is rejected, or an extra + keyword duplicates a group name or another extra keyword, + none of the six files is written. :param argv: The command-line arguments, or None for ``sys.argv``. @@ -505,6 +599,7 @@ def main(argv: list[str] | None = None) -> int: labels: Any = cluster_embeddings(embeddings, args.clusters) groups: dict[str, list[str]] = build_groups( keywords, embeddings, labels) + versions: dict[str, str] = collect_versions() except (RuntimeError, ValueError) as error: print(f"error: {error}", file=sys.stderr) return 1 @@ -525,6 +620,9 @@ def main(argv: list[str] | None = None) -> int: write_keywords_to_merge( args.output_dir / KEYWORDS_TO_MERGE_JSON, groups, extra_keywords) + meta: dict[str, Any] = build_meta( + run1, run2, args, len(keywords), extra_keywords, versions) + write_meta(args.output_dir / META_JSON, meta) elapsed: str = format_duration(time.monotonic() - started) print( f"done: {len(keywords)} keywords pooled from" diff --git a/tools/tests/test_cluster_keywords.py b/tools/tests/test_cluster_keywords.py index 15ee7e0..3f9c67e 100644 --- a/tools/tests/test_cluster_keywords.py +++ b/tools/tests/test_cluster_keywords.py @@ -51,6 +51,8 @@ class TestClusterKeywords(unittest.TestCase): self.__keywords_to_merge_json: Path \ = self.__output_dir \ / cluster_keywords.KEYWORDS_TO_MERGE_JSON + self.__meta_json: Path \ + = self.__output_dir / cluster_keywords.META_JSON @staticmethod def __write_output( @@ -110,22 +112,40 @@ class TestClusterKeywords(unittest.TestCase): [vectors[x] for x in keywords], dtype=np.float32) return fake + @staticmethod + def __fake_versions() -> dict[str, str]: + """Return a fixed version mapping test double. + + :return: A fixed mapping with the same keys + :func:`cluster_keywords.collect_versions` returns. + """ + return { + "python": "9.9.9", + "torch": "9.9.9", + "transformers": "9.9.9", + "sentence-transformers": "9.9.9", + "scikit-learn": "9.9.9", + "numpy": "9.9.9", + } + def __run_cluster(self, extra_args: list[str] | None = None, vectors: Vectors | None = None, + clusters: str = "2", ) -> tuple[int, str]: - """Run the clusterer with a fake encoder and captured - standard error. + """Run the clusterer with a fake encoder, a fake version + mapping, and captured standard error. :param extra_args: Extra command-line arguments appended after the three positional arguments. :param vectors: The fixed embedding to encode with; the two-cluster fixture is used when None. + :param clusters: The ``--clusters`` option value. :return: A tuple of the exit status and the standard error. """ argv: list[str] = [ str(self.__run1), str(self.__run2), - str(self.__output_dir)] + str(self.__output_dir), "--clusters", clusters] argv.extend(extra_args or []) fake: Any = self.__fake_encode( vectors if vectors is not None @@ -133,11 +153,21 @@ class TestClusterKeywords(unittest.TestCase): stderr: io.StringIO = io.StringIO() with mock.patch.object( cluster_keywords, "encode_keywords", fake), \ + mock.patch.object( + cluster_keywords, "collect_versions", + return_value=self.__fake_versions()), \ redirect_stderr(stderr): - status: int = cluster_keywords.main( - argv + ["--clusters", "2"]) + status: int = cluster_keywords.main(argv) return status, stderr.getvalue() + def __read_meta(self) -> dict[str, Any]: + """Read the run metadata JSON file. + + :return: The parsed metadata. + """ + return json.loads( + self.__meta_json.read_text(encoding="utf-8")) + def __read_source_keywords(self) -> list[str]: """Read the pooled source keyword text file. @@ -277,6 +307,7 @@ class TestClusterKeywords(unittest.TestCase): self.assertFalse(self.__result_groups_csv.exists()) self.assertFalse(self.__result_keywords_txt.exists()) self.assertFalse(self.__keywords_to_merge_json.exists()) + self.assertFalse(self.__meta_json.exists()) def test_non_object_text_rejected(self) -> None: """Test that a "text" JSON value that is not an object @@ -297,6 +328,7 @@ class TestClusterKeywords(unittest.TestCase): self.assertFalse(self.__result_groups_csv.exists()) self.assertFalse(self.__result_keywords_txt.exists()) self.assertFalse(self.__keywords_to_merge_json.exists()) + self.assertFalse(self.__meta_json.exists()) def test_provenance_content_and_ordering(self) -> None: """Test the provenance content and its ordering: rows @@ -478,15 +510,6 @@ class TestClusterKeywords(unittest.TestCase): ["a-center", "aaa-extra", "b-middle", "zzz-extra"]) self.assertEqual(keywords, sorted(keywords)) - def test_missing_clusters_option_rejected(self) -> None: - """Test that omitting --clusters fails the run.""" - with self.assertRaises(SystemExit) as caught, \ - redirect_stderr(io.StringIO()): - cluster_keywords.parse_args( - [str(self.__run1), str(self.__run2), - str(self.__output_dir)]) - self.assertNotEqual(caught.exception.code, 0) - def test_duplicate_extra_keyword_rejected(self) -> None: """Test that repeating the same ``--extra-keyword`` value fails the run without writing any output file.""" @@ -510,6 +533,7 @@ class TestClusterKeywords(unittest.TestCase): self.assertFalse(self.__result_groups_csv.exists()) self.assertFalse(self.__result_keywords_txt.exists()) self.assertFalse(self.__keywords_to_merge_json.exists()) + self.assertFalse(self.__meta_json.exists()) def test_extra_keyword_duplicating_group_name_rejected( self) -> None: @@ -535,3 +559,106 @@ class TestClusterKeywords(unittest.TestCase): self.assertFalse(self.__result_groups_csv.exists()) self.assertFalse(self.__result_keywords_txt.exists()) self.assertFalse(self.__keywords_to_merge_json.exists()) + self.assertFalse(self.__meta_json.exists()) + + def test_meta_json_records_documented_keys(self) -> None: + """Test that the metadata JSON file records exactly the + documented keys.""" + self.__write_output(self.__run1, [ + {"id": "song-1", "text": json.dumps( + {"a-left": 1, "a-center": 1, "a-right": 1})}, + ]) + self.__write_output(self.__run2, [ + {"id": "song-2", "text": json.dumps( + {"b-north": 1, "b-middle": 1, "b-south": 1})}, + ]) + status: int + status, _ = self.__run_cluster() + self.assertEqual(status, 0) + meta: dict[str, Any] = self.__read_meta() + self.assertEqual(set(meta.keys()), { + "script_version", "source_runs", "source_records", + "embedding", "clustering", "extra_keywords", + "keyword_count", "versions"}) + self.assertEqual( + meta["script_version"], cluster_keywords.SCRIPT_VERSION) + self.assertEqual( + meta["versions"], self.__fake_versions()) + + def test_meta_json_source_runs_and_records(self) -> None: + """Test that the metadata records the given run + directories and their valid record counts.""" + self.__write_output(self.__run1, [ + {"id": "song-1", "text": json.dumps( + {"a-left": 1, "a-center": 1, "a-right": 1})}, + {"id": "song-9", "error": "invalid_request_error"}, + ]) + self.__write_output(self.__run2, [ + {"id": "song-2", "text": json.dumps( + {"b-north": 1, "b-middle": 1, "b-south": 1})}, + ]) + status: int + status, _ = self.__run_cluster() + self.assertEqual(status, 0) + meta: dict[str, Any] = self.__read_meta() + self.assertEqual( + meta["source_runs"], + [str(self.__run1), str(self.__run2)]) + self.assertEqual(meta["source_records"], [1, 1]) + + def test_meta_json_clustering_and_embedding(self) -> None: + """Test that the metadata records the given cluster count + and the embedding model and revision.""" + self.__write_output(self.__run1, [ + {"id": "song-1", "text": json.dumps( + {"a-left": 1, "a-center": 1, "a-right": 1})}, + ]) + self.__write_output(self.__run2, [ + {"id": "song-2", "text": json.dumps( + {"b-north": 1, "b-middle": 1, "b-south": 1})}, + ]) + status: int + status, _ = self.__run_cluster(extra_args=[ + "--model", "some-model", "--revision", "abc123"]) + self.assertEqual(status, 0) + meta: dict[str, Any] = self.__read_meta() + self.assertEqual(meta["clustering"]["clusters"], 2) + self.assertEqual( + meta["clustering"]["algorithm"], + "AgglomerativeClustering") + self.assertEqual(meta["embedding"], { + "model": "some-model", "revision": "abc123"}) + + def test_meta_json_extra_keywords_order_and_count(self) -> None: + """Test that the metadata's ``extra_keywords`` reflects the + given options in the given order, and ``keyword_count`` + matches the pooled keyword count.""" + self.__write_output(self.__run1, [ + {"id": "song-1", "text": json.dumps( + {"a-left": 1, "a-center": 1, "a-right": 1})}, + ]) + self.__write_output(self.__run2, [ + {"id": "song-2", "text": json.dumps( + {"b-north": 1, "b-middle": 1, "b-south": 1})}, + ]) + status: int + status, _ = self.__run_cluster(extra_args=[ + "--extra-keyword", "zzz-extra", + "--extra-keyword", "aaa-extra"]) + self.assertEqual(status, 0) + meta: dict[str, Any] = self.__read_meta() + self.assertEqual( + meta["extra_keywords"], ["zzz-extra", "aaa-extra"]) + self.assertEqual(meta["keyword_count"], 6) + + def test_missing_clusters_option_rejected(self) -> None: + """Test that omitting ``--clusters`` exits with an + argparse error.""" + argv: list[str] = [ + str(self.__run1), str(self.__run2), + str(self.__output_dir)] + stderr: io.StringIO = io.StringIO() + with redirect_stderr(stderr): + with self.assertRaises(SystemExit) as context: + cluster_keywords.parse_args(argv) + self.assertNotEqual(context.exception.code, 0)