Record the clustering invocation in a meta file

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

docs/methodology.md

@@ -59,14 +59,20 @@ Run、Song，一列一筆出現，依三欄排序），供收斂軌跡分析；
   實作本步，四種模型六次執行全部無法維持完整分割，
   已棄用（詳見 `decision-log.md` 2026-08-05；棄用的
   定義檔止於 git 歷史，見 `git log -- prompts/`）。
-- **產物**：五份，前綴分別標示來源與結果。
+- **產物**：六份，前綴分別標示來源與結果。
   `source-keywords.txt`（進池後的關鍵字，一行一個）與
   `source-provenance.csv`（出處）記錄進來的是什麼；
   `result-keywords.txt`（組名，一行一個）與
   `groups.csv`（欄位 Group、Keyword，一列一個成員）
   記錄算出來的分割；`keywords-to-merge.json`
   （`{"keywords": [...]}`）是實際交給模型的碼，即組名
-  加上先驗主題詞。只有最後一份含研究者的介入。
+  加上先驗主題詞——六份中只有這一份含研究者的介入。
+  `meta.json` 記錄執行本身：進池的兩份執行歸檔與其有效
+  筆數、嵌入模型與釘定 revision、分群參數與組數、外加
+  的先驗詞、關鍵字總數，以及產生數字的套件版本。凡命令
+  列上的選擇與環境事實皆在此，不記時間戳與輸入雜湊
+  ——前者使同環境重跑逐位元組可再生，後者只會重述 git
+  已保證的事。
 - **可重現性**：同一輸入、同一釘定模型、同一參數逐次
   重現。不同 CPU／BLAS 實作的浮點尾數差異可能使邊界
   詞的歸屬翻動，屬已揭露的限制；論文所用碼表逐字
@@ -130,7 +136,7 @@ Run、Song，一列一筆出現，依三欄排序），供收斂軌跡分析；
   `output.jsonl`（一律以換行字元 `\n` 切行——歌詞含
   U+0085 等控制字元時，`str.splitlines()` 類的通用切行
   會截斷 JSON 字串，實測踩中），進池後直接分群，一次
-  產出上列五份檔案。
+  產出上列六份檔案。
 - **步驟 2 → 3 輸入檔**：`export-llm-input --extras
   <定案碼表>` 自工作儲存產出步驟 3 的輸入，每筆
   `{"id": "song-<ID>", "content": <字串>}`，`content` 為

tools/src/pop_fem_audit_tools/commands/cluster_keywords.py

@@ -5,7 +5,7 @@
 """The deterministic vocabulary-building step.
 
 Goes from the two tagging runs' archives straight to the coding
-vocabulary, writing five fixed-named artifacts under the output
+vocabulary, writing six fixed-named artifacts under the output
 directory given as the third positional command-line argument.
 First, the keywords produced by the two runs of the tagging step
 are pooled into the pooled keyword list, per the project's handoff
@@ -16,18 +16,22 @@ sorted, written as a plain text file with one keyword per line, as
 every keyword came from for audit purposes as a CSV file, as
 :data:`SOURCE_PROVENANCE_CSV`; it never enters any LLM input.  Then
 the coding groups are built from the pooled keyword list by
-sentence-embedding every keyword and clustering the embeddings: the
+sentence-embedding every keyword and clustering the embeddings into
-group membership is written as a CSV file holding the clustering
+the number of groups given by the required ``--clusters``
-result alone, as :data:`RESULT_GROUPS_CSV`.  The group name
+command-line option: the group membership is written as a CSV file
-keywords alone are written as a text file, one per line, as
+holding the clustering result alone, as :data:`RESULT_GROUPS_CSV`.
-:data:`RESULT_KEYWORDS_TXT`.  The coding keyword set for
+The group name keywords alone are written as a text file, one per
+line, as :data:`RESULT_KEYWORDS_TXT`.  The coding keyword set for
 ``export-llm-input --extras`` is written as a JSON file holding the
 group name keywords plus every extra a-priori keyword the caller
 gives with the repeatable ``--extra-keyword`` command-line option,
 as :data:`KEYWORDS_TO_MERGE_JSON`; with no ``--extra-keyword``, it
 holds the group names alone.  No default extra keyword is ever
-injected; the caller supplies each one consciously.  The step is
+injected; the caller supplies each one consciously.  Finally, the
-fully deterministic; no LLM call is made.
+command-line choices and the environment that produced the numbers
+-- neither recoverable from the committed inputs and outputs -- are
+written as a JSON file, as :data:`META_JSON`.  The step is fully
+deterministic; no LLM call is made.
 """
 import argparse
 import csv
@@ -40,6 +44,8 @@ from typing import Any
 from ..utils import format_duration
 
 MODEL: str = "sentence-transformers/all-mpnet-base-v2"
+SCRIPT_VERSION: str = "cluster_keywords.py 1.0.0"
+"""The script version recorded into :data:`META_JSON`."""
 CLUSTER_EXTRA_MESSAGE: str = (
     "cluster-keywords requires the optional \"cluster\""
     " dependency group; install it with"
@@ -61,6 +67,9 @@ directory."""
 KEYWORDS_TO_MERGE_JSON: str = "keywords-to-merge.json"
 """The coding keyword set JSON file's fixed name under the output
 directory."""
+META_JSON: str = "meta.json"
+"""The run metadata JSON file's fixed name under the output
+directory."""
 
 type Records = list[tuple[int, dict[str, Any]]]
 """The valid records of one run: (song ID, keyword mapping) pairs."""
@@ -91,8 +100,8 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
         help="the output directory, created if missing, that"
              f" receives {SOURCE_KEYWORDS_TXT},"
              f" {SOURCE_PROVENANCE_CSV}, {RESULT_KEYWORDS_TXT},"
-             f" {RESULT_GROUPS_CSV}, and"
+             f" {RESULT_GROUPS_CSV}, {KEYWORDS_TO_MERGE_JSON},"
-             f" {KEYWORDS_TO_MERGE_JSON}")
+             f" and {META_JSON}")
     parser.add_argument(
         "--model", default=MODEL,
         help=f"the sentence embedding model (default \"{MODEL}\")")
@@ -101,8 +110,9 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
         help="the model revision to pin (default: unpinned)")
     parser.add_argument(
         "--clusters", type=int, required=True,
-        help="the number of clusters; required, so that the\n"
+        help="the number of clusters; required, as the study's"
-             "group count is stated on every invocation")
+             " chosen cluster count must be stated on every"
+             " invocation")
     parser.add_argument(
         "--extra-keyword", dest="extra_keywords", action="append",
         default=None,
@@ -364,6 +374,35 @@ def build_groups(keywords: list[str], embeddings: Any,
     return groups
 
 
+def collect_versions() -> dict[str, str]:
+    """Collect the versions of the running environment.
+
+    :return: The version strings, keyed by "python", "torch",
+        "transformers", "sentence-transformers",
+        "scikit-learn", and "numpy".
+    :raises RuntimeError: When the optional clustering
+        dependencies are not installed.
+    """
+    try:
+        import numpy
+        import sentence_transformers
+        import sklearn
+        import torch
+        import transformers
+    except ImportError as error:
+        raise RuntimeError(CLUSTER_EXTRA_MESSAGE) from error
+    import platform
+    return {
+        "python": platform.python_version(),
+        "torch": str(torch.__version__),
+        "transformers": str(transformers.__version__),
+        "sentence-transformers": str(
+            sentence_transformers.__version__),
+        "scikit-learn": str(sklearn.__version__),
+        "numpy": str(numpy.__version__),
+    }
+
+
 def write_groups(path: Path, groups: dict[str, list[str]]) -> None:
     """Write the group membership CSV file.
 
@@ -467,20 +506,75 @@ def write_keywords_to_merge(path: Path,
         encoding="utf-8")
 
 
+def build_meta(
+        run1: tuple[str, Records], run2: tuple[str, Records],
+        args: argparse.Namespace, keyword_count: int,
+        extra_keywords: list[str],
+        versions: dict[str, str]) -> dict[str, Any]:
+    """Build the run metadata recorded into :data:`META_JSON`.
+
+    :param run1: The first run's label and valid records.
+    :param run2: The second run's label and valid records.
+    :param args: The parsed command-line arguments.
+    :param keyword_count: The number of pooled keywords.
+    :param extra_keywords: The extra a-priori keywords given via
+        ``--extra-keyword``, in the given order.
+    :param versions: The version strings of the running
+        environment, as returned by :func:`collect_versions`.
+    :return: The metadata, in the documented key order.
+    """
+    return {
+        "script_version": SCRIPT_VERSION,
+        "source_runs": [str(args.run_dir_1), str(args.run_dir_2)],
+        "source_records": [len(run1[1]), len(run2[1])],
+        "embedding": {
+            "model": args.model, "revision": args.revision},
+        "clustering": {
+            "algorithm": "AgglomerativeClustering",
+            "linkage": "ward", "metric": "euclidean",
+            "clusters": args.clusters},
+        "extra_keywords": extra_keywords,
+        "keyword_count": keyword_count,
+        "versions": versions,
+    }
+
+
+def write_meta(path: Path, meta: dict[str, Any]) -> None:
+    """Write the run metadata JSON file.
+
+    Writes a JSON file holding the researcher's command-line
+    choices and the environment that produced the numbers --
+    neither recoverable from the committed inputs and outputs --
+    UTF-8, with a trailing newline.  No timestamp or input digest
+    is recorded, so re-running in the same environment reproduces
+    the file byte for byte.
+
+    :param path: The path of the metadata JSON file to write.
+    :param meta: The metadata to write, as built by
+        :func:`build_meta`.
+    :return: None.
+    :raises OSError: When the file cannot be written.
+    """
+    path.write_text(
+        json.dumps(meta, ensure_ascii=False, indent=1) + "\n",
+        encoding="utf-8")
+
+
 def main(argv: list[str] | None = None) -> int:
     """Pool the two tagging runs' keywords and cluster them.
 
-    Writes the five fixed-named artifacts under the output
+    Writes the six fixed-named artifacts under the output
     directory, creating it (with parents) if it does not exist:
     the pooled keyword text file and the keyword provenance CSV
     file; then the group membership CSV file, holding the
     clustering result alone; the group name keyword text file,
-    holding the same group names as a readable list; and the
+    holding the same group names as a readable list; the coding
-    coding keyword set JSON file, holding the group names plus
+    keyword set JSON file, holding the group names plus every
-    every extra keyword given via ``--extra-keyword``.  When the
+    extra keyword given via ``--extra-keyword``; and the run
-    input is rejected, or an extra keyword duplicates a group
+    metadata JSON file, recording the command-line choices and
-    name or another extra keyword, none of the five files is
+    the environment.  When the input is rejected, or an extra
-    written.
+    keyword duplicates a group name or another extra keyword,
+    none of the six files is written.
 
     :param argv: The command-line arguments, or None for
         ``sys.argv``.
@@ -505,6 +599,7 @@ def main(argv: list[str] | None = None) -> int:
         labels: Any = cluster_embeddings(embeddings, args.clusters)
         groups: dict[str, list[str]] = build_groups(
             keywords, embeddings, labels)
+        versions: dict[str, str] = collect_versions()
     except (RuntimeError, ValueError) as error:
         print(f"error: {error}", file=sys.stderr)
         return 1
@@ -525,6 +620,9 @@ def main(argv: list[str] | None = None) -> int:
     write_keywords_to_merge(
         args.output_dir / KEYWORDS_TO_MERGE_JSON, groups,
         extra_keywords)
+    meta: dict[str, Any] = build_meta(
+        run1, run2, args, len(keywords), extra_keywords, versions)
+    write_meta(args.output_dir / META_JSON, meta)
     elapsed: str = format_duration(time.monotonic() - started)
     print(
         f"done: {len(keywords)} keywords pooled from"

tools/tests/test_cluster_keywords.py

@@ -51,6 +51,8 @@ class TestClusterKeywords(unittest.TestCase):
         self.__keywords_to_merge_json: Path \
             = self.__output_dir \
             / cluster_keywords.KEYWORDS_TO_MERGE_JSON
+        self.__meta_json: Path \
+            = self.__output_dir / cluster_keywords.META_JSON
 
     @staticmethod
     def __write_output(
@@ -110,22 +112,40 @@ class TestClusterKeywords(unittest.TestCase):
                 [vectors[x] for x in keywords], dtype=np.float32)
         return fake
 
+    @staticmethod
+    def __fake_versions() -> dict[str, str]:
+        """Return a fixed version mapping test double.
+
+        :return: A fixed mapping with the same keys
+            :func:`cluster_keywords.collect_versions` returns.
+        """
+        return {
+            "python": "9.9.9",
+            "torch": "9.9.9",
+            "transformers": "9.9.9",
+            "sentence-transformers": "9.9.9",
+            "scikit-learn": "9.9.9",
+            "numpy": "9.9.9",
+        }
+
     def __run_cluster(self, extra_args: list[str] | None = None,
                       vectors: Vectors | None = None,
+                      clusters: str = "2",
                       ) -> tuple[int, str]:
-        """Run the clusterer with a fake encoder and captured
+        """Run the clusterer with a fake encoder, a fake version
-        standard error.
+        mapping, and captured standard error.
 
         :param extra_args: Extra command-line arguments appended
             after the three positional arguments.
         :param vectors: The fixed embedding to encode with; the
             two-cluster fixture is used when None.
+        :param clusters: The ``--clusters`` option value.
         :return: A tuple of the exit status and the standard
             error.
         """
         argv: list[str] = [
             str(self.__run1), str(self.__run2),
-            str(self.__output_dir)]
+            str(self.__output_dir), "--clusters", clusters]
         argv.extend(extra_args or [])
         fake: Any = self.__fake_encode(
             vectors if vectors is not None
@@ -133,11 +153,21 @@ class TestClusterKeywords(unittest.TestCase):
         stderr: io.StringIO = io.StringIO()
         with mock.patch.object(
                 cluster_keywords, "encode_keywords", fake), \
+                mock.patch.object(
+                    cluster_keywords, "collect_versions",
+                    return_value=self.__fake_versions()), \
                 redirect_stderr(stderr):
-            status: int = cluster_keywords.main(
+            status: int = cluster_keywords.main(argv)
-                argv + ["--clusters", "2"])
         return status, stderr.getvalue()
 
+    def __read_meta(self) -> dict[str, Any]:
+        """Read the run metadata JSON file.
+
+        :return: The parsed metadata.
+        """
+        return json.loads(
+            self.__meta_json.read_text(encoding="utf-8"))
+
     def __read_source_keywords(self) -> list[str]:
         """Read the pooled source keyword text file.
 
@@ -277,6 +307,7 @@ class TestClusterKeywords(unittest.TestCase):
         self.assertFalse(self.__result_groups_csv.exists())
         self.assertFalse(self.__result_keywords_txt.exists())
         self.assertFalse(self.__keywords_to_merge_json.exists())
+        self.assertFalse(self.__meta_json.exists())
 
     def test_non_object_text_rejected(self) -> None:
         """Test that a "text" JSON value that is not an object
@@ -297,6 +328,7 @@ class TestClusterKeywords(unittest.TestCase):
         self.assertFalse(self.__result_groups_csv.exists())
         self.assertFalse(self.__result_keywords_txt.exists())
         self.assertFalse(self.__keywords_to_merge_json.exists())
+        self.assertFalse(self.__meta_json.exists())
 
     def test_provenance_content_and_ordering(self) -> None:
         """Test the provenance content and its ordering: rows
@@ -478,15 +510,6 @@ class TestClusterKeywords(unittest.TestCase):
             ["a-center", "aaa-extra", "b-middle", "zzz-extra"])
         self.assertEqual(keywords, sorted(keywords))
 
-    def test_missing_clusters_option_rejected(self) -> None:
-        """Test that omitting --clusters fails the run."""
-        with self.assertRaises(SystemExit) as caught, \
-                redirect_stderr(io.StringIO()):
-            cluster_keywords.parse_args(
-                [str(self.__run1), str(self.__run2),
-                 str(self.__output_dir)])
-        self.assertNotEqual(caught.exception.code, 0)
-
     def test_duplicate_extra_keyword_rejected(self) -> None:
         """Test that repeating the same ``--extra-keyword`` value
         fails the run without writing any output file."""
@@ -510,6 +533,7 @@ class TestClusterKeywords(unittest.TestCase):
         self.assertFalse(self.__result_groups_csv.exists())
         self.assertFalse(self.__result_keywords_txt.exists())
         self.assertFalse(self.__keywords_to_merge_json.exists())
+        self.assertFalse(self.__meta_json.exists())
 
     def test_extra_keyword_duplicating_group_name_rejected(
             self) -> None:
@@ -535,3 +559,106 @@ class TestClusterKeywords(unittest.TestCase):
         self.assertFalse(self.__result_groups_csv.exists())
         self.assertFalse(self.__result_keywords_txt.exists())
         self.assertFalse(self.__keywords_to_merge_json.exists())
+        self.assertFalse(self.__meta_json.exists())
+
+    def test_meta_json_records_documented_keys(self) -> None:
+        """Test that the metadata JSON file records exactly the
+        documented keys."""
+        self.__write_output(self.__run1, [
+            {"id": "song-1", "text": json.dumps(
+                {"a-left": 1, "a-center": 1, "a-right": 1})},
+        ])
+        self.__write_output(self.__run2, [
+            {"id": "song-2", "text": json.dumps(
+                {"b-north": 1, "b-middle": 1, "b-south": 1})},
+        ])
+        status: int
+        status, _ = self.__run_cluster()
+        self.assertEqual(status, 0)
+        meta: dict[str, Any] = self.__read_meta()
+        self.assertEqual(set(meta.keys()), {
+            "script_version", "source_runs", "source_records",
+            "embedding", "clustering", "extra_keywords",
+            "keyword_count", "versions"})
+        self.assertEqual(
+            meta["script_version"], cluster_keywords.SCRIPT_VERSION)
+        self.assertEqual(
+            meta["versions"], self.__fake_versions())
+
+    def test_meta_json_source_runs_and_records(self) -> None:
+        """Test that the metadata records the given run
+        directories and their valid record counts."""
+        self.__write_output(self.__run1, [
+            {"id": "song-1", "text": json.dumps(
+                {"a-left": 1, "a-center": 1, "a-right": 1})},
+            {"id": "song-9", "error": "invalid_request_error"},
+        ])
+        self.__write_output(self.__run2, [
+            {"id": "song-2", "text": json.dumps(
+                {"b-north": 1, "b-middle": 1, "b-south": 1})},
+        ])
+        status: int
+        status, _ = self.__run_cluster()
+        self.assertEqual(status, 0)
+        meta: dict[str, Any] = self.__read_meta()
+        self.assertEqual(
+            meta["source_runs"],
+            [str(self.__run1), str(self.__run2)])
+        self.assertEqual(meta["source_records"], [1, 1])
+
+    def test_meta_json_clustering_and_embedding(self) -> None:
+        """Test that the metadata records the given cluster count
+        and the embedding model and revision."""
+        self.__write_output(self.__run1, [
+            {"id": "song-1", "text": json.dumps(
+                {"a-left": 1, "a-center": 1, "a-right": 1})},
+        ])
+        self.__write_output(self.__run2, [
+            {"id": "song-2", "text": json.dumps(
+                {"b-north": 1, "b-middle": 1, "b-south": 1})},
+        ])
+        status: int
+        status, _ = self.__run_cluster(extra_args=[
+            "--model", "some-model", "--revision", "abc123"])
+        self.assertEqual(status, 0)
+        meta: dict[str, Any] = self.__read_meta()
+        self.assertEqual(meta["clustering"]["clusters"], 2)
+        self.assertEqual(
+            meta["clustering"]["algorithm"],
+            "AgglomerativeClustering")
+        self.assertEqual(meta["embedding"], {
+            "model": "some-model", "revision": "abc123"})
+
+    def test_meta_json_extra_keywords_order_and_count(self) -> None:
+        """Test that the metadata's ``extra_keywords`` reflects the
+        given options in the given order, and ``keyword_count``
+        matches the pooled keyword count."""
+        self.__write_output(self.__run1, [
+            {"id": "song-1", "text": json.dumps(
+                {"a-left": 1, "a-center": 1, "a-right": 1})},
+        ])
+        self.__write_output(self.__run2, [
+            {"id": "song-2", "text": json.dumps(
+                {"b-north": 1, "b-middle": 1, "b-south": 1})},
+        ])
+        status: int
+        status, _ = self.__run_cluster(extra_args=[
+            "--extra-keyword", "zzz-extra",
+            "--extra-keyword", "aaa-extra"])
+        self.assertEqual(status, 0)
+        meta: dict[str, Any] = self.__read_meta()
+        self.assertEqual(
+            meta["extra_keywords"], ["zzz-extra", "aaa-extra"])
+        self.assertEqual(meta["keyword_count"], 6)
+
+    def test_missing_clusters_option_rejected(self) -> None:
+        """Test that omitting ``--clusters`` exits with an
+        argparse error."""
+        argv: list[str] = [
+            str(self.__run1), str(self.__run2),
+            str(self.__output_dir)]
+        stderr: io.StringIO = io.StringIO()
+        with redirect_stderr(stderr):
+            with self.assertRaises(SystemExit) as context:
+                cluster_keywords.parse_args(argv)
+        self.assertNotEqual(context.exception.code, 0)