imacat/pop-fem-audit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Record the clustering invocation in a meta file

Browse Source 
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
imacat
2026-08-17 22:38:31 +08:00
co-authored by Claude Opus 5
parent 15f1843cde
commit 4dc526c4b0
3 changed files with 267 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/src/pop_fem_audit_tools/commands/cluster_keywords.py
+117 -19
View File
@@ -5,7 +5,7 @@
"""The deterministic vocabulary-building step.
Goes from the two tagging runs' archives straight to the coding
vocabulary, writing five fixed-named artifacts under the output
vocabulary, writing six fixed-named artifacts under the output
directory given as the third positional command-line argument.
First, the keywords produced by the two runs of the tagging step
are pooled into the pooled keyword list, per the project's handoff
@@ -16,18 +16,22 @@ sorted, written as a plain text file with one keyword per line, as
every keyword came from for audit purposes as a CSV file, as
:data:`SOURCE_PROVENANCE_CSV`; it never enters any LLM input. Then
the coding groups are built from the pooled keyword list by
sentence-embedding every keyword and clustering the embeddings: the
group membership is written as a CSV file holding the clustering
result alone, as :data:`RESULT_GROUPS_CSV`. The group name
keywords alone are written as a text file, one per line, as
:data:`RESULT_KEYWORDS_TXT`. The coding keyword set for
sentence-embedding every keyword and clustering the embeddings into
the number of groups given by the required ``--clusters``
command-line option: the group membership is written as a CSV file
holding the clustering result alone, as :data:`RESULT_GROUPS_CSV`.
The group name keywords alone are written as a text file, one per
line, as :data:`RESULT_KEYWORDS_TXT`. The coding keyword set for
``export-llm-input --extras`` is written as a JSON file holding the
group name keywords plus every extra a-priori keyword the caller
gives with the repeatable ``--extra-keyword`` command-line option,
as :data:`KEYWORDS_TO_MERGE_JSON`; with no ``--extra-keyword``, it
holds the group names alone. No default extra keyword is ever
injected; the caller supplies each one consciously. The step is
fully deterministic; no LLM call is made.
injected; the caller supplies each one consciously. Finally, the
command-line choices and the environment that produced the numbers
-- neither recoverable from the committed inputs and outputs -- are
written as a JSON file, as :data:`META_JSON`. The step is fully
deterministic; no LLM call is made.
"""
import argparse
import csv
@@ -40,6 +44,8 @@ from typing import Any
from ..utils import format_duration
MODEL: str = "sentence-transformers/all-mpnet-base-v2"
SCRIPT_VERSION: str = "cluster_keywords.py 1.0.0"
"""The script version recorded into :data:`META_JSON`."""
CLUSTER_EXTRA_MESSAGE: str = (
"cluster-keywords requires the optional \"cluster\""
" dependency group; install it with"
@@ -61,6 +67,9 @@ directory."""
KEYWORDS_TO_MERGE_JSON: str = "keywords-to-merge.json"
"""The coding keyword set JSON file's fixed name under the output
directory."""
META_JSON: str = "meta.json"
"""The run metadata JSON file's fixed name under the output
directory."""
type Records = list[tuple[int, dict[str, Any]]]
"""The valid records of one run: (song ID, keyword mapping) pairs."""
@@ -91,8 +100,8 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
help="the output directory, created if missing, that"
f" receives {SOURCE_KEYWORDS_TXT},"
f" {SOURCE_PROVENANCE_CSV}, {RESULT_KEYWORDS_TXT},"
f" {RESULT_GROUPS_CSV}, and"
f" {KEYWORDS_TO_MERGE_JSON}")
f" {RESULT_GROUPS_CSV}, {KEYWORDS_TO_MERGE_JSON},"
f" and {META_JSON}")
parser.add_argument(
"--model", default=MODEL,
help=f"the sentence embedding model (default \"{MODEL}\")")
@@ -101,8 +110,9 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
help="the model revision to pin (default: unpinned)")
parser.add_argument(
"--clusters", type=int, required=True,
help="the number of clusters; required, so that the\n"
"group count is stated on every invocation")
help="the number of clusters; required, as the study's"
" chosen cluster count must be stated on every"
" invocation")
parser.add_argument(
"--extra-keyword", dest="extra_keywords", action="append",
default=None,
@@ -364,6 +374,35 @@ def build_groups(keywords: list[str], embeddings: Any,
return groups
def collect_versions() -> dict[str, str]:
"""Collect the versions of the running environment.
:return: The version strings, keyed by "python", "torch",
"transformers", "sentence-transformers",
"scikit-learn", and "numpy".
:raises RuntimeError: When the optional clustering
dependencies are not installed.
"""
try:
import numpy
import sentence_transformers
import sklearn
import torch
import transformers
except ImportError as error:
raise RuntimeError(CLUSTER_EXTRA_MESSAGE) from error
import platform
return {
"python": platform.python_version(),
"torch": str(torch.__version__),
"transformers": str(transformers.__version__),
"sentence-transformers": str(
sentence_transformers.__version__),
"scikit-learn": str(sklearn.__version__),
"numpy": str(numpy.__version__),
}
def write_groups(path: Path, groups: dict[str, list[str]]) -> None:
"""Write the group membership CSV file.
@@ -467,20 +506,75 @@ def write_keywords_to_merge(path: Path,
encoding="utf-8")
def build_meta(
run1: tuple[str, Records], run2: tuple[str, Records],
args: argparse.Namespace, keyword_count: int,
extra_keywords: list[str],
versions: dict[str, str]) -> dict[str, Any]:
"""Build the run metadata recorded into :data:`META_JSON`.
:param run1: The first run's label and valid records.
:param run2: The second run's label and valid records.
:param args: The parsed command-line arguments.
:param keyword_count: The number of pooled keywords.
:param extra_keywords: The extra a-priori keywords given via
``--extra-keyword``, in the given order.
:param versions: The version strings of the running
environment, as returned by :func:`collect_versions`.
:return: The metadata, in the documented key order.
"""
return {
"script_version": SCRIPT_VERSION,
"source_runs": [str(args.run_dir_1), str(args.run_dir_2)],
"source_records": [len(run1[1]), len(run2[1])],
"embedding": {
"model": args.model, "revision": args.revision},
"clustering": {
"algorithm": "AgglomerativeClustering",
"linkage": "ward", "metric": "euclidean",
"clusters": args.clusters},
"extra_keywords": extra_keywords,
"keyword_count": keyword_count,
"versions": versions,
}
def write_meta(path: Path, meta: dict[str, Any]) -> None:
"""Write the run metadata JSON file.
Writes a JSON file holding the researcher's command-line
choices and the environment that produced the numbers --
neither recoverable from the committed inputs and outputs --
UTF-8, with a trailing newline. No timestamp or input digest
is recorded, so re-running in the same environment reproduces
the file byte for byte.
:param path: The path of the metadata JSON file to write.
:param meta: The metadata to write, as built by
:func:`build_meta`.
:return: None.
:raises OSError: When the file cannot be written.
"""
path.write_text(
json.dumps(meta, ensure_ascii=False, indent=1) + "\n",
encoding="utf-8")
def main(argv: list[str] | None = None) -> int:
"""Pool the two tagging runs' keywords and cluster them.
Writes the five fixed-named artifacts under the output
Writes the six fixed-named artifacts under the output
directory, creating it (with parents) if it does not exist:
the pooled keyword text file and the keyword provenance CSV
file; then the group membership CSV file, holding the
clustering result alone; the group name keyword text file,
holding the same group names as a readable list; and the
coding keyword set JSON file, holding the group names plus
every extra keyword given via ``--extra-keyword``. When the
input is rejected, or an extra keyword duplicates a group
name or another extra keyword, none of the five files is
written.
holding the same group names as a readable list; the coding
keyword set JSON file, holding the group names plus every
extra keyword given via ``--extra-keyword``; and the run
metadata JSON file, recording the command-line choices and
the environment. When the input is rejected, or an extra
keyword duplicates a group name or another extra keyword,
none of the six files is written.
:param argv: The command-line arguments, or None for
``sys.argv``.
@@ -505,6 +599,7 @@ def main(argv: list[str] | None = None) -> int:
labels: Any = cluster_embeddings(embeddings, args.clusters)
groups: dict[str, list[str]] = build_groups(
keywords, embeddings, labels)
versions: dict[str, str] = collect_versions()
except (RuntimeError, ValueError) as error:
print(f"error: {error}", file=sys.stderr)
return 1
@@ -525,6 +620,9 @@ def main(argv: list[str] | None = None) -> int:
write_keywords_to_merge(
args.output_dir / KEYWORDS_TO_MERGE_JSON, groups,
extra_keywords)
meta: dict[str, Any] = build_meta(
run1, run2, args, len(keywords), extra_keywords, versions)
write_meta(args.output_dir / META_JSON, meta)
elapsed: str = format_duration(time.monotonic() - started)
print(
f"done: {len(keywords)} keywords pooled from"
tools/tests/test_cluster_keywords.py
+141 -14
View File
@@ -51,6 +51,8 @@ class TestClusterKeywords(unittest.TestCase):
self.__keywords_to_merge_json: Path \
= self.__output_dir \
/ cluster_keywords.KEYWORDS_TO_MERGE_JSON
self.__meta_json: Path \
= self.__output_dir / cluster_keywords.META_JSON
@staticmethod
def __write_output(
@@ -110,22 +112,40 @@ class TestClusterKeywords(unittest.TestCase):
[vectors[x] for x in keywords], dtype=np.float32)
return fake
@staticmethod
def __fake_versions() -> dict[str, str]:
"""Return a fixed version mapping test double.
:return: A fixed mapping with the same keys
:func:`cluster_keywords.collect_versions` returns.
"""
return {
"python": "9.9.9",
"torch": "9.9.9",
"transformers": "9.9.9",
"sentence-transformers": "9.9.9",
"scikit-learn": "9.9.9",
"numpy": "9.9.9",
}
def __run_cluster(self, extra_args: list[str] | None = None,
vectors: Vectors | None = None,
clusters: str = "2",
) -> tuple[int, str]:
"""Run the clusterer with a fake encoder and captured
standard error.
"""Run the clusterer with a fake encoder, a fake version
mapping, and captured standard error.
:param extra_args: Extra command-line arguments appended
after the three positional arguments.
:param vectors: The fixed embedding to encode with; the
two-cluster fixture is used when None.
:param clusters: The ``--clusters`` option value.
:return: A tuple of the exit status and the standard
error.
"""
argv: list[str] = [
str(self.__run1), str(self.__run2),
str(self.__output_dir)]
str(self.__output_dir), "--clusters", clusters]
argv.extend(extra_args or [])
fake: Any = self.__fake_encode(
vectors if vectors is not None
@@ -133,11 +153,21 @@ class TestClusterKeywords(unittest.TestCase):
stderr: io.StringIO = io.StringIO()
with mock.patch.object(
cluster_keywords, "encode_keywords", fake), \
mock.patch.object(
cluster_keywords, "collect_versions",
return_value=self.__fake_versions()), \
redirect_stderr(stderr):
status: int = cluster_keywords.main(
argv + ["--clusters", "2"])
status: int = cluster_keywords.main(argv)
return status, stderr.getvalue()
def __read_meta(self) -> dict[str, Any]:
"""Read the run metadata JSON file.
:return: The parsed metadata.
"""
return json.loads(
self.__meta_json.read_text(encoding="utf-8"))
def __read_source_keywords(self) -> list[str]:
"""Read the pooled source keyword text file.
@@ -277,6 +307,7 @@ class TestClusterKeywords(unittest.TestCase):
self.assertFalse(self.__result_groups_csv.exists())
self.assertFalse(self.__result_keywords_txt.exists())
self.assertFalse(self.__keywords_to_merge_json.exists())
self.assertFalse(self.__meta_json.exists())
def test_non_object_text_rejected(self) -> None:
"""Test that a "text" JSON value that is not an object
@@ -297,6 +328,7 @@ class TestClusterKeywords(unittest.TestCase):
self.assertFalse(self.__result_groups_csv.exists())
self.assertFalse(self.__result_keywords_txt.exists())
self.assertFalse(self.__keywords_to_merge_json.exists())
self.assertFalse(self.__meta_json.exists())
def test_provenance_content_and_ordering(self) -> None:
"""Test the provenance content and its ordering: rows
@@ -478,15 +510,6 @@ class TestClusterKeywords(unittest.TestCase):
["a-center", "aaa-extra", "b-middle", "zzz-extra"])
self.assertEqual(keywords, sorted(keywords))
def test_missing_clusters_option_rejected(self) -> None:
"""Test that omitting --clusters fails the run."""
with self.assertRaises(SystemExit) as caught, \
redirect_stderr(io.StringIO()):
cluster_keywords.parse_args(
[str(self.__run1), str(self.__run2),
str(self.__output_dir)])
self.assertNotEqual(caught.exception.code, 0)
def test_duplicate_extra_keyword_rejected(self) -> None:
"""Test that repeating the same ``--extra-keyword`` value
fails the run without writing any output file."""
@@ -510,6 +533,7 @@ class TestClusterKeywords(unittest.TestCase):
self.assertFalse(self.__result_groups_csv.exists())
self.assertFalse(self.__result_keywords_txt.exists())
self.assertFalse(self.__keywords_to_merge_json.exists())
self.assertFalse(self.__meta_json.exists())
def test_extra_keyword_duplicating_group_name_rejected(
self) -> None:
@@ -535,3 +559,106 @@ class TestClusterKeywords(unittest.TestCase):
self.assertFalse(self.__result_groups_csv.exists())
self.assertFalse(self.__result_keywords_txt.exists())
self.assertFalse(self.__keywords_to_merge_json.exists())
self.assertFalse(self.__meta_json.exists())
def test_meta_json_records_documented_keys(self) -> None:
"""Test that the metadata JSON file records exactly the
documented keys."""
self.__write_output(self.__run1, [
{"id": "song-1", "text": json.dumps(
{"a-left": 1, "a-center": 1, "a-right": 1})},
])
self.__write_output(self.__run2, [
{"id": "song-2", "text": json.dumps(
{"b-north": 1, "b-middle": 1, "b-south": 1})},
])
status: int
status, _ = self.__run_cluster()
self.assertEqual(status, 0)
meta: dict[str, Any] = self.__read_meta()
self.assertEqual(set(meta.keys()), {
"script_version", "source_runs", "source_records",
"embedding", "clustering", "extra_keywords",
"keyword_count", "versions"})
self.assertEqual(
meta["script_version"], cluster_keywords.SCRIPT_VERSION)
self.assertEqual(
meta["versions"], self.__fake_versions())
def test_meta_json_source_runs_and_records(self) -> None:
"""Test that the metadata records the given run
directories and their valid record counts."""
self.__write_output(self.__run1, [
{"id": "song-1", "text": json.dumps(
{"a-left": 1, "a-center": 1, "a-right": 1})},
{"id": "song-9", "error": "invalid_request_error"},
])
self.__write_output(self.__run2, [
{"id": "song-2", "text": json.dumps(
{"b-north": 1, "b-middle": 1, "b-south": 1})},
])
status: int
status, _ = self.__run_cluster()
self.assertEqual(status, 0)
meta: dict[str, Any] = self.__read_meta()
self.assertEqual(
meta["source_runs"],
[str(self.__run1), str(self.__run2)])
self.assertEqual(meta["source_records"], [1, 1])
def test_meta_json_clustering_and_embedding(self) -> None:
"""Test that the metadata records the given cluster count
and the embedding model and revision."""
self.__write_output(self.__run1, [
{"id": "song-1", "text": json.dumps(
{"a-left": 1, "a-center": 1, "a-right": 1})},
])
self.__write_output(self.__run2, [
{"id": "song-2", "text": json.dumps(
{"b-north": 1, "b-middle": 1, "b-south": 1})},
])
status: int
status, _ = self.__run_cluster(extra_args=[
"--model", "some-model", "--revision", "abc123"])
self.assertEqual(status, 0)
meta: dict[str, Any] = self.__read_meta()
self.assertEqual(meta["clustering"]["clusters"], 2)
self.assertEqual(
meta["clustering"]["algorithm"],
"AgglomerativeClustering")
self.assertEqual(meta["embedding"], {
"model": "some-model", "revision": "abc123"})
def test_meta_json_extra_keywords_order_and_count(self) -> None:
"""Test that the metadata's ``extra_keywords`` reflects the
given options in the given order, and ``keyword_count``
matches the pooled keyword count."""
self.__write_output(self.__run1, [
{"id": "song-1", "text": json.dumps(
{"a-left": 1, "a-center": 1, "a-right": 1})},
])
self.__write_output(self.__run2, [
{"id": "song-2", "text": json.dumps(
{"b-north": 1, "b-middle": 1, "b-south": 1})},
])
status: int
status, _ = self.__run_cluster(extra_args=[
"--extra-keyword", "zzz-extra",
"--extra-keyword", "aaa-extra"])
self.assertEqual(status, 0)
meta: dict[str, Any] = self.__read_meta()
self.assertEqual(
meta["extra_keywords"], ["zzz-extra", "aaa-extra"])
self.assertEqual(meta["keyword_count"], 6)
def test_missing_clusters_option_rejected(self) -> None:
"""Test that omitting ``--clusters`` exits with an
argparse error."""
argv: list[str] = [
str(self.__run1), str(self.__run2),
str(self.__output_dir)]
stderr: io.StringIO = io.StringIO()
with redirect_stderr(stderr):
with self.assertRaises(SystemExit) as context:
cluster_keywords.parse_args(argv)
self.assertNotEqual(context.exception.code, 0)