imacat/pop-fem-audit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge the keyword pooling into cluster-keywords

Browse Source 
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
imacat
2026-08-17 22:38:30 +08:00
co-authored by Claude Opus 5
parent 5cf2b8ee8c
commit de8b8ee678
18 changed files with 526 additions and 626 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/tests/test_cluster_keywords.py
+256 -63
View File
@@ -26,16 +26,45 @@ class TestClusterKeywords(unittest.TestCase):
"""Test cases for the keyword clusterer."""
def setUp(self) -> None:
"""Create a temporary directory for the output files."""
"""Create a temporary directory with two run directories."""
tmp: tempfile.TemporaryDirectory[str] \
= tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.__dir: Path = Path(tmp.name)
self.__pool_txt: Path = self.__dir / "pool.txt"
self.__groups_csv: Path = self.__dir / "groups.csv"
self.__keywords_txt: Path = self.__dir / "keywords.txt"
self.__run1: Path = self.__dir / "run1"
self.__run2: Path = self.__dir / "run2"
self.__run1.mkdir()
self.__run2.mkdir()
self.__output_dir: Path = self.__dir / "output"
self.__source_keywords_txt: Path \
= self.__output_dir \
/ cluster_keywords.SOURCE_KEYWORDS_TXT
self.__source_provenance_csv: Path \
= self.__output_dir \
/ cluster_keywords.SOURCE_PROVENANCE_CSV
self.__result_keywords_txt: Path \
= self.__output_dir \
/ cluster_keywords.RESULT_KEYWORDS_TXT
self.__result_groups_csv: Path \
= self.__output_dir \
/ cluster_keywords.RESULT_GROUPS_CSV
self.__keywords_to_merge_json: Path \
= self.__dir / "keywords-to-merge.json"
= self.__output_dir \
/ cluster_keywords.KEYWORDS_TO_MERGE_JSON
@staticmethod
def __write_output(
run_dir: Path, records: list[dict[str, Any]]) -> None:
"""Write the ``output.jsonl`` file of one run.
:param run_dir: The run's archive directory.
:param records: The envelope records, in file order.
:return: None.
"""
lines: list[str] = [
json.dumps(x, ensure_ascii=False) for x in records]
(run_dir / "output.jsonl").write_text(
"\n".join(lines) + "\n", encoding="utf-8")
@staticmethod
def __two_cluster_vectors() -> Vectors:
@@ -58,15 +87,6 @@ class TestClusterKeywords(unittest.TestCase):
"b-south": (-0.9396926, -0.3420201),
}
def __write_pool(self, keywords: list[str]) -> None:
"""Write the pooled keyword input file.
:param keywords: The keywords, one per line.
:return: None.
"""
self.__pool_txt.write_text(
"".join(f"{x}\n" for x in keywords), encoding="utf-8")
@staticmethod
def __fake_encode(vectors: Vectors) -> Any:
"""Build a test double for :func:`encode_keywords`.
@@ -97,16 +117,15 @@ class TestClusterKeywords(unittest.TestCase):
standard error.
:param extra_args: Extra command-line arguments appended
after the four positional arguments.
after the three positional arguments.
:param vectors: The fixed embedding to encode with; the
two-cluster fixture is used when None.
:return: A tuple of the exit status and the standard
error.
"""
argv: list[str] = [
str(self.__pool_txt), str(self.__groups_csv),
str(self.__keywords_txt),
str(self.__keywords_to_merge_json)]
str(self.__run1), str(self.__run2),
str(self.__output_dir)]
argv.extend(extra_args or [])
fake: Any = self.__fake_encode(
vectors if vectors is not None
@@ -119,22 +138,43 @@ class TestClusterKeywords(unittest.TestCase):
argv + ["--clusters", "2"])
return status, stderr.getvalue()
def __read_source_keywords(self) -> list[str]:
"""Read the pooled source keyword text file.
:return: The keyword list, one keyword per line, with the
trailing empty line from the final newline removed.
"""
lines: list[str] = self.__source_keywords_txt.read_text(
encoding="utf-8").split("\n")
self.assertEqual(lines[-1], "")
return lines[:-1]
def __read_source_provenance(self) -> list[list[str]]:
"""Read the source provenance CSV file.
:return: All rows, including the header row, in file
order.
"""
with open(self.__source_provenance_csv, encoding="utf-8",
newline="") as file:
return list(csv.reader(file))
def __read_groups(self) -> list[list[str]]:
"""Read the group membership CSV file.
:return: All rows, including the header row, in file
order.
"""
with open(self.__groups_csv, encoding="utf-8",
with open(self.__result_groups_csv, encoding="utf-8",
newline="") as file:
return list(csv.reader(file))
def __read_keyword_names(self) -> list[str]:
def __read_result_keywords(self) -> list[str]:
"""Read the group name keyword text file.
:return: The group names, in file order.
"""
text: str = self.__keywords_txt.read_text(
text: str = self.__result_keywords_txt.read_text(
encoding="utf-8")
lines: list[str] = text.split("\n")
if len(lines) > 0 and lines[-1] == "":
@@ -152,12 +192,171 @@ class TestClusterKeywords(unittest.TestCase):
encoding="utf-8"))
return data["keywords"]
def test_pools_union_dedup_sorted(self) -> None:
"""Test the union, dedup, and lexicographic ordering, and
the plain one-keyword-per-line source keyword file
shape."""
self.__write_output(self.__run1, [
{"id": "song-1",
"text": json.dumps({"a-left": 1, "shared": 1})},
])
self.__write_output(self.__run2, [
{"id": "song-3",
"text": json.dumps({"b-middle": 1, "shared": 1})},
])
vectors: Vectors = {
**self.__two_cluster_vectors(),
"shared": (1.0, 0.0)}
status: int
stderr: str
status, stderr = self.__run_cluster(vectors=vectors)
self.assertEqual(status, 0)
self.assertEqual(
self.__read_source_keywords(),
["a-left", "b-middle", "shared"])
self.assertIn(
"done: 3 keywords pooled from 1+1 records", stderr)
def test_skips_error_records(self) -> None:
"""Test that records carrying an "error" field are
excluded from the pool and the record count."""
self.__write_output(self.__run1, [
{"id": "song-1",
"text": json.dumps({"a-left": 1})},
{"id": "song-2", "error": "invalid_request_error"},
])
self.__write_output(self.__run2, [
{"id": "song-3", "text": json.dumps({"b-middle": 1})},
])
status: int
stderr: str
status, stderr = self.__run_cluster()
self.assertEqual(status, 0)
self.assertEqual(
self.__read_source_keywords(), ["a-left", "b-middle"])
self.assertIn(
"done: 2 keywords pooled from 1+1 records", stderr)
def test_skips_non_json_text_records(self) -> None:
"""Test that a refusal, whose "text" does not parse as
JSON, is skipped rather than failing the run."""
self.__write_output(self.__run1, [
{"id": "song-1",
"text": json.dumps({"a-left": 1})},
{"id": "song-2", "text": "I cannot help with that."},
])
self.__write_output(self.__run2, [
{"id": "song-3", "text": json.dumps({"b-middle": 1})},
])
status: int
stderr: str
status, stderr = self.__run_cluster()
self.assertEqual(status, 0)
self.assertEqual(
self.__read_source_keywords(), ["a-left", "b-middle"])
self.assertIn(
"done: 2 keywords pooled from 1+1 records", stderr)
def test_duplicate_key_in_text_rejected(self) -> None:
"""Test that a "text" JSON object with a duplicate key
fails the run without writing any output file."""
self.__write_output(self.__run1, [
{"id": "song-1",
"text": '{"a-left": 1, "a-left": 2}'},
])
self.__write_output(self.__run2, [
{"id": "song-3", "text": json.dumps({"b-middle": 1})},
])
status: int
stderr: str
status, stderr = self.__run_cluster()
self.assertEqual(status, 1)
self.assertIn("duplicate key", stderr)
self.assertFalse(self.__source_keywords_txt.exists())
self.assertFalse(self.__source_provenance_csv.exists())
self.assertFalse(self.__result_groups_csv.exists())
self.assertFalse(self.__result_keywords_txt.exists())
self.assertFalse(self.__keywords_to_merge_json.exists())
def test_non_object_text_rejected(self) -> None:
"""Test that a "text" JSON value that is not an object
fails the run without writing any output file."""
self.__write_output(self.__run1, [
{"id": "song-1", "text": json.dumps(["a-left"])},
])
self.__write_output(self.__run2, [
{"id": "song-3", "text": json.dumps({"b-middle": 1})},
])
status: int
stderr: str
status, stderr = self.__run_cluster()
self.assertEqual(status, 1)
self.assertIn("song-1", stderr)
self.assertFalse(self.__source_keywords_txt.exists())
self.assertFalse(self.__source_provenance_csv.exists())
self.assertFalse(self.__result_groups_csv.exists())
self.assertFalse(self.__result_keywords_txt.exists())
self.assertFalse(self.__keywords_to_merge_json.exists())
def test_provenance_content_and_ordering(self) -> None:
"""Test the provenance content and its ordering: rows
sorted by keyword lexicographically, then by run label,
then by song ID."""
self.__write_output(self.__run1, [
{"id": "song-2", "text": json.dumps({"shared": 1})},
{"id": "song-1", "text": json.dumps({"shared": 1})},
])
self.__write_output(self.__run2, [
{"id": "song-5",
"text": json.dumps({"shared": 1, "b-middle": 1})},
])
vectors: Vectors = {
**self.__two_cluster_vectors(),
"shared": (1.0, 0.0)}
status: int
status, _ = self.__run_cluster(vectors=vectors)
self.assertEqual(status, 0)
rows: list[list[str]] = self.__read_source_provenance()
self.assertEqual(rows[1:], [
["b-middle", "run2", "5"],
["shared", "run1", "1"],
["shared", "run1", "2"],
["shared", "run2", "5"],
])
def test_provenance_file_header_and_row_count(self) -> None:
"""Test that the provenance CSV file starts with the
``Keyword,Run,Song`` header row and has exactly one row
per keyword occurrence."""
self.__write_output(self.__run1, [
{"id": "song-2", "text": json.dumps({"shared": 1})},
{"id": "song-1", "text": json.dumps({"shared": 1})},
])
self.__write_output(self.__run2, [
{"id": "song-5",
"text": json.dumps({"shared": 1, "b-middle": 1})},
])
vectors: Vectors = {
**self.__two_cluster_vectors(),
"shared": (1.0, 0.0)}
status: int
status, _ = self.__run_cluster(vectors=vectors)
self.assertEqual(status, 0)
rows: list[list[str]] = self.__read_source_provenance()
self.assertEqual(rows[0], ["Keyword", "Run", "Song"])
self.assertEqual(len(rows), 1 + 4)
def test_groups_csv_header_and_ordering(self) -> None:
"""Test the header row and the group/keyword ordering of
the group membership CSV file."""
self.__write_pool([
"a-left", "a-center", "a-right",
"b-north", "b-middle", "b-south"])
self.__write_output(self.__run1, [
{"id": "song-1", "text": json.dumps(
{"a-left": 1, "a-center": 1, "a-right": 1})},
])
self.__write_output(self.__run2, [
{"id": "song-2", "text": json.dumps(
{"b-north": 1, "b-middle": 1, "b-south": 1})},
])
status: int
status, _ = self.__run_cluster()
self.assertEqual(status, 0)
@@ -178,7 +377,11 @@ class TestClusterKeywords(unittest.TestCase):
keywords: list[str] = [
"a-left", "a-center", "a-right",
"b-north", "b-middle", "b-south"]
self.__write_pool(keywords)
self.__write_output(self.__run1, [
{"id": "song-1", "text": json.dumps(
{x: 1 for x in keywords})},
])
self.__write_output(self.__run2, [])
status: int
status, _ = self.__run_cluster()
self.assertEqual(status, 0)
@@ -187,15 +390,21 @@ class TestClusterKeywords(unittest.TestCase):
sorted(x[1] for x in rows), sorted(keywords))
def test_keywords_txt_sorted_medoids(self) -> None:
"""Test that the keyword text file holds the sorted medoid
group names without the extra a-priori keyword."""
self.__write_pool([
"a-left", "a-center", "a-right",
"b-north", "b-middle", "b-south"])
"""Test that the result keyword text file holds the
sorted medoid group names without the extra a-priori
keyword."""
self.__write_output(self.__run1, [
{"id": "song-1", "text": json.dumps(
{"a-left": 1, "a-center": 1, "a-right": 1})},
])
self.__write_output(self.__run2, [
{"id": "song-2", "text": json.dumps(
{"b-north": 1, "b-middle": 1, "b-south": 1})},
])
status: int
status, _ = self.__run_cluster()
self.assertEqual(status, 0)
names: list[str] = self.__read_keyword_names()
names: list[str] = self.__read_result_keywords()
self.assertEqual(names, ["a-center", "b-middle"])
self.assertEqual(names, sorted(names))
self.assertNotIn(cluster_keywords.EXTRA_KEYWORD, names)
@@ -204,9 +413,14 @@ class TestClusterKeywords(unittest.TestCase):
"""Test that the coding keyword set JSON file holds the
sorted medoid group names plus the extra a-priori
keyword."""
self.__write_pool([
"a-left", "a-center", "a-right",
"b-north", "b-middle", "b-south"])
self.__write_output(self.__run1, [
{"id": "song-1", "text": json.dumps(
{"a-left": 1, "a-center": 1, "a-right": 1})},
])
self.__write_output(self.__run2, [
{"id": "song-2", "text": json.dumps(
{"b-north": 1, "b-middle": 1, "b-south": 1})},
])
status: int
status, _ = self.__run_cluster()
self.assertEqual(status, 0)
@@ -220,38 +434,17 @@ class TestClusterKeywords(unittest.TestCase):
def test_extra_keyword_absent_from_groups_csv(self) -> None:
"""Test that the extra a-priori keyword appears in no row
of the group membership CSV file."""
self.__write_pool([
"a-left", "a-center", "a-right",
"b-north", "b-middle", "b-south"])
self.__write_output(self.__run1, [
{"id": "song-1", "text": json.dumps(
{"a-left": 1, "a-center": 1, "a-right": 1})},
])
self.__write_output(self.__run2, [
{"id": "song-2", "text": json.dumps(
{"b-north": 1, "b-middle": 1, "b-south": 1})},
])
status: int
status, _ = self.__run_cluster()
self.assertEqual(status, 0)
rows: list[list[str]] = self.__read_groups()
for row in rows:
self.assertNotIn(cluster_keywords.EXTRA_KEYWORD, row)
def test_duplicate_keyword_rejected(self) -> None:
"""Test that a duplicate keyword line fails the run
without writing any output file."""
self.__write_pool(["shared", "shared"])
status: int
stderr: str
status, stderr = self.__run_cluster()
self.assertEqual(status, 1)
self.assertIn("duplicate keyword", stderr)
self.assertFalse(self.__groups_csv.exists())
self.assertFalse(self.__keywords_txt.exists())
self.assertFalse(self.__keywords_to_merge_json.exists())
def test_empty_input_rejected(self) -> None:
"""Test that an empty keyword file fails the run without
writing any output file."""
self.__pool_txt.write_text("", encoding="utf-8")
status: int
stderr: str
status, stderr = self.__run_cluster()
self.assertEqual(status, 1)
self.assertIn("no keywords", stderr)
self.assertFalse(self.__groups_csv.exists())
self.assertFalse(self.__keywords_txt.exists())
self.assertFalse(self.__keywords_to_merge_json.exists())
tools/tests/test_pool_keywords.py
-217
View File
@@ -1,217 +0,0 @@
# Tools for A Feminist Audit of Pop Music.
# Copyright 2026 imacat. All rights reserved.
# Authors:
# imacat@mail.imacat.idv.tw (imacat), 2026/8/5
"""Unit tests for the keyword pooler module."""
import csv
import io
import json
import tempfile
import unittest
from contextlib import redirect_stderr
from pathlib import Path
from typing import Any
from pop_fem_audit_tools.commands import pool_keywords
class TestPoolKeywords(unittest.TestCase):
"""Test cases for the keyword pooler."""
def setUp(self) -> None:
"""Create a temporary directory with two run directories."""
tmp: tempfile.TemporaryDirectory[str] \
= tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.__dir: Path = Path(tmp.name)
self.__run1: Path = self.__dir / "run1"
self.__run2: Path = self.__dir / "run2"
self.__run1.mkdir()
self.__run2.mkdir()
self.__pool: Path = self.__dir / "pool.txt"
self.__provenance: Path = self.__dir / "provenance.csv"
@staticmethod
def __write_output(
run_dir: Path, records: list[dict[str, Any]]) -> None:
"""Write the ``output.jsonl`` file of one run.
:param run_dir: The run's archive directory.
:param records: The envelope records, in file order.
:return: None.
"""
lines: list[str] = [
json.dumps(x, ensure_ascii=False) for x in records]
(run_dir / "output.jsonl").write_text(
"\n".join(lines) + "\n", encoding="utf-8")
def __run_pool(self) -> tuple[int, str]:
"""Run the pooler with the standard error captured.
:return: A tuple of the exit status and the standard
error.
"""
stderr: io.StringIO = io.StringIO()
with redirect_stderr(stderr):
status: int = pool_keywords.main([
str(self.__run1), str(self.__run2),
str(self.__pool), str(self.__provenance)])
return status, stderr.getvalue()
def __read_pool(self) -> list[str]:
"""Read the pool text file.
:return: The keyword list, one keyword per line, with the
trailing empty line from the final newline removed.
"""
lines: list[str] = self.__pool.read_text(
encoding="utf-8").split("\n")
self.assertEqual(lines[-1], "")
return lines[:-1]
def __read_provenance(self) -> list[list[str]]:
"""Read the provenance CSV file.
:return: All rows, including the header row, in file
order.
"""
with open(self.__provenance, encoding="utf-8",
newline="") as file:
return list(csv.reader(file))
def test_pools_union_dedup_sorted(self) -> None:
"""Test the union, dedup, and lexicographic ordering, and
the plain one-keyword-per-line pool file shape."""
self.__write_output(self.__run1, [
{"id": "song-1",
"text": json.dumps({"strength": 1, "shared": 1})},
])
self.__write_output(self.__run2, [
{"id": "song-3",
"text": json.dumps({"warrior": 1, "shared": 1})},
])
status: int
stderr: str
status, stderr = self.__run_pool()
self.assertEqual(status, 0)
self.assertEqual(
self.__read_pool(), ["shared", "strength", "warrior"])
self.assertIn(
"done: 3 keywords pooled from 1+1 records", stderr)
def test_skips_error_records(self) -> None:
"""Test that records carrying an "error" field are
excluded from the pool and the record count."""
self.__write_output(self.__run1, [
{"id": "song-1",
"text": json.dumps({"strength": 1})},
{"id": "song-2", "error": "invalid_request_error"},
])
self.__write_output(self.__run2, [
{"id": "song-3", "text": json.dumps({"warrior": 1})},
])
status: int
stderr: str
status, stderr = self.__run_pool()
self.assertEqual(status, 0)
self.assertEqual(
self.__read_pool(), ["strength", "warrior"])
self.assertIn(
"done: 2 keywords pooled from 1+1 records", stderr)
def test_skips_non_json_text_records(self) -> None:
"""Test that a refusal, whose "text" does not parse as
JSON, is skipped rather than failing the run."""
self.__write_output(self.__run1, [
{"id": "song-1",
"text": json.dumps({"strength": 1})},
{"id": "song-2", "text": "I cannot help with that."},
])
self.__write_output(self.__run2, [
{"id": "song-3", "text": json.dumps({"warrior": 1})},
])
status: int
stderr: str
status, stderr = self.__run_pool()
self.assertEqual(status, 0)
self.assertEqual(
self.__read_pool(), ["strength", "warrior"])
self.assertIn(
"done: 2 keywords pooled from 1+1 records", stderr)
def test_duplicate_key_in_text_rejected(self) -> None:
"""Test that a "text" JSON object with a duplicate key
fails the run without writing any output file."""
self.__write_output(self.__run1, [
{"id": "song-1",
"text": '{"strength": 1, "strength": 2}'},
])
self.__write_output(self.__run2, [
{"id": "song-3", "text": json.dumps({"warrior": 1})},
])
status: int
stderr: str
status, stderr = self.__run_pool()
self.assertEqual(status, 1)
self.assertIn("duplicate key", stderr)
self.assertFalse(self.__pool.exists())
self.assertFalse(self.__provenance.exists())
def test_non_object_text_rejected(self) -> None:
"""Test that a "text" JSON value that is not an object
fails the run without writing any output file."""
self.__write_output(self.__run1, [
{"id": "song-1", "text": json.dumps(["strength"])},
])
self.__write_output(self.__run2, [
{"id": "song-3", "text": json.dumps({"warrior": 1})},
])
status: int
stderr: str
status, stderr = self.__run_pool()
self.assertEqual(status, 1)
self.assertIn("song-1", stderr)
self.assertFalse(self.__pool.exists())
self.assertFalse(self.__provenance.exists())
def test_provenance_content_and_ordering(self) -> None:
"""Test the provenance content and its ordering: rows
sorted by keyword lexicographically, then by run label,
then by song ID."""
self.__write_output(self.__run1, [
{"id": "song-2", "text": json.dumps({"shared": 1})},
{"id": "song-1", "text": json.dumps({"shared": 1})},
])
self.__write_output(self.__run2, [
{"id": "song-5",
"text": json.dumps({"shared": 1, "warrior": 1})},
])
status: int
status, _ = self.__run_pool()
self.assertEqual(status, 0)
rows: list[list[str]] = self.__read_provenance()
self.assertEqual(rows[1:], [
["shared", "run1", "1"],
["shared", "run1", "2"],
["shared", "run2", "5"],
["warrior", "run2", "5"],
])
def test_provenance_file_header_and_row_count(self) -> None:
"""Test that the provenance CSV file starts with the
``Keyword,Run,Song`` header row and has exactly one row
per keyword occurrence."""
self.__write_output(self.__run1, [
{"id": "song-2", "text": json.dumps({"shared": 1})},
{"id": "song-1", "text": json.dumps({"shared": 1})},
])
self.__write_output(self.__run2, [
{"id": "song-5",
"text": json.dumps({"shared": 1, "warrior": 1})},
])
status: int
status, _ = self.__run_pool()
self.assertEqual(status, 0)
rows: list[list[str]] = self.__read_provenance()
self.assertEqual(rows[0], ["Keyword", "Run", "Song"])
self.assertEqual(len(rows), 1 + 4)