imacat/pop-fem-audit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Fix the SonarQube findings

Browse Source 
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
imacat
2026-08-19 13:06:20 +08:00
co-authored by Claude Opus 5
parent 2fa55f29d7
commit e1e2c86661
4 changed files with 96 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/src/pop_fem_audit_tools/commands/build_db.py
+1 -1
View File
@@ -632,7 +632,7 @@ class PerformerGenderDeriver:
known: set[str] = {x for x in values if x}
if len(known) > 1:
return cls.__MIXED
if len(known) == 1 and all(x for x in values):
if len(known) == 1 and all(values):
return known.pop()
return None
tools/src/pop_fem_audit_tools/commands/fetch_artists.py
+7 -6
View File
@@ -161,6 +161,10 @@ class RetryExhausted(Exception):
"""
NOTE_NOT_FOUND: str = "not found"
"""The note marking an artist that could not be resolved."""
class ArtistFetcher:
"""A fetcher of artist metadata from Wikidata.
@@ -252,9 +256,6 @@ class ArtistFetcher:
= ("band", "group", "duo", "trio")
"""The label keywords that suggest a musical ensemble,
covering labels like "boy band" and "girl group"."""
__NOTE_NOT_FOUND: ClassVar[str] = "not found"
"""The note sentinel of an artist without a resolved
Wikidata item."""
__MIXED_GENDER: ClassVar[str] = "mixed"
"""The gender recorded for a group whose members do not
share one gender."""
@@ -301,7 +302,7 @@ class ArtistFetcher:
try:
qid: str | None = self.__resolve_qid(name, titles)
if qid is None:
snapshot.note = self.__NOTE_NOT_FOUND
snapshot.note = NOTE_NOT_FOUND
return snapshot
snapshot.qid = qid
self.__resolve(snapshot)
@@ -1039,9 +1040,9 @@ class ArtistSnapshotUpdater:
artist.name, titles)
self.__append_row(csv_file, snapshot)
status: str = snapshot.qid
if snapshot.note == "not found":
if snapshot.note == NOTE_NOT_FOUND:
not_found += 1
status = "not found"
status = NOTE_NOT_FOUND
elif snapshot.note.startswith("error: "):
errors += 1
status = snapshot.note
tools/src/pop_fem_audit_tools/commands/tally_annotations.py
+71 -32
View File
@@ -82,11 +82,11 @@ class AnnotationTallier:
song's stored performer gender; a gender missing here
(including None) takes every prefix."""
__NUMBERING_RE: ClassVar[re.Pattern[str]] = re.compile(
r"^(?:模式[一二三四五六七八九十]+|[一二三四五六七八九十]+)[：、]")
r"^(?:模式)?[一二三四五六七八九十]+[：、]")
"""The leading numbering token of a pattern heading, stripped
to yield the pattern name."""
__HEADING_RE: ClassVar[re.Pattern[str]] = re.compile(
r"^#+\s*(.*)$")
r"^#++\s*+(.*)$")
"""A Markdown heading line, the heading text captured."""
def __init__(self, male_synthesis: Path, female_synthesis: Path,
@@ -311,36 +311,7 @@ class AnnotationTallier:
pooled: dict[int, list[list[str]]] = {}
run_dir: Path
for run_dir in self.__run_dirs:
path: Path = run_dir / "output.jsonl"
text: str
try:
text = path.read_text(encoding="utf-8")
except OSError as error:
raise TallyError(str(error)) from error
line: str
for line in text.split("\n"):
if line.strip() == "":
continue
record: Any
try:
record = json.loads(line)
except json.JSONDecodeError as error:
raise TallyError(
f"{path}: malformed JSON: {error}") \
from error
song_id: int = self.__parse_song_id(
record["id"], run_dir)
if "text" not in record:
print(
f"warning: {run_dir}: song-{song_id}: no"
" \"text\" field, skipped",
file=sys.stderr)
continue
ballot: list[str] | None = self.__parse_ballot(
record["text"], run_dir, song_id)
if ballot is None:
continue
pooled.setdefault(song_id, []).append(ballot)
self.__load_run_ballots(run_dir, pooled)
song_id: int
ballots: list[list[str]]
for song_id, ballots in pooled.items():
@@ -351,6 +322,74 @@ class AnnotationTallier:
f" {self.__BALLOTS_PER_SONG}")
return pooled
@classmethod
def __load_run_ballots(
cls, run_dir: Path,
pooled: dict[int, list[list[str]]]) -> None:
"""Load one run's ballots into the pool.
:param run_dir: The annotation run's archive directory,
containing ``output.jsonl``.
:param pooled: The pool to append the run's ballots
into, keyed by the numeric song ID; mutated in
place.
:return: None.
:raises TallyError: When the run's ``output.jsonl``
cannot be read, or a line is malformed JSON.
"""
path: Path = run_dir / "output.jsonl"
text: str
try:
text = path.read_text(encoding="utf-8")
except OSError as error:
raise TallyError(str(error)) from error
line: str
for line in text.split("\n"):
if line.strip() == "":
continue
record: Any
try:
record = json.loads(line)
except json.JSONDecodeError as error:
raise TallyError(
f"{path}: malformed JSON: {error}") \
from error
cls.__load_ballot_record(record, run_dir, pooled)
@classmethod
def __load_ballot_record(
cls, record: Any, run_dir: Path,
pooled: dict[int, list[list[str]]]) -> None:
"""Parse and pool one ballot record.
A record whose "text" field is missing, or does not
parse to a JSON array of strings, is skipped, a warning
naming the run directory and song reported on standard
error, as an observable side effect.
:param record: The parsed JSON record.
:param run_dir: The run directory the record came from,
for the warning and error messages.
:param pooled: The pool to append the record's ballot
into, keyed by the numeric song ID; mutated in
place.
:return: None.
:raises TallyError: When the record's "id" field is not
in the ``song-<ID>`` form.
"""
song_id: int = cls.__parse_song_id(record["id"], run_dir)
if "text" not in record:
print(
f"warning: {run_dir}: song-{song_id}: no"
" \"text\" field, skipped",
file=sys.stderr)
return
ballot: list[str] | None = cls.__parse_ballot(
record["text"], run_dir, song_id)
if ballot is None:
return
pooled.setdefault(song_id, []).append(ballot)
@classmethod
def __parse_song_id(cls, item_id: Any, run_dir: Path) -> int:
"""Parse the numeric song ID out of an annotation record
tools/tests/test_run_llm.py
+17 -17
View File
@@ -21,7 +21,7 @@ from pop_fem_audit_tools.commands import run_llm
class RunLLMTestCase(unittest.TestCase):
"""The common base test case with the shared helpers."""
def make_temp_dir(self) -> Path:
def _make_temp_dir(self) -> Path:
"""Create a temporary directory removed on test cleanup.
:return: The path of the temporary directory.
@@ -31,8 +31,8 @@ class RunLLMTestCase(unittest.TestCase):
self.addCleanup(tmp.cleanup)
return Path(tmp.name)
def make_success_entry(self, custom_id: str,
text: str) -> mock.Mock:
def _make_success_entry(self, custom_id: str,
text: str) -> mock.Mock:
"""Create a mock succeeded batch result entry.
:param custom_id: The custom ID of the entry.
@@ -47,8 +47,8 @@ class RunLLMTestCase(unittest.TestCase):
return entry
@staticmethod
def make_error_entry(custom_id: str,
error_type: str) -> mock.Mock:
def _make_error_entry(custom_id: str,
error_type: str) -> mock.Mock:
"""Create a mock errored batch result entry.
The error object is shaped as the SDK envelope: the outer
@@ -95,7 +95,7 @@ class TestLoadItems(RunLLMTestCase):
def setUp(self) -> None:
"""Create the prompt and input paths for a dry run."""
directory: Path = self.make_temp_dir()
directory: Path = self._make_temp_dir()
self.__prompt: Path = directory / "task.md"
self.__prompt.write_text("The task.\n", encoding="utf-8")
self.__input: Path = directory / "items.jsonl"
@@ -171,7 +171,7 @@ class TestRequestBuilding(RunLLMTestCase):
def setUp(self) -> None:
"""Create the prompt and input files for a dry run."""
directory: Path = self.make_temp_dir()
directory: Path = self._make_temp_dir()
self.__prompt: Path = directory / "task.md"
self.__prompt.write_text(
"the system prompt", encoding="utf-8")
@@ -224,7 +224,7 @@ class TestMainFlow(RunLLMTestCase):
def setUp(self) -> None:
"""Create a temporary directory with the input files."""
directory: Path = self.make_temp_dir()
directory: Path = self._make_temp_dir()
self.__runs: Path = directory / "runs"
self.__archive_dir: Path = self.__runs / "task_v1" / "run1"
self.__prompt: Path = directory / "task_v1.md"
@@ -310,8 +310,8 @@ class TestMainFlow(RunLLMTestCase):
the item order preserved and the token usage summed, the
output text written unescaped."""
client: mock.Mock = self.__make_client(
[self.make_success_entry("a", "answer 中文 a"),
self.make_success_entry("b", "answer b")])
[self._make_success_entry("a", "answer 中文 a"),
self._make_success_entry("b", "answer b")])
status: int
stderr: str
with mock.patch(
@@ -356,8 +356,8 @@ class TestMainFlow(RunLLMTestCase):
(run_dir / "stale.jsonl").write_text(
"stale", encoding="utf-8")
client: mock.Mock = self.__make_client(
[self.make_success_entry("a", "answer a"),
self.make_success_entry("b", "answer b")])
[self._make_success_entry("a", "answer a"),
self._make_success_entry("b", "answer b")])
status: int = self.__run_main(
self.__argv + ["--replace"], client)[0]
self.assertEqual(status, 0)
@@ -375,8 +375,8 @@ class TestMainFlow(RunLLMTestCase):
(run2_dir / "output.jsonl").write_text(
"stale run2 data", encoding="utf-8")
client: mock.Mock = self.__make_client(
[self.make_success_entry("a", "answer a"),
self.make_success_entry("b", "answer b")])
[self._make_success_entry("a", "answer a"),
self._make_success_entry("b", "answer b")])
argv: list[str] = [
str(self.__prompt), str(self.__input),
str(run2_dir), "--replace"]
@@ -393,8 +393,8 @@ class TestMainFlow(RunLLMTestCase):
"""Test that a failed item aborts with a non-zero status,
the summed usage counting only the succeeded item."""
client: mock.Mock = self.__make_client(
[self.make_success_entry("a", "answer a"),
self.make_error_entry("b", "invalid_request_error")])
[self._make_success_entry("a", "answer a"),
self._make_error_entry("b", "invalid_request_error")])
status: int
stderr: str
status, _, stderr = self.__run_main(self.__argv, client)
@@ -417,7 +417,7 @@ class TestMainFlow(RunLLMTestCase):
"""Test that an item missing from the batch results is
reported as a failed item."""
client: mock.Mock = self.__make_client(
[self.make_success_entry("a", "answer a")])
[self._make_success_entry("a", "answer a")])
status: int
stderr: str
status, _, stderr = self.__run_main(self.__argv, client)