Gather the CLI command modules into a commands sub-package
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5
@@ -0,0 +1,10 @@
|
||||
# Tools for A Feminist Audit of Pop Music.
|
||||
# Copyright 2026 imacat. All rights reserved.
|
||||
# Authors:
|
||||
# imacat@mail.imacat.idv.tw (imacat), 2026/8/4
|
||||
"""The registry of the CLI subcommands."""
|
||||
from .build_db import main as build_db_command
|
||||
from .export_llm_input import main as export_llm_input_command
|
||||
from .fetch_artists import main as fetch_artists_command
|
||||
from .fetch_lyrics import main as fetch_lyrics_command
|
||||
from .run_llm import main as run_llm_command
|
||||
@@ -0,0 +1,829 @@
|
||||
# Tools for A Feminist Audit of Pop Music.
|
||||
# Copyright 2026 imacat. All rights reserved.
|
||||
# Authors:
|
||||
# imacat@mail.imacat.idv.tw (imacat), 2026/7/31
|
||||
"""The builder of the SQLite working store.
|
||||
|
||||
Rebuilds the working store from scratch out of the committed
|
||||
inputs: the year-end chart CSV and the output directory for the
|
||||
review CSV files, given as the two positional command-line
|
||||
arguments, and the optional capture inputs, each given as an
|
||||
option: the lyrics cache directory and the Wikidata artist
|
||||
snapshot CSV. An omitted option
|
||||
leaves its capture layer unloaded; a given option whose path does
|
||||
not exist fails the build. Missing tables
|
||||
are created on a fresh store; existing tables are never altered,
|
||||
as the schema lifecycle belongs to the migrations. Every rebuild
|
||||
deletes all the rows, loads the data, and validates it in one
|
||||
transaction, committed only after the data passes the validation
|
||||
invariants; a failed build leaves the previous store contents
|
||||
intact.
|
||||
|
||||
The rebuild is deterministic: the builder assigns the song and
|
||||
artist IDs itself, as 1, 2, 3, ... in the first-occurrence file
|
||||
order, so the IDs are reproducible across rebuilds on every
|
||||
database engine, given the frozen input file.
|
||||
|
||||
A song is identified by its raw title together with its artist
|
||||
credit, the credit canonicalized through
|
||||
``SongImporter.CANONICAL_ARTIST_CREDITS``; a credit listed there
|
||||
collapses onto the same song as its canonical form, and the
|
||||
stored artist credit is always the canonical form. Artist
|
||||
deduplication is by the identity key resolved from the parsed
|
||||
artist name (see `ArtistImporter.resolve_artist_identity`): the
|
||||
case-folded name, or, when that case-folded name is listed in
|
||||
``ArtistImporter.CANONICAL_ARTIST_NAMES``, the case-folded
|
||||
canonical spelling, so letter-case variants and alternate
|
||||
spellings mapped to the same canonical name all collapse onto a
|
||||
single artist row. The stored artist name is the first-seen
|
||||
spelling, except for the names listed in
|
||||
``ArtistImporter.CANONICAL_ARTIST_NAMES``, which always store the
|
||||
canonical spelling regardless of which variant is seen first.
|
||||
|
||||
On a successful build, two review CSV files, ``songs.csv`` and
|
||||
``artists.csv``, are (re)written under the given output directory,
|
||||
mirroring the stored songs and artists without their IDs; see
|
||||
`CSVExporter`. A failed build leaves any existing review CSV
|
||||
files untouched, matching the store rollback.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Self
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..database import Base, ds
|
||||
from ..models import (
|
||||
Artist,
|
||||
ChartEntry,
|
||||
Role,
|
||||
Song,
|
||||
SongArtist,
|
||||
)
|
||||
|
||||
ARTIST_FIELDS: dict[str, str] = {
|
||||
"qid": "wikidata_qid",
|
||||
"gender": "gender",
|
||||
"type": "type",
|
||||
"genre": "genre",
|
||||
"country": "country",
|
||||
}
|
||||
"""The artist CSV columns mapped to the Artist attributes."""
|
||||
|
||||
|
||||
class BuildError(Exception):
|
||||
"""An error that fails the build."""
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None) -> argparse.Namespace:
|
||||
"""Parse the command-line arguments.
|
||||
|
||||
:param argv: The command-line arguments, or None for
|
||||
``sys.argv``.
|
||||
:return: The parsed arguments.
|
||||
"""
|
||||
parser: argparse.ArgumentParser = argparse.ArgumentParser(
|
||||
description="Rebuild the SQLite working store from the"
|
||||
" committed inputs.")
|
||||
parser.add_argument(
|
||||
"chart_csv", type=Path,
|
||||
help="the year-end chart CSV file")
|
||||
parser.add_argument(
|
||||
"derived_dir", type=Path,
|
||||
help="the output directory for the review CSV files")
|
||||
parser.add_argument(
|
||||
"--lyrics-dir", type=Path, default=None,
|
||||
help="the lyrics cache directory to load")
|
||||
parser.add_argument(
|
||||
"--wikidata-csv", type=Path, default=None,
|
||||
help="the Wikidata artist snapshot CSV file to apply")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
class SongImporter:
|
||||
"""The song-import job: loads the chart CSV into songs and
|
||||
chart entries."""
|
||||
|
||||
YEARS: Sequence[int] = range(2016, 2026)
|
||||
"""The expected chart years."""
|
||||
RANKS_PER_YEAR: int = 100
|
||||
"""The expected number of ranks on the chart of each year."""
|
||||
CANONICAL_ARTIST_CREDITS: dict[str, str] = {
|
||||
"benny blanco, Halsey & Khalid": "Benny Blanco, Halsey"
|
||||
" & Khalid",
|
||||
}
|
||||
"""The canonical artist credit spellings, keyed by a variant
|
||||
credit string."""
|
||||
|
||||
def __init__(self, session: Session) -> None:
|
||||
"""Initialize the importer.
|
||||
|
||||
:param session: The database session.
|
||||
"""
|
||||
self.__session: Session = session
|
||||
self.__songs: dict[tuple[str, str], Song] = {}
|
||||
|
||||
def import_songs(self, path: Path) -> None:
|
||||
"""Load the chart CSV into songs and chart entries.
|
||||
|
||||
A song repeated across the rows is stored once, matched by
|
||||
its identity key (see `song_identity`); every row yields
|
||||
one chart entry. The stored title is the raw title; the
|
||||
stored artist credit is the canonical credit from the
|
||||
identity key. The songs take the IDs 1, 2, 3, ... in the
|
||||
first-occurrence row order. When the method returns, the
|
||||
imported songs and chart entries are queryable in the
|
||||
session.
|
||||
|
||||
:param path: The chart CSV file with the columns year,
|
||||
rank, title, and artist.
|
||||
:return: None.
|
||||
:raises BuildError: When the chart entries do not cover
|
||||
each of ``YEARS`` and each rank from 1 to
|
||||
``RANKS_PER_YEAR`` exactly once.
|
||||
:raises OSError: When the file cannot be read.
|
||||
"""
|
||||
counts: Counter[tuple[int, int]] = Counter()
|
||||
with open(path, encoding="utf-8", newline="") as file:
|
||||
row: dict[str, str]
|
||||
for row in csv.DictReader(file):
|
||||
key: tuple[str, str] = self.song_identity(
|
||||
row["title"], row["artist"])
|
||||
if key not in self.__songs:
|
||||
title: str
|
||||
credit: str
|
||||
title, credit = key
|
||||
song: Song = Song(
|
||||
id=len(self.__songs) + 1, title=title,
|
||||
artist_credit=credit)
|
||||
self.__session.add(song)
|
||||
self.__songs[key] = song
|
||||
year: int = int(row["year"])
|
||||
rank: int = int(row["rank"])
|
||||
counts[(year, rank)] += 1
|
||||
if counts[(year, rank)] == 1:
|
||||
self.__session.add(ChartEntry(
|
||||
year=year, rank=rank,
|
||||
song=self.__songs[key]))
|
||||
self.__session.flush()
|
||||
self.__check_chart_coverage(counts)
|
||||
|
||||
@classmethod
|
||||
def __check_chart_coverage(
|
||||
cls, counts: Counter[tuple[int, int]]) -> None:
|
||||
"""Verify the chart entries cover the expected grid exactly.
|
||||
|
||||
:param counts: The number of chart entries seen for each
|
||||
(year, rank) pair.
|
||||
:return: None.
|
||||
:raises BuildError: When a (year, rank) pair from the
|
||||
expected grid is missing, an unexpected pair is
|
||||
present, or a pair is duplicated.
|
||||
"""
|
||||
expected: set[tuple[int, int]] = {
|
||||
(year, rank) for year in cls.YEARS
|
||||
for rank in range(1, cls.RANKS_PER_YEAR + 1)}
|
||||
actual: set[tuple[int, int]] = set(counts)
|
||||
violations: list[str] = []
|
||||
year: int
|
||||
rank: int
|
||||
for year, rank in sorted(expected - actual):
|
||||
violations.append(
|
||||
f"missing chart entry: year {year} rank {rank}")
|
||||
for year, rank in sorted(actual - expected):
|
||||
violations.append(
|
||||
f"unexpected chart entry: year {year} rank {rank}")
|
||||
for year, rank in sorted(counts):
|
||||
if counts[(year, rank)] > 1:
|
||||
violations.append(
|
||||
f"duplicated chart entry: year {year} rank"
|
||||
f" {rank}")
|
||||
if len(violations) > 0:
|
||||
raise BuildError("\n".join(violations))
|
||||
|
||||
@staticmethod
|
||||
def song_identity(title: str, credit: str) -> tuple[str, str]:
|
||||
"""Compute the identity key of a chart row.
|
||||
|
||||
The key pairs the raw title with the artist credit,
|
||||
canonicalized through ``CANONICAL_ARTIST_CREDITS``; a
|
||||
credit absent from the table maps to itself. Two chart
|
||||
rows denote the same song iff their identity keys are
|
||||
equal.
|
||||
|
||||
:param title: The song title as printed on the chart.
|
||||
:param credit: The combined artist credit string.
|
||||
:return: The identity key: the raw title paired with the
|
||||
canonical artist credit.
|
||||
"""
|
||||
return title, SongImporter.CANONICAL_ARTIST_CREDITS.get(
|
||||
credit, credit)
|
||||
|
||||
|
||||
class ArtistImporter:
|
||||
"""The artist-import job: parses the stored songs' artist
|
||||
credits into artists and song-artist credits."""
|
||||
|
||||
FEATURING_PATTERN: re.Pattern[str] = re.compile(
|
||||
r" featuring | feat\. ", re.IGNORECASE)
|
||||
"""The pattern splitting the primary and featured sides."""
|
||||
DELIMITER_PATTERN: re.Pattern[str] = re.compile(
|
||||
r", | & | \+ | / |(?i: and | x | with )")
|
||||
"""The pattern splitting the artist names within a side."""
|
||||
COLON_PATTERN: re.Pattern[str] = re.compile(r": ")
|
||||
"""The pattern separating a group prefix from its members in a
|
||||
"<group>: <members>" credit."""
|
||||
PAREN_MEMBERS_PATTERN: re.Pattern[str] = re.compile(
|
||||
r"^.+ \((?P<members>.+)\)$")
|
||||
"""The pattern separating a group name from its members in a
|
||||
"<group> (<members>)" credit spanning the whole credit."""
|
||||
DUET_WITH_PATTERN: re.Pattern[str] = re.compile(
|
||||
r" Duet With ", re.IGNORECASE)
|
||||
"""The pattern normalizing the "Duet With" co-billing connector
|
||||
to the plain "with" delimiter."""
|
||||
PROTECTED_ARTIST_NAMES: tuple[str, ...] = (
|
||||
"Tyler, The Creator",
|
||||
"Lil Nas X",
|
||||
"Tones And I",
|
||||
)
|
||||
"""The exact artist names guarded from the delimiter splitting,
|
||||
because each contains a delimiter word or punctuation as part
|
||||
of the name itself."""
|
||||
EXCEPTION_CREDITS: dict[str, list[tuple[str, Role]]] = {
|
||||
"SpotemGottem Featuring Pooh Shiesty Or DaBaby": [
|
||||
("SpotemGottem", Role.PRIMARY),
|
||||
("Pooh Shiesty", Role.FEATURED),
|
||||
("DaBaby", Role.FEATURED),
|
||||
],
|
||||
"THE SCOTTS, Travis Scott & Kid Cudi": [
|
||||
("Travis Scott", Role.PRIMARY),
|
||||
("Kid Cudi", Role.PRIMARY),
|
||||
],
|
||||
"Drake Featuring The Throne": [
|
||||
("Drake", Role.PRIMARY),
|
||||
("Jay Z", Role.FEATURED),
|
||||
("Kanye West", Role.FEATURED),
|
||||
],
|
||||
}
|
||||
"""The single-credit exceptions parsed by an explicit lookup
|
||||
rather than by the general rules, because the credit text alone
|
||||
does not spell out the correct member split."""
|
||||
CANONICAL_ARTIST_NAMES: dict[str, str] = {
|
||||
"beyonce": "Beyoncé",
|
||||
"5 seconds of summer": "5 Seconds of Summer",
|
||||
"a boogie wit da hoodie": "A Boogie wit da Hoodie",
|
||||
"benny blanco": "benny blanco",
|
||||
"blackbear": "blackbear",
|
||||
"chance the rapper": "Chance the Rapper",
|
||||
"xxxtentacion": "XXXTENTACION",
|
||||
"maneskin": "Måneskin",
|
||||
"rose": "ROSÉ",
|
||||
"mo": "MØ",
|
||||
"wizkid": "Wizkid",
|
||||
"ye": "Kanye West",
|
||||
"amine": "Aminé",
|
||||
"bomba estereo": "Bomba Estéreo",
|
||||
"carolina gaitan": "Carolina Gaitán",
|
||||
"casper magico": "Casper Mágico",
|
||||
"eslabon armado": "Eslabón Armado",
|
||||
"jhene aiko": "Jhené Aiko",
|
||||
"neton vega": "Netón Vega",
|
||||
"nio garcia": "Nio García",
|
||||
"oscar maydon": "Óscar Maydon",
|
||||
"silento": "Silentó",
|
||||
"the marias": "The Marías",
|
||||
"victoria monet": "Victoria Monét",
|
||||
"dan": "Dan Smyers",
|
||||
"shay": "Shay Mooney",
|
||||
"cris mj": "Cris MJ",
|
||||
"mariah the scientist": "Mariah the Scientist",
|
||||
"surf mesa": "Surf Mesa",
|
||||
}
|
||||
"""The canonical artist spellings, keyed by the case-folded
|
||||
identity."""
|
||||
|
||||
def __init__(self, session: Session) -> None:
|
||||
"""Initialize the importer.
|
||||
|
||||
:param session: The database session.
|
||||
"""
|
||||
self.__session: Session = session
|
||||
self.__artists: dict[str, Artist] = {}
|
||||
|
||||
def import_artists(self) -> None:
|
||||
"""Parse the stored songs' credits into artists and
|
||||
song-artist credits.
|
||||
|
||||
Reads the songs back from the database in ``Song.id`` order,
|
||||
including any songs pending in the same session, and for
|
||||
each song parses ``Song.artist_credit`` (see
|
||||
`parse_artist_credit`). An artist parsed out of a credit is
|
||||
matched against the known artists by its identity key (see
|
||||
`resolve_artist_identity`); a newly seen one takes the ID
|
||||
following the known artists, keyed by its identity key,
|
||||
assigned in first-seen order across the songs, and its
|
||||
stored name is the resolved stored spelling. An artist
|
||||
duplicated within one song's credit, by its identity key,
|
||||
is kept only at its first occurrence within that credit,
|
||||
with a warning to the standard error. When the method
|
||||
returns, the imported artists and credits are queryable in
|
||||
the session.
|
||||
|
||||
:return: None.
|
||||
"""
|
||||
song: Song
|
||||
for song in self.__session.scalars(
|
||||
sa.select(Song).order_by(Song.id)):
|
||||
self.__import_song_artists(song)
|
||||
self.__session.flush()
|
||||
|
||||
def __import_song_artists(self, song: Song) -> None:
|
||||
"""Parse and store the artist credits of one song.
|
||||
|
||||
:param song: The song with its stored artist credit.
|
||||
:return: None.
|
||||
:raises BuildError: When the parsed credit has no primary
|
||||
artist or contains a blank artist name (see
|
||||
`__check_parsed_credit`).
|
||||
"""
|
||||
parsed: list[tuple[str, Role]] = self.parse_artist_credit(
|
||||
song.artist_credit)
|
||||
self.__check_parsed_credit(song, parsed)
|
||||
seen: set[str] = set()
|
||||
position: int = 0
|
||||
name: str
|
||||
role: Role
|
||||
for name, role in parsed:
|
||||
key: str
|
||||
stored_name: str
|
||||
key, stored_name = self.resolve_artist_identity(name)
|
||||
if key in seen:
|
||||
print(f"warning: {song.artist_credit}: duplicated"
|
||||
f" artist \"{name}\"", file=sys.stderr)
|
||||
continue
|
||||
seen.add(key)
|
||||
if key not in self.__artists:
|
||||
self.__artists[key] = Artist(
|
||||
id=len(self.__artists) + 1, name=stored_name)
|
||||
self.__session.add(SongArtist(
|
||||
song=song, artist=self.__artists[key], role=role,
|
||||
position=position))
|
||||
position += 1
|
||||
|
||||
@staticmethod
|
||||
def __check_parsed_credit(
|
||||
song: Song, parsed: list[tuple[str, Role]]) -> None:
|
||||
"""Verify a song's parsed artist credit is well-formed.
|
||||
|
||||
:param song: The song whose credit was parsed.
|
||||
:param parsed: The (name, role) pairs parsed from
|
||||
``song.artist_credit``.
|
||||
:return: None.
|
||||
:raises BuildError: When ``parsed`` is empty, has no
|
||||
``Role.PRIMARY`` entry, or contains a name blank after
|
||||
stripping.
|
||||
"""
|
||||
role: Role
|
||||
if len(parsed) == 0 or not any(
|
||||
role == Role.PRIMARY for _, role in parsed):
|
||||
raise BuildError(
|
||||
f"song {song.id} \"{song.artist_credit}\": no"
|
||||
" primary artist parsed")
|
||||
name: str
|
||||
for name, role in parsed:
|
||||
if name.strip() == "":
|
||||
raise BuildError(
|
||||
f"song {song.id} \"{song.artist_credit}\":"
|
||||
" blank artist name parsed")
|
||||
|
||||
@staticmethod
|
||||
def parse_artist_credit(credit: str) -> list[tuple[str, Role]]:
|
||||
"""Parse a combined artist credit into artists and roles.
|
||||
|
||||
A credit listed in ``EXCEPTION_CREDITS`` is looked up
|
||||
verbatim, because its correct split is not derivable from
|
||||
the credit text alone. Otherwise the credit first reduces
|
||||
to an effective credit: a "<group>: <members>" prefix
|
||||
(split at the first ": ") drops the group and keeps the
|
||||
members; failing that, a "<group> (<members>)" suffix
|
||||
spanning the whole credit drops the group and keeps the
|
||||
members. The "Duet With" connector, case-insensitively,
|
||||
then normalizes to "with". The effective credit splits
|
||||
into a primary side and a featured side on the word
|
||||
"featuring" or "feat.", case-insensitively; without them,
|
||||
every artist is primary. Each side splits into artist
|
||||
names on the delimiters ", ", " & ", " + ", " / "
|
||||
(literally) and " and ", " x ", " with "
|
||||
(case-insensitively), except for the names listed in
|
||||
``PROTECTED_ARTIST_NAMES``, which are never split even
|
||||
though each contains a delimiter word or punctuation.
|
||||
|
||||
Known limitation: a compound act name that contains one of
|
||||
the delimiters, other than the protected names, is
|
||||
over-split.
|
||||
|
||||
:param credit: The combined artist credit string.
|
||||
:return: The (name, role) pairs in credit order, primary
|
||||
side first, with the role ``Role.PRIMARY`` or
|
||||
``Role.FEATURED``.
|
||||
"""
|
||||
if credit in ArtistImporter.EXCEPTION_CREDITS:
|
||||
return list(ArtistImporter.EXCEPTION_CREDITS[credit])
|
||||
effective: str = credit
|
||||
colon_match: re.Match[str] | None = \
|
||||
ArtistImporter.COLON_PATTERN.search(effective)
|
||||
if colon_match is not None:
|
||||
effective = effective[colon_match.end():]
|
||||
else:
|
||||
paren_match: re.Match[str] | None = \
|
||||
ArtistImporter.PAREN_MEMBERS_PATTERN.match(
|
||||
effective)
|
||||
if paren_match is not None:
|
||||
effective = paren_match.group("members")
|
||||
effective = ArtistImporter.DUET_WITH_PATTERN.sub(
|
||||
" with ", effective)
|
||||
placeholders: dict[str, str] = {}
|
||||
index: int
|
||||
protected: str
|
||||
for index, protected in enumerate(
|
||||
ArtistImporter.PROTECTED_ARTIST_NAMES):
|
||||
if protected in effective:
|
||||
placeholder: str = f"{index}"
|
||||
placeholders[placeholder] = protected
|
||||
effective = effective.replace(protected, placeholder)
|
||||
sides: list[str] = ArtistImporter.FEATURING_PATTERN.split(
|
||||
effective, maxsplit=1)
|
||||
pairs: list[tuple[str, Role]] = []
|
||||
role: Role
|
||||
side: str
|
||||
for side, role in zip(sides, (Role.PRIMARY, Role.FEATURED)):
|
||||
token: str
|
||||
for token in ArtistImporter.DELIMITER_PATTERN.split(
|
||||
side):
|
||||
name: str = token.strip()
|
||||
placeholder = ""
|
||||
original: str
|
||||
for placeholder, original in placeholders.items():
|
||||
name = name.replace(placeholder, original)
|
||||
if name != "":
|
||||
pairs.append((name, role))
|
||||
return pairs
|
||||
|
||||
@staticmethod
|
||||
def resolve_artist_identity(name: str) -> tuple[str, str]:
|
||||
"""Resolve the dedup key and the stored spelling of a name.
|
||||
|
||||
The name's case-folded form is looked up in
|
||||
``CANONICAL_ARTIST_NAMES`` first; when it is listed there,
|
||||
the dedup key is the canonical spelling case-folded and the
|
||||
stored spelling is the canonical spelling, so every variant
|
||||
of the name, canonical or not, resolves to the same
|
||||
identity. Otherwise the dedup key is the name case-folded
|
||||
and the stored spelling is the given name.
|
||||
|
||||
:param name: An artist name, as parsed from a credit.
|
||||
:return: A tuple of the dedup key and the stored spelling.
|
||||
"""
|
||||
folded: str = name.casefold()
|
||||
canonical: str | None = \
|
||||
ArtistImporter.CANONICAL_ARTIST_NAMES.get(folded)
|
||||
if canonical is not None:
|
||||
return canonical.casefold(), canonical
|
||||
return folded, name
|
||||
|
||||
|
||||
class CaptureImporter:
|
||||
"""The capture-import job: applies the optional capture-layer
|
||||
inputs onto the stored songs and artists."""
|
||||
|
||||
def __init__(self, session: Session) -> None:
|
||||
"""Initialize the importer.
|
||||
|
||||
:param session: The database session.
|
||||
"""
|
||||
self.__session: Session = session
|
||||
|
||||
def import_captures(self, lyrics_dir: Path | None,
|
||||
wikidata_csv: Path | None) -> None:
|
||||
"""Apply the optional capture-layer inputs onto the store.
|
||||
|
||||
A None input leaves its capture layer unloaded. When
|
||||
``lyrics_dir`` is given, its cached lyrics files load into
|
||||
the matching songs (see `__load_lyrics`). When
|
||||
``wikidata_csv`` is given, it applies onto the artist rows
|
||||
(see `__apply_artist_csv`). When the method returns, the
|
||||
applied changes are queryable in the session.
|
||||
|
||||
:param lyrics_dir: The lyrics cache directory to load, or
|
||||
None to skip the lyrics capture layer.
|
||||
:param wikidata_csv: The Wikidata artist snapshot CSV file
|
||||
to apply, or None to skip the artist capture layer.
|
||||
:return: None.
|
||||
:raises BuildError: When ``lyrics_dir`` does not exist, or
|
||||
a name in ``wikidata_csv`` matches no artist.
|
||||
:raises OSError: When a capture file cannot be read.
|
||||
"""
|
||||
if lyrics_dir is not None:
|
||||
if not lyrics_dir.is_dir():
|
||||
raise BuildError(
|
||||
f"{lyrics_dir}: no such directory")
|
||||
self.__load_lyrics(lyrics_dir)
|
||||
if wikidata_csv is not None:
|
||||
self.__apply_artist_csv(wikidata_csv)
|
||||
self.__session.flush()
|
||||
|
||||
def __load_lyrics(self, directory: Path) -> None:
|
||||
"""Load the cached lyrics files into the matching songs.
|
||||
|
||||
A file whose stem is not an existing song ID is skipped
|
||||
with a warning to the standard error.
|
||||
|
||||
:param directory: The existing lyrics cache directory with
|
||||
one ``<song_id>.txt`` file per song.
|
||||
:return: None.
|
||||
:raises OSError: When a lyrics file cannot be read.
|
||||
"""
|
||||
for path in sorted(directory.glob("*.txt")):
|
||||
song: Song | None = None
|
||||
if path.stem.isdigit():
|
||||
song = self.__session.get(Song, int(path.stem))
|
||||
if song is None:
|
||||
print(f"warning: {path}: no song with ID"
|
||||
f" \"{path.stem}\"", file=sys.stderr)
|
||||
continue
|
||||
song.lyrics = path.read_text(encoding="utf-8")
|
||||
|
||||
def __apply_artist_csv(self, path: Path) -> None:
|
||||
"""Apply an artist attribute CSV onto the artist rows.
|
||||
|
||||
Artists match by exact name. Only the non-empty cells are
|
||||
applied, field by field. The note column is ignored.
|
||||
|
||||
:param path: The CSV file with the columns name, qid,
|
||||
gender, type, genre, country, and note.
|
||||
:return: None.
|
||||
:raises BuildError: When a name matches no artist.
|
||||
:raises OSError: When the file cannot be read.
|
||||
"""
|
||||
with open(path, encoding="utf-8", newline="") as file:
|
||||
row: dict[str, str]
|
||||
for row in csv.DictReader(file):
|
||||
artist: Artist | None = self.__session.scalar(
|
||||
sa.select(Artist)
|
||||
.where(Artist.name == row["name"]))
|
||||
if artist is None:
|
||||
raise BuildError(
|
||||
f"{path}: no artist named"
|
||||
f" \"{row['name']}\"")
|
||||
column: str
|
||||
attribute: str
|
||||
for column, attribute in ARTIST_FIELDS.items():
|
||||
if row.get(column):
|
||||
setattr(artist, attribute, row[column])
|
||||
|
||||
|
||||
@dataclass
|
||||
class StoreCounts:
|
||||
"""The row counts of the working store, for the build summary."""
|
||||
|
||||
songs: int
|
||||
"""The number of the songs."""
|
||||
chart_entries: int
|
||||
"""The number of the chart entries."""
|
||||
artists: int
|
||||
"""The number of the artists."""
|
||||
credits: int
|
||||
"""The number of the song-artist credits."""
|
||||
songs_with_lyrics: int
|
||||
"""The number of the songs with lyrics."""
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls, session: Session) -> Self:
|
||||
"""Counts the loaded rows and returns the counts.
|
||||
|
||||
:param session: The database session with the loaded data
|
||||
flushed.
|
||||
:return: The row counts of the working store.
|
||||
"""
|
||||
def count(selectable: sa.Select[tuple[int]]) -> int:
|
||||
value: int | None = session.scalar(selectable)
|
||||
assert value is not None
|
||||
return value
|
||||
|
||||
return cls(
|
||||
songs=count(
|
||||
sa.select(sa.func.count()).select_from(Song)),
|
||||
chart_entries=count(
|
||||
sa.select(sa.func.count())
|
||||
.select_from(ChartEntry)),
|
||||
artists=count(
|
||||
sa.select(sa.func.count()).select_from(Artist)),
|
||||
credits=count(
|
||||
sa.select(sa.func.count())
|
||||
.select_from(SongArtist)),
|
||||
songs_with_lyrics=count(
|
||||
sa.select(sa.func.count()).select_from(Song)
|
||||
.where(Song.lyrics.is_not(None))))
|
||||
|
||||
|
||||
def reset_store(session: Session) -> None:
|
||||
"""Delete all the rows from every table of the store.
|
||||
|
||||
:param session: The database session.
|
||||
:return: None.
|
||||
"""
|
||||
model: type[Base]
|
||||
for model in (SongArtist, ChartEntry, Song, Artist):
|
||||
session.execute(sa.delete(model))
|
||||
|
||||
|
||||
class CSVExporter:
|
||||
"""Writes the review CSV files mirroring the working store."""
|
||||
|
||||
__SONGS_HEADER: tuple[str, ...] = ("Title", "Artists", "Positions")
|
||||
"""The header row of ``songs.csv``, for human readers."""
|
||||
__ARTISTS_HEADER: tuple[str, ...] = (
|
||||
"Name", "Wikidata QID", "Gender", "Type", "Genre", "Country",
|
||||
"Songs")
|
||||
"""The header row of ``artists.csv``, for human readers."""
|
||||
|
||||
def __init__(self, session: Session, derived_dir: Path) -> None:
|
||||
"""Initialize the exporter.
|
||||
|
||||
:param session: The database session with the loaded data
|
||||
flushed.
|
||||
:param derived_dir: The output directory for the review CSV
|
||||
files.
|
||||
"""
|
||||
self.__session: Session = session
|
||||
self.__derived_dir: Path = derived_dir
|
||||
|
||||
def write(self) -> None:
|
||||
"""Write the review CSV files mirroring the loaded data.
|
||||
|
||||
Fully overwrites ``songs.csv`` and ``artists.csv`` under the
|
||||
output directory, creating it when missing, with normal
|
||||
minimal CSV quoting. Neither file carries a song or an
|
||||
artist ID; a multi-valued field is a plain joined string,
|
||||
itself CSV-quoted as a whole only when its content requires
|
||||
it: "/" joins the chart appearances of one song, and "|"
|
||||
joins the distinct songs credited to one artist.
|
||||
|
||||
:return: None.
|
||||
:raises OSError: When a CSV file cannot be written.
|
||||
"""
|
||||
self.__derived_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.__write_csv(
|
||||
self.__derived_dir / "songs.csv", self.__SONGS_HEADER,
|
||||
self.__songs_rows())
|
||||
self.__write_csv(
|
||||
self.__derived_dir / "artists.csv", self.__ARTISTS_HEADER,
|
||||
self.__artists_rows())
|
||||
|
||||
@staticmethod
|
||||
def __write_csv(path: Path, header: Sequence[str],
|
||||
rows: Iterable[Sequence[str]]) -> None:
|
||||
"""Write a CSV file with LF line endings, fully overwritten.
|
||||
|
||||
:param path: The output CSV file.
|
||||
:param header: The header row.
|
||||
:param rows: The data rows, in the given order.
|
||||
:return: None.
|
||||
:raises OSError: When the file cannot be written.
|
||||
"""
|
||||
with open(path, "w", encoding="utf-8", newline="") as file:
|
||||
writer: Any = csv.writer(file)
|
||||
writer.writerow(header)
|
||||
writer.writerows(rows)
|
||||
|
||||
@staticmethod
|
||||
def __sorted_chart_entries(song: Song) -> list[ChartEntry]:
|
||||
"""Sort the chart entries of a song by year then rank.
|
||||
|
||||
:param song: The song with its chart entries loaded.
|
||||
:return: The chart entries, ordered by year then rank.
|
||||
"""
|
||||
return sorted(
|
||||
song.chart_entries, key=lambda x: (x.year, x.rank))
|
||||
|
||||
@classmethod
|
||||
def __song_positions(cls, song: Song) -> str:
|
||||
"""Format the chart positions of a song for the songs.csv
|
||||
value.
|
||||
|
||||
:param song: The song with its chart entries loaded.
|
||||
:return: The "YEAR#RANK" tokens, ordered by year then rank,
|
||||
joined by "/".
|
||||
"""
|
||||
entries: list[ChartEntry] = cls.__sorted_chart_entries(song)
|
||||
return "/".join(f"{x.year}#{x.rank}" for x in entries)
|
||||
|
||||
@classmethod
|
||||
def __formatted_song_positions(cls, song: Song) -> str:
|
||||
"""Format the chart positions of a song for the artists.csv
|
||||
value.
|
||||
|
||||
:param song: The song with its chart entries loaded.
|
||||
:return: The "YEAR#RANK" tokens, ordered by year then rank,
|
||||
joined by "/".
|
||||
"""
|
||||
entries: list[ChartEntry] = cls.__sorted_chart_entries(song)
|
||||
return "/".join(f"{x.year}#{x.rank}" for x in entries)
|
||||
|
||||
def __songs_rows(self) -> list[list[str]]:
|
||||
"""Build the sorted data rows of ``songs.csv``.
|
||||
|
||||
:return: The rows, sorted by the case-folded title, then
|
||||
the case-folded artist credit.
|
||||
"""
|
||||
songs: list[Song] = sorted(
|
||||
self.__session.scalars(sa.select(Song)),
|
||||
key=lambda x: (x.title.casefold(),
|
||||
x.artist_credit.casefold()))
|
||||
rows: list[list[str]] = []
|
||||
song: Song
|
||||
for song in songs:
|
||||
row: list[str] = [
|
||||
song.title, song.artist_credit,
|
||||
self.__song_positions(song)]
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
@classmethod
|
||||
def __artist_songs(cls, artist: Artist) -> str:
|
||||
"""Format the credited songs for the artists.csv value.
|
||||
|
||||
:param artist: The artist with its song credits loaded.
|
||||
:return: The credited songs, each formatted as
|
||||
"TITLE (YEAR#RANK[/YEAR#RANK...])" with its chart
|
||||
appearances, sorted alphabetically case-folded by
|
||||
title, joined by "|".
|
||||
"""
|
||||
songs: list[Song] = sorted(
|
||||
(x.song for x in artist.song_artists),
|
||||
key=lambda x: x.title.casefold())
|
||||
entries: list[str] = [
|
||||
f"{song.title} ({cls.__formatted_song_positions(song)})"
|
||||
for song in songs]
|
||||
return "|".join(entries)
|
||||
|
||||
def __artists_rows(self) -> list[list[str]]:
|
||||
"""Build the sorted data rows of ``artists.csv``.
|
||||
|
||||
:return: The rows, sorted by the case-folded name.
|
||||
"""
|
||||
artists: list[Artist] = sorted(
|
||||
self.__session.scalars(sa.select(Artist)),
|
||||
key=lambda x: x.name.casefold())
|
||||
rows: list[list[str]] = []
|
||||
artist: Artist
|
||||
for artist in artists:
|
||||
row: list[str] = [
|
||||
artist.name, artist.wikidata_qid or "",
|
||||
artist.gender or "", artist.type or "",
|
||||
artist.genre or "", artist.country or "",
|
||||
self.__artist_songs(artist)]
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""Rebuild the SQLite working store from the inputs.
|
||||
|
||||
:param argv: The command-line arguments, or None for
|
||||
``sys.argv``.
|
||||
:return: The exit status: 0 on success, non-zero on failure.
|
||||
"""
|
||||
args: argparse.Namespace = parse_args(argv)
|
||||
Base.metadata.create_all(ds.engine)
|
||||
session: Session = ds.get_db()
|
||||
counts: StoreCounts
|
||||
try:
|
||||
reset_store(session)
|
||||
SongImporter(session).import_songs(args.chart_csv)
|
||||
ArtistImporter(session).import_artists()
|
||||
CaptureImporter(session).import_captures(
|
||||
args.lyrics_dir, args.wikidata_csv)
|
||||
counts = StoreCounts.get_instance(session)
|
||||
CSVExporter(session, args.derived_dir).write()
|
||||
session.commit()
|
||||
except (OSError, BuildError) as error:
|
||||
session.rollback()
|
||||
print(f"error: {error}", file=sys.stderr)
|
||||
return 1
|
||||
finally:
|
||||
session.close()
|
||||
print(f"done: {counts.songs} songs,"
|
||||
f" {counts.chart_entries} chart entries,"
|
||||
f" {counts.artists} artists,"
|
||||
f" {counts.credits} credits,"
|
||||
f" {counts.songs_with_lyrics} songs with lyrics",
|
||||
file=sys.stderr)
|
||||
return 0
|
||||
@@ -0,0 +1,84 @@
|
||||
# Tools for A Feminist Audit of Pop Music.
|
||||
# Copyright 2026 imacat. All rights reserved.
|
||||
# Authors:
|
||||
# imacat@mail.imacat.idv.tw (imacat), 2026/8/4
|
||||
"""The exporter of the LLM input JSONL file.
|
||||
|
||||
Exports the songs from the SQLite working store into the JSONL
|
||||
input file for the ``run-llm`` subcommand, given as the positional
|
||||
command-line argument. This is the enforcement point of the
|
||||
project's lyrics-only firewall: the output carries only the
|
||||
lyrics text of each song, identified by an opaque song key; no
|
||||
title, artist, or chart data crosses into the LLM input.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..database import ds
|
||||
from ..models import Song
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None) -> argparse.Namespace:
|
||||
"""Parse the command-line arguments.
|
||||
|
||||
:param argv: The command-line arguments, or None for
|
||||
``sys.argv``.
|
||||
:return: The parsed arguments.
|
||||
"""
|
||||
parser: argparse.ArgumentParser = argparse.ArgumentParser(
|
||||
description="Export the LLM input JSONL file (lyrics"
|
||||
" only) from the SQLite working store.")
|
||||
parser.add_argument(
|
||||
"output_jsonl", type=Path,
|
||||
help="the JSONL output file")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def build_lines(session: Session) -> list[str]:
|
||||
"""Build the JSONL lines of every song's lyrics.
|
||||
|
||||
:param session: The database session.
|
||||
:return: The JSON lines, one per song, ordered by song ID.
|
||||
:raises ValueError: When a song has no lyrics.
|
||||
"""
|
||||
lines: list[str] = []
|
||||
song: Song
|
||||
for song in session.scalars(sa.select(Song).order_by(Song.id)):
|
||||
if song.lyrics is None:
|
||||
raise ValueError(
|
||||
f"song {song.id} \"{song.title}\": no lyrics")
|
||||
record: dict[str, str] = {
|
||||
"id": f"song-{song.id}", "content": song.lyrics}
|
||||
lines.append(json.dumps(record, ensure_ascii=False))
|
||||
return lines
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""Export the LLM input JSONL file from the working store.
|
||||
|
||||
:param argv: The command-line arguments, or None for
|
||||
``sys.argv``.
|
||||
:return: The exit status: 0 on success, non-zero on failure.
|
||||
"""
|
||||
args: argparse.Namespace = parse_args(argv)
|
||||
session: Session = ds.get_db()
|
||||
lines: list[str]
|
||||
try:
|
||||
lines = build_lines(session)
|
||||
except (OSError, sa.exc.SQLAlchemyError, ValueError) as error:
|
||||
print(f"error: {error}", file=sys.stderr)
|
||||
return 1
|
||||
finally:
|
||||
session.close()
|
||||
args.output_jsonl.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(args.output_jsonl, "w", encoding="utf-8") as file:
|
||||
line: str
|
||||
for line in lines:
|
||||
file.write(line + "\n")
|
||||
print(f"done: {len(lines)} songs exported", file=sys.stderr)
|
||||
return 0
|
||||
@@ -0,0 +1,840 @@
|
||||
# Tools for A Feminist Audit of Pop Music.
|
||||
# Copyright 2026 imacat. All rights reserved.
|
||||
# Authors:
|
||||
# imacat@mail.imacat.idv.tw (imacat), 2026/7/31
|
||||
"""The fetcher of the artist metadata.
|
||||
|
||||
Fetches the metadata of the artists without a snapshot row from
|
||||
Wikidata into the capture layer: the Wikidata artist snapshot
|
||||
CSV, given as the positional command-line argument. The working
|
||||
store is only read, never written; the ``build-db`` subcommand
|
||||
assembles the captured files into the store on the next rebuild.
|
||||
|
||||
Every fetched row is meant for later human verification: the
|
||||
description of the resolved item is recorded in the note column
|
||||
so that a bad match can be spotted. An unresolved artist or an
|
||||
error on one artist is noted on its row and does not fail the
|
||||
run.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import enum
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import asdict, dataclass, field, fields
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, TextIO
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .. import VERSION
|
||||
from ..database import ds
|
||||
from ..models import Artist, Song, SongArtist
|
||||
from ..utils import format_duration
|
||||
|
||||
API_URL: str = "https://www.wikidata.org/w/api.php"
|
||||
"""The URL of the Wikidata API endpoint."""
|
||||
SPARQL_URL: str = "https://query.wikidata.org/sparql"
|
||||
"""The URL of the Wikidata Query Service SPARQL endpoint."""
|
||||
USER_AGENT: str = (
|
||||
f"pop-fem-audit-tools/{VERSION}"
|
||||
" (https://github.com/imacat/pop-fem-audit;"
|
||||
" mailto:imacat@mail.imacat.idv.tw)")
|
||||
"""The User-Agent header sent on every HTTP request."""
|
||||
TIMEOUT: float = 30.0
|
||||
"""The timeout of an API HTTP request, in seconds."""
|
||||
SPARQL_TIMEOUT: float = 90.0
|
||||
"""The timeout of a SPARQL HTTP request, in seconds.
|
||||
|
||||
Higher than the API timeout: the WDQS server aborts a slow
|
||||
query at 60 seconds, and a lower client timeout would race
|
||||
that server-side abort and misclassify a slow-but-answerable
|
||||
query as a client-side timeout instead of letting the server's
|
||||
own HTTP error response arrive and enter the retry path."""
|
||||
SLEEP_SECONDS: float = 1.0
|
||||
"""The delay between consecutive HTTP requests, in seconds."""
|
||||
MAX_ATTEMPTS: int = 5
|
||||
"""The maximum number of attempts on a transient error."""
|
||||
RETRY_SECONDS: float = 15.0
|
||||
"""The back-off unit on a transient error, in seconds;
|
||||
multiplied by the attempt number already made."""
|
||||
RETRY_STATUSES: frozenset[int] = frozenset({429, 500, 502, 503})
|
||||
"""The HTTP statuses that are retried with a back-off."""
|
||||
MAX_STAGE1_TITLES: int = 3
|
||||
"""The maximum number of charted titles used for the stage-1 song
|
||||
corroboration."""
|
||||
HUMAN_QID: str = "Q5"
|
||||
"""The Wikidata item ID of "human"."""
|
||||
ENSEMBLE_QID: str = "Q2088357"
|
||||
"""The Wikidata item ID of "musical ensemble"."""
|
||||
ORIGINAL_CAST_QID: str = "Q106497009"
|
||||
"""The Wikidata item ID of "original cast"."""
|
||||
GROUP_KEYWORDS: Sequence[str] = ("band", "group", "duo", "trio")
|
||||
"""The label keywords that suggest a musical ensemble, covering
|
||||
labels like "boy band" and "girl group"."""
|
||||
NOTE_NOT_FOUND: str = "not found"
|
||||
"""The note sentinel of an artist without a resolved Wikidata
|
||||
item, written to the snapshot and read back for the
|
||||
classification."""
|
||||
PINNED_QIDS: dict[str, str] = {
|
||||
"Pinkfong": "Q55735607",
|
||||
}
|
||||
"""The last-resort pinned item IDs, keyed by the artist name.
|
||||
|
||||
Each entry is for an artist the algorithm documented on
|
||||
``ArtistFetcher`` is structurally unable to resolve, with its
|
||||
justification recorded here:
|
||||
|
||||
- "Pinkfong": the only charting act whose item is typed as a
|
||||
brand (P31 = Q431289), which the type gate (human / musical
|
||||
ensemble / original cast) excludes by design.
|
||||
|
||||
A pinned name skips the candidate retrieval and corroboration
|
||||
steps; its item ID is used directly."""
|
||||
|
||||
|
||||
class ArtistType(enum.StrEnum):
|
||||
"""The decided artist type of a snapshot row."""
|
||||
|
||||
SOLO = "solo"
|
||||
"""A solo artist: a human."""
|
||||
GROUP = "group"
|
||||
"""A musical ensemble."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ArtistSnapshot:
|
||||
"""One row of the Wikidata artist snapshot CSV file."""
|
||||
|
||||
name: str
|
||||
"""The artist name."""
|
||||
qid: str = ""
|
||||
"""The Wikidata item ID, or empty when unresolved."""
|
||||
gender: str = ""
|
||||
"""The gender label, or empty when unresolved."""
|
||||
type: str = ""
|
||||
"""The artist type, an ``ArtistType`` value, or empty for the
|
||||
human to decide."""
|
||||
genre: str = ""
|
||||
"""The genre labels, joined with ``; ``."""
|
||||
country: str = ""
|
||||
"""The country label, or empty when unresolved."""
|
||||
note: str = ""
|
||||
"""The note for human verification: the description of the
|
||||
resolved item, ``not found``, or ``error: <reason>``."""
|
||||
|
||||
def to_row(self) -> dict[str, str]:
|
||||
"""Return this snapshot as a CSV row.
|
||||
|
||||
:return: The row values, keyed by the column name.
|
||||
"""
|
||||
return asdict(self)
|
||||
|
||||
|
||||
SNAPSHOT_FIELDS: Sequence[str] = tuple(
|
||||
x.name for x in fields(ArtistSnapshot))
|
||||
"""The header columns of the Wikidata artist snapshot CSV file."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ArtistClaims:
|
||||
"""The item-ID claim targets and description of a Wikidata
|
||||
artist item."""
|
||||
|
||||
gender_ids: list[str] = field(default_factory=list)
|
||||
"""The item IDs of the gender targets."""
|
||||
instance_of_ids: list[str] = field(default_factory=list)
|
||||
"""The item IDs of the instance-of targets."""
|
||||
genre_ids: list[str] = field(default_factory=list)
|
||||
"""The item IDs of the genre targets."""
|
||||
country_ids: list[str] = field(default_factory=list)
|
||||
"""The item IDs of the country-of-citizenship targets."""
|
||||
origin_country_ids: list[str] = field(default_factory=list)
|
||||
"""The item IDs of the country-of-origin targets."""
|
||||
description: str = ""
|
||||
"""The English description of the item, or empty when
|
||||
absent."""
|
||||
|
||||
|
||||
class RetryExhausted(Exception):
|
||||
"""The retries on a transient error are exhausted.
|
||||
|
||||
A transient error is a retryable HTTP status (429, 500,
|
||||
502, or 503) or a read timeout.
|
||||
"""
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None) -> argparse.Namespace:
|
||||
"""Parse the command-line arguments.
|
||||
|
||||
:param argv: The command-line arguments, or None for
|
||||
``sys.argv``.
|
||||
:return: The parsed arguments.
|
||||
"""
|
||||
parser: argparse.ArgumentParser = argparse.ArgumentParser(
|
||||
description="Fetch the artist metadata from Wikidata"
|
||||
" into the capture layer.")
|
||||
parser.add_argument(
|
||||
"wikidata_csv", type=Path,
|
||||
help="the Wikidata artist snapshot CSV file")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
class ArtistFetcher:
|
||||
"""A fetcher of artist metadata from Wikidata.
|
||||
|
||||
An artist name is resolved to a Wikidata item ID through the
|
||||
Wikidata Query Service (SPARQL), not the search API, so that
|
||||
every step is a deterministic indexed lookup with no ranking
|
||||
and no case folding of the label text itself:
|
||||
|
||||
1. Candidate retrieval: the items whose ``rdfs:label`` or
|
||||
``skos:altLabel`` exactly equals the artist name at
|
||||
``@en`` or ``@mul`` (the multilingual default layer),
|
||||
restricted to a human (P31 = Q5), a musical ensemble
|
||||
(P31/P279* = Q2088357), or an original cast
|
||||
(P31 = Q106497009).
|
||||
2. A single candidate is selected outright.
|
||||
3. With multiple candidates, stage 1 corroborates with up to
|
||||
the first 3 charted titles: the song items whose label or
|
||||
alias exactly equals a title at ``@en`` or ``@mul`` are
|
||||
looked up with their P175 performers and, optionally,
|
||||
those performers' P527 members. The candidates are
|
||||
intersected with the performers, and, only when that
|
||||
intersection is empty, with the members; a stage succeeds
|
||||
only when the intersection has exactly one item.
|
||||
4. Stage 2 is an anchored, case-insensitive fallback: every
|
||||
song performed by a candidate, directly or via a parent
|
||||
group, is compared against the charted titles with a
|
||||
casefold match on the song label, without a language
|
||||
restriction; a single matching candidate is selected.
|
||||
5. Zero candidates, or no stage narrowing to exactly one
|
||||
item, leaves the artist unresolved.
|
||||
|
||||
As a last resort, a name listed in ``PINNED_QIDS`` uses its
|
||||
pinned item ID directly, skipping every step above.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Construct the fetcher."""
|
||||
self.__sent: int = 0
|
||||
"""The number of the HTTP requests already sent."""
|
||||
|
||||
def fetch(self, name: str,
|
||||
titles: Sequence[str]) -> ArtistSnapshot:
|
||||
"""Fetch the metadata of an artist.
|
||||
|
||||
The note carries the description of the resolved item
|
||||
for human verification. ``not found`` is reserved for
|
||||
the algorithm genuinely finding nothing for the artist.
|
||||
Any HTTP, network, or decoding error -- including the
|
||||
retries on a transient error being exhausted -- yields a
|
||||
snapshot with what was resolved so far and the note
|
||||
``error: <reason>``.
|
||||
|
||||
:param name: The artist name to resolve.
|
||||
:param titles: The charted song titles credited to the
|
||||
artist, used for the multi-candidate resolution.
|
||||
:return: The snapshot of the artist.
|
||||
"""
|
||||
snapshot: ArtistSnapshot = ArtistSnapshot(name=name)
|
||||
try:
|
||||
qid: str | None = self.__resolve_qid(name, titles)
|
||||
if qid is None:
|
||||
snapshot.note = NOTE_NOT_FOUND
|
||||
return snapshot
|
||||
snapshot.qid = qid
|
||||
self.__resolve(snapshot)
|
||||
except (RetryExhausted, OSError, ValueError) as error:
|
||||
snapshot.note = f"error: {error}"
|
||||
return snapshot
|
||||
|
||||
def __resolve_qid(self, name: str,
|
||||
titles: Sequence[str]) -> str | None:
|
||||
"""Resolve an artist name to a Wikidata item ID.
|
||||
|
||||
:param name: The artist name.
|
||||
:param titles: The charted song titles credited to the
|
||||
artist.
|
||||
:return: The pinned item ID from ``PINNED_QIDS`` when the
|
||||
name is listed there; otherwise the resolved item
|
||||
ID, or None when the algorithm documented on this
|
||||
class does not narrow to exactly one item.
|
||||
:raises OSError: On a non-retryable HTTP or network
|
||||
error.
|
||||
:raises RetryExhausted: When the retries on a
|
||||
transient error are exhausted.
|
||||
:raises ValueError: On a JSON decoding error.
|
||||
"""
|
||||
if name in PINNED_QIDS:
|
||||
return PINNED_QIDS[name]
|
||||
candidates: list[str] = self.__candidates(name)
|
||||
if len(candidates) == 0:
|
||||
return None
|
||||
if len(candidates) == 1:
|
||||
return candidates[0]
|
||||
selected: str | None = self.__stage1(candidates, titles)
|
||||
if selected is not None:
|
||||
return selected
|
||||
return self.__stage2(candidates, titles)
|
||||
|
||||
def __candidates(self, name: str) -> list[str]:
|
||||
"""Retrieve the Wikidata items matching an artist name.
|
||||
|
||||
:param name: The artist name.
|
||||
:return: The item IDs of the human, musical-ensemble, or
|
||||
original-cast items whose label or alias exactly
|
||||
equals the name.
|
||||
:raises OSError: On a non-retryable HTTP or network
|
||||
error.
|
||||
:raises RetryExhausted: When the retries on a
|
||||
transient error are exhausted.
|
||||
:raises ValueError: On a JSON decoding error.
|
||||
"""
|
||||
query: str = f"""
|
||||
SELECT DISTINCT ?item WHERE {{
|
||||
VALUES ?name {{ {self.__literals([name])} }}
|
||||
{{ ?item rdfs:label ?name }}
|
||||
UNION {{ ?item skos:altLabel ?name }}
|
||||
{{
|
||||
?item wdt:P31 wd:{HUMAN_QID}
|
||||
}} UNION {{
|
||||
?item wdt:P31/wdt:P279* wd:{ENSEMBLE_QID}
|
||||
}} UNION {{
|
||||
?item wdt:P31 wd:{ORIGINAL_CAST_QID}
|
||||
}}
|
||||
}}
|
||||
"""
|
||||
rows: list[dict[str, str]] = self.__sparql(query)
|
||||
return [self.__qid(x["item"]) for x in rows
|
||||
if "item" in x]
|
||||
|
||||
def __stage1(self, candidates: Sequence[str],
|
||||
titles: Sequence[str]) -> str | None:
|
||||
"""Corroborate the candidates with the charted titles.
|
||||
|
||||
:param candidates: The candidate item IDs.
|
||||
:param titles: The charted song titles credited to the
|
||||
artist.
|
||||
:return: The single candidate among the performers of a
|
||||
matching song, or, only when no such single
|
||||
candidate exists, among those performers' group
|
||||
members; None when neither intersection has exactly
|
||||
one item, or there are no titles to try.
|
||||
:raises OSError: On a non-retryable HTTP or network
|
||||
error.
|
||||
:raises RetryExhausted: When the retries on a
|
||||
transient error are exhausted.
|
||||
:raises ValueError: On a JSON decoding error.
|
||||
"""
|
||||
subset: Sequence[str] = titles[:MAX_STAGE1_TITLES]
|
||||
if len(subset) == 0:
|
||||
return None
|
||||
query: str = f"""
|
||||
SELECT DISTINCT ?performer ?member WHERE {{
|
||||
VALUES ?title {{ {self.__literals(subset)} }}
|
||||
{{ ?song rdfs:label ?title }}
|
||||
UNION {{ ?song skos:altLabel ?title }}
|
||||
?song wdt:P175 ?performer .
|
||||
OPTIONAL {{ ?performer wdt:P527 ?member . }}
|
||||
}}
|
||||
"""
|
||||
rows: list[dict[str, str]] = self.__sparql(query)
|
||||
performers: set[str] = {
|
||||
self.__qid(x["performer"]) for x in rows
|
||||
if "performer" in x}
|
||||
hit: set[str] = set(candidates) & performers
|
||||
if len(hit) == 1:
|
||||
return next(iter(hit))
|
||||
members: set[str] = {
|
||||
self.__qid(x["member"]) for x in rows
|
||||
if "member" in x}
|
||||
hit = set(candidates) & members
|
||||
if len(hit) == 1:
|
||||
return next(iter(hit))
|
||||
return None
|
||||
|
||||
def __stage2(self, candidates: Sequence[str],
|
||||
titles: Sequence[str]) -> str | None:
|
||||
"""Rescue a single candidate by a case-insensitive match.
|
||||
|
||||
:param candidates: The candidate item IDs.
|
||||
:param titles: The charted song titles credited to the
|
||||
artist.
|
||||
:return: The single candidate with a charted song among
|
||||
the songs it, or a parent group, performs, matched
|
||||
case-insensitively against the song label; None when
|
||||
no such single candidate exists.
|
||||
:raises OSError: On a non-retryable HTTP or network
|
||||
error.
|
||||
:raises RetryExhausted: When the retries on a
|
||||
transient error are exhausted.
|
||||
:raises ValueError: On a JSON decoding error.
|
||||
"""
|
||||
values: str = " ".join(f"wd:{x}" for x in candidates)
|
||||
query: str = f"""
|
||||
SELECT DISTINCT ?cand ?label WHERE {{
|
||||
VALUES ?cand {{ {values} }}
|
||||
{{ ?song wdt:P175 ?cand }}
|
||||
UNION {{ ?g wdt:P527 ?cand . ?song wdt:P175 ?g }}
|
||||
?song rdfs:label ?label .
|
||||
}}
|
||||
"""
|
||||
rows: list[dict[str, str]] = self.__sparql(query)
|
||||
folded: set[str] = {x.casefold() for x in titles}
|
||||
hits: set[str] = {
|
||||
self.__qid(x["cand"]) for x in rows
|
||||
if "cand" in x and "label" in x
|
||||
and x["label"].casefold() in folded}
|
||||
if len(hits) == 1:
|
||||
return next(iter(hits))
|
||||
return None
|
||||
|
||||
def __resolve(self, snapshot: ArtistSnapshot) -> None:
|
||||
"""Resolve the claims of an artist into the snapshot.
|
||||
|
||||
:param snapshot: The snapshot, with the QID set.
|
||||
:return: None.
|
||||
:raises OSError: On a non-retryable HTTP or network
|
||||
error.
|
||||
:raises RetryExhausted: When the retries on a
|
||||
transient error are exhausted.
|
||||
:raises ValueError: On a JSON decoding error.
|
||||
"""
|
||||
claims: ArtistClaims = self.__get_claims(snapshot.qid)
|
||||
snapshot.note = claims.description
|
||||
country_ids: list[str] = claims.country_ids
|
||||
if len(country_ids) == 0:
|
||||
country_ids = claims.origin_country_ids
|
||||
labels: dict[str, str] = self.__get_labels(
|
||||
claims.gender_ids + claims.instance_of_ids
|
||||
+ claims.genre_ids + country_ids)
|
||||
if len(claims.gender_ids) > 0:
|
||||
snapshot.gender = labels.get(claims.gender_ids[0], "")
|
||||
snapshot.type = self.__artist_type(
|
||||
claims.instance_of_ids, labels)
|
||||
snapshot.genre = "; ".join(
|
||||
labels[x] for x in claims.genre_ids if x in labels)
|
||||
if len(country_ids) > 0:
|
||||
snapshot.country = labels.get(country_ids[0], "")
|
||||
|
||||
def __get_claims(self, qid: str) -> ArtistClaims:
|
||||
"""Fetch the claims and the description of a Wikidata
|
||||
item.
|
||||
|
||||
:param qid: The item ID.
|
||||
:return: The item-ID targets of the gender, instance-of,
|
||||
genre, and country properties, and the English
|
||||
description.
|
||||
:raises OSError: On a non-retryable HTTP or network
|
||||
error.
|
||||
:raises RetryExhausted: When the retries on a
|
||||
transient error are exhausted.
|
||||
:raises ValueError: On a JSON decoding error.
|
||||
"""
|
||||
data: Any = self.__get_json({
|
||||
"action": "wbgetentities", "ids": qid,
|
||||
"props": "claims|descriptions", "languages": "en",
|
||||
"format": "json"})
|
||||
entity: Any = None
|
||||
if isinstance(data, dict) \
|
||||
and isinstance(data.get("entities"), dict):
|
||||
entity = data["entities"].get(qid)
|
||||
claims: Any = entity.get("claims") \
|
||||
if isinstance(entity, dict) else None
|
||||
if not isinstance(claims, dict):
|
||||
claims = {}
|
||||
return ArtistClaims(
|
||||
gender_ids=self.__targets(claims.get("P21")),
|
||||
instance_of_ids=self.__targets(claims.get("P31")),
|
||||
genre_ids=self.__targets(claims.get("P136")),
|
||||
country_ids=self.__targets(claims.get("P27")),
|
||||
origin_country_ids=self.__targets(claims.get("P495")),
|
||||
description=self.__description(entity))
|
||||
|
||||
@staticmethod
|
||||
def __description(entity: Any) -> str:
|
||||
"""Extract the English description of a Wikidata entity.
|
||||
|
||||
:param entity: The entity data, or None.
|
||||
:return: The description, or the empty string when
|
||||
absent.
|
||||
"""
|
||||
descriptions: Any = entity.get("descriptions") \
|
||||
if isinstance(entity, dict) else None
|
||||
if not isinstance(descriptions, dict):
|
||||
return ""
|
||||
description: Any = descriptions.get("en")
|
||||
if isinstance(description, dict) \
|
||||
and isinstance(description.get("value"), str):
|
||||
return description["value"]
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def __targets(statements: Any) -> list[str]:
|
||||
"""Extract the item-ID targets of the property statements.
|
||||
|
||||
:param statements: The statements of a property, or None.
|
||||
:return: The item IDs of the statement targets.
|
||||
"""
|
||||
if not isinstance(statements, list):
|
||||
return []
|
||||
ids: list[str] = []
|
||||
statement: Any
|
||||
for statement in statements:
|
||||
if not isinstance(statement, dict):
|
||||
continue
|
||||
snak: Any = statement.get("mainsnak")
|
||||
if not isinstance(snak, dict):
|
||||
continue
|
||||
datavalue: Any = snak.get("datavalue")
|
||||
if not isinstance(datavalue, dict):
|
||||
continue
|
||||
value: Any = datavalue.get("value")
|
||||
if isinstance(value, dict) \
|
||||
and isinstance(value.get("id"), str):
|
||||
ids.append(value["id"])
|
||||
return ids
|
||||
|
||||
def __get_labels(self, qids: Sequence[str]) \
|
||||
-> dict[str, str]:
|
||||
"""Resolve item IDs to their English labels in one batch.
|
||||
|
||||
:param qids: The item IDs, duplicates allowed.
|
||||
:return: The English labels, keyed by the item ID; the
|
||||
items without an English label are left out.
|
||||
:raises OSError: On a non-retryable HTTP or network
|
||||
error.
|
||||
:raises RetryExhausted: When the retries on a
|
||||
transient error are exhausted.
|
||||
:raises ValueError: On a JSON decoding error.
|
||||
"""
|
||||
unique: list[str] = list(dict.fromkeys(qids))
|
||||
if len(unique) == 0:
|
||||
return {}
|
||||
data: Any = self.__get_json({
|
||||
"action": "wbgetentities", "ids": "|".join(unique),
|
||||
"props": "labels", "languages": "en",
|
||||
"format": "json"})
|
||||
entities: Any = data.get("entities") \
|
||||
if isinstance(data, dict) else None
|
||||
if not isinstance(entities, dict):
|
||||
return {}
|
||||
labels: dict[str, str] = {}
|
||||
qid: str
|
||||
for qid in unique:
|
||||
entity: Any = entities.get(qid)
|
||||
if not isinstance(entity, dict) \
|
||||
or not isinstance(entity.get("labels"), dict):
|
||||
continue
|
||||
label: Any = entity["labels"].get("en")
|
||||
if isinstance(label, dict) \
|
||||
and isinstance(label.get("value"), str):
|
||||
labels[qid] = label["value"]
|
||||
return labels
|
||||
|
||||
@staticmethod
|
||||
def __artist_type(type_ids: Sequence[str],
|
||||
labels: dict[str, str]) \
|
||||
-> ArtistType | Literal[""]:
|
||||
"""Derive the artist type from the instance-of targets.
|
||||
|
||||
:param type_ids: The item IDs of the instance-of targets.
|
||||
:param labels: The English labels, keyed by the item ID.
|
||||
:return: ``ArtistType.SOLO`` for a human,
|
||||
``ArtistType.GROUP`` for a musical ensemble, or the
|
||||
empty string for the human to decide.
|
||||
"""
|
||||
if HUMAN_QID in type_ids:
|
||||
return ArtistType.SOLO
|
||||
qid: str
|
||||
for qid in type_ids:
|
||||
label: str = labels.get(qid, "").lower()
|
||||
if any(x in label for x in GROUP_KEYWORDS):
|
||||
return ArtistType.GROUP
|
||||
return ""
|
||||
|
||||
def __sparql(self, query: str) -> list[dict[str, str]]:
|
||||
"""Run a SPARQL query against the Wikidata Query Service.
|
||||
|
||||
:param query: The SPARQL query text.
|
||||
:return: The result bindings, each variable name mapped
|
||||
to its bound value.
|
||||
:raises OSError: On a non-retryable HTTP or network
|
||||
error.
|
||||
:raises RetryExhausted: When the retries on a
|
||||
transient error are exhausted.
|
||||
:raises ValueError: On a JSON decoding error.
|
||||
"""
|
||||
url: str = (f"{SPARQL_URL}?"
|
||||
f"{urllib.parse.urlencode({'query': query})}")
|
||||
request: urllib.request.Request = urllib.request.Request(
|
||||
url, headers={
|
||||
"User-Agent": USER_AGENT,
|
||||
"Accept": "application/sparql-results+json"})
|
||||
body: bytes = self.__send(
|
||||
request, timeout=SPARQL_TIMEOUT)
|
||||
data: Any = json.loads(body)
|
||||
bindings: Any = None
|
||||
if isinstance(data, dict) \
|
||||
and isinstance(data.get("results"), dict):
|
||||
bindings = data["results"].get("bindings")
|
||||
if not isinstance(bindings, list):
|
||||
return []
|
||||
rows: list[dict[str, str]] = []
|
||||
binding: Any
|
||||
for binding in bindings:
|
||||
if not isinstance(binding, dict):
|
||||
continue
|
||||
row: dict[str, str] = {}
|
||||
key: str
|
||||
cell: Any
|
||||
for key, cell in binding.items():
|
||||
if isinstance(cell, dict) \
|
||||
and isinstance(cell.get("value"), str):
|
||||
row[key] = cell["value"]
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
def __get_json(self, params: dict[str, str]) -> Any:
|
||||
"""Send a GET request to the API and return the JSON body.
|
||||
|
||||
:param params: The query parameters.
|
||||
:return: The parsed JSON body.
|
||||
:raises OSError: On a non-retryable HTTP or network
|
||||
error.
|
||||
:raises RetryExhausted: When the retries on a
|
||||
transient error are exhausted.
|
||||
:raises ValueError: On a JSON decoding error.
|
||||
"""
|
||||
url: str = f"{API_URL}?{urllib.parse.urlencode(params)}"
|
||||
request: urllib.request.Request = urllib.request.Request(
|
||||
url, headers={"User-Agent": USER_AGENT})
|
||||
return json.loads(self.__send(request))
|
||||
|
||||
def __send(self, request: urllib.request.Request,
|
||||
timeout: float = TIMEOUT) -> bytes:
|
||||
"""Send an HTTP request, retrying on a transient error.
|
||||
|
||||
Consecutive requests are separated by a fixed delay. A
|
||||
transient error -- a response with a retryable HTTP
|
||||
status, or a read timeout -- is retried with an
|
||||
increasing back-off, up to ``MAX_ATTEMPTS`` attempts in
|
||||
total.
|
||||
|
||||
:param request: The prepared HTTP request.
|
||||
:param timeout: The read timeout, in seconds.
|
||||
:return: The raw response body.
|
||||
:raises OSError: On a non-retryable HTTP or network
|
||||
error.
|
||||
:raises RetryExhausted: When the retries on a
|
||||
transient error are exhausted.
|
||||
"""
|
||||
if self.__sent > 0:
|
||||
time.sleep(SLEEP_SECONDS)
|
||||
self.__sent += 1
|
||||
attempt: int = 1
|
||||
reason: str
|
||||
cause: BaseException
|
||||
while True:
|
||||
try:
|
||||
with urllib.request.urlopen(
|
||||
request, timeout=timeout) as response:
|
||||
return response.read()
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code not in RETRY_STATUSES:
|
||||
raise
|
||||
reason = str(error)
|
||||
cause = error
|
||||
except TimeoutError as error:
|
||||
reason = str(error) or "timed out"
|
||||
cause = error
|
||||
except urllib.error.URLError as error:
|
||||
if not isinstance(error.reason, TimeoutError):
|
||||
raise
|
||||
reason = str(error.reason) or "timed out"
|
||||
cause = error
|
||||
if attempt >= MAX_ATTEMPTS:
|
||||
raise RetryExhausted(
|
||||
f"retries exhausted ({reason})") from cause
|
||||
time.sleep(RETRY_SECONDS * attempt)
|
||||
attempt += 1
|
||||
|
||||
@staticmethod
|
||||
def __literals(texts: Sequence[str]) -> str:
|
||||
"""Build the SPARQL literals of texts at both languages.
|
||||
|
||||
:param texts: The texts to embed as string literals.
|
||||
:return: The literals, each text once tagged ``@en`` and
|
||||
once tagged ``@mul``, space-separated.
|
||||
"""
|
||||
parts: list[str] = []
|
||||
text: str
|
||||
for text in texts:
|
||||
escaped: str = ArtistFetcher.__escape(text)
|
||||
parts.append(f'"{escaped}"@en')
|
||||
parts.append(f'"{escaped}"@mul')
|
||||
return " ".join(parts)
|
||||
|
||||
@staticmethod
|
||||
def __escape(text: str) -> str:
|
||||
"""Escape a text for embedding as a SPARQL string literal.
|
||||
|
||||
:param text: The text to embed.
|
||||
:return: The text with the backslashes and double quotes
|
||||
escaped.
|
||||
"""
|
||||
return text.replace("\\", "\\\\").replace('"', '\\"')
|
||||
|
||||
@staticmethod
|
||||
def __qid(uri: str) -> str:
|
||||
"""Extract the item ID from a Wikidata entity URI.
|
||||
|
||||
:param uri: The entity URI.
|
||||
:return: The item ID, the last path segment of the URI.
|
||||
"""
|
||||
return uri.rsplit("/", 1)[-1]
|
||||
|
||||
|
||||
def read_snapshot_rows(file: TextIO) -> list[dict[str, str]]:
|
||||
"""Read the current rows of a snapshot CSV file handle.
|
||||
|
||||
:param file: The open, seekable snapshot CSV file.
|
||||
:return: The rows, keyed by the column name.
|
||||
:raises OSError: When the file cannot be read.
|
||||
"""
|
||||
file.seek(0)
|
||||
reader: csv.DictReader[str] = csv.DictReader(file)
|
||||
return list(reader)
|
||||
|
||||
|
||||
def read_artist_titles(session: Session,
|
||||
artist_id: int) -> list[str]:
|
||||
"""Read the charted song titles credited to an artist.
|
||||
|
||||
:param session: The database session.
|
||||
:param artist_id: The artist ID.
|
||||
:return: The song titles credited to the artist, ordered by
|
||||
the song ID, with the duplicate titles removed.
|
||||
"""
|
||||
titles: Sequence[str] = session.scalars(
|
||||
sa.select(Song.title)
|
||||
.join(SongArtist, SongArtist.song_id == Song.id)
|
||||
.where(SongArtist.artist_id == artist_id)
|
||||
.order_by(Song.id)).all()
|
||||
return list(dict.fromkeys(titles))
|
||||
|
||||
|
||||
def ensure_snapshot_header(file: TextIO) -> None:
|
||||
"""Write the snapshot CSV header row if the file is empty.
|
||||
|
||||
:param file: The open, seekable snapshot CSV file.
|
||||
:return: None.
|
||||
:raises OSError: When the file cannot be written.
|
||||
"""
|
||||
file.seek(0, os.SEEK_END)
|
||||
if file.tell() == 0:
|
||||
csv.writer(file).writerow(SNAPSHOT_FIELDS)
|
||||
file.flush()
|
||||
|
||||
|
||||
def append_row(file: TextIO, snapshot: ArtistSnapshot) -> None:
|
||||
"""Append a snapshot row to a snapshot CSV file handle.
|
||||
|
||||
:param file: The open snapshot CSV file, opened for append.
|
||||
:param snapshot: The snapshot of an artist.
|
||||
:return: None.
|
||||
:raises OSError: When the file cannot be written.
|
||||
"""
|
||||
csv.DictWriter(file, SNAPSHOT_FIELDS).writerow(
|
||||
snapshot.to_row())
|
||||
file.flush()
|
||||
|
||||
|
||||
def write_snapshot(file: TextIO) -> None:
|
||||
"""Rewrite a snapshot CSV file handle sorted by artist name.
|
||||
|
||||
Reads back the current rows and rewrites the header and the
|
||||
rows ordered by the case-folded artist name, matching the
|
||||
convention of the derived ``artists.csv``.
|
||||
|
||||
:param file: The open, seekable snapshot CSV file.
|
||||
:return: None.
|
||||
:raises OSError: When the file cannot be read or written.
|
||||
"""
|
||||
ordered: list[dict[str, str]] = sorted(
|
||||
read_snapshot_rows(file),
|
||||
key=lambda row: row["name"].casefold())
|
||||
file.seek(0)
|
||||
file.truncate()
|
||||
writer: csv.DictWriter[str] = csv.DictWriter(
|
||||
file, SNAPSHOT_FIELDS)
|
||||
writer.writeheader()
|
||||
writer.writerows(ordered)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""Fetch the artist metadata from Wikidata.
|
||||
|
||||
:param argv: The command-line arguments, or None for
|
||||
``sys.argv``.
|
||||
:return: The exit status: 0 on success, misses and errors
|
||||
included, non-zero on a setup error.
|
||||
"""
|
||||
started: float = time.monotonic()
|
||||
args: argparse.Namespace = parse_args(argv)
|
||||
fetcher: ArtistFetcher = ArtistFetcher()
|
||||
fetched: int = 0
|
||||
not_found: int = 0
|
||||
errors: int = 0
|
||||
skipped: int = 0
|
||||
session: Session = ds.get_db()
|
||||
try:
|
||||
args.wikidata_csv.parent.mkdir(
|
||||
parents=True, exist_ok=True)
|
||||
with open(args.wikidata_csv, "a+", encoding="utf-8",
|
||||
newline="") as csv_file:
|
||||
done: set[str] = {x["name"] for x in
|
||||
read_snapshot_rows(csv_file)}
|
||||
ensure_snapshot_header(csv_file)
|
||||
artist: Artist
|
||||
for artist in session.scalars(
|
||||
sa.select(Artist).order_by(Artist.id)):
|
||||
if artist.name in done:
|
||||
skipped += 1
|
||||
continue
|
||||
titles: list[str] = read_artist_titles(
|
||||
session, artist.id)
|
||||
snapshot: ArtistSnapshot = fetcher.fetch(
|
||||
artist.name, titles)
|
||||
append_row(csv_file, snapshot)
|
||||
status: str = snapshot.qid
|
||||
if snapshot.note == NOTE_NOT_FOUND:
|
||||
not_found += 1
|
||||
status = "not found"
|
||||
elif snapshot.note.startswith("error: "):
|
||||
errors += 1
|
||||
status = snapshot.note
|
||||
else:
|
||||
fetched += 1
|
||||
print(f"artist \"{artist.name}\": {status}",
|
||||
file=sys.stderr)
|
||||
write_snapshot(csv_file)
|
||||
except (OSError, sa.exc.SQLAlchemyError) as error:
|
||||
print(f"error: {error}", file=sys.stderr)
|
||||
return 1
|
||||
finally:
|
||||
session.close()
|
||||
attempted: int = fetched + not_found + errors
|
||||
elapsed: str = format_duration(time.monotonic() - started)
|
||||
print(f"Done. Resolved {fetched}/{attempted} artists."
|
||||
f" {elapsed} elapsed.",
|
||||
file=sys.stderr)
|
||||
return 0
|
||||
@@ -0,0 +1,274 @@
|
||||
# Tools for A Feminist Audit of Pop Music.
|
||||
# Copyright 2026 imacat. All rights reserved.
|
||||
# Authors:
|
||||
# imacat@mail.imacat.idv.tw (imacat), 2026/7/31
|
||||
"""The fetcher of the missing song lyrics.
|
||||
|
||||
Fetches the lyrics of the songs without a cache file from the
|
||||
public lyrics APIs, Lyrics.ovh and LRCLIB, into the capture
|
||||
layer: the lyrics cache directory and the provenance CSV, each
|
||||
given as a positional command-line argument. The working store
|
||||
is only read, never written; the ``build-db`` subcommand
|
||||
assembles the captured files into the store on the next rebuild.
|
||||
|
||||
Each song is first queried with the name of its primary-role
|
||||
artist with the lowest position. When every API misses that
|
||||
query and the song's full artist credit differs from that
|
||||
artist name, the same APIs are queried again with the artist
|
||||
credit, to catch songs cataloged only under a joint credit such
|
||||
as "Dan + Shay".
|
||||
|
||||
A song that every API misses on both queries is reported on the
|
||||
standard error, but does not fail the run; misses are expected.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import datetime
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..database import ds
|
||||
from ..models import (
|
||||
Artist,
|
||||
Role,
|
||||
Song,
|
||||
SongArtist,
|
||||
)
|
||||
from ..utils import format_duration
|
||||
|
||||
PROVENANCE_FIELDS: Sequence[str] = (
|
||||
"song_id", "source", "method", "acquired_at", "note")
|
||||
"""The header columns of the lyrics provenance CSV file."""
|
||||
USER_AGENT: str = ("pop-fem-audit-tools"
|
||||
" (https://github.com/imacat/pop-fem-audit)")
|
||||
"""The User-Agent header sent on every HTTP request."""
|
||||
TIMEOUT: float = 30.0
|
||||
"""The timeout of an HTTP request, in seconds."""
|
||||
SLEEP_SECONDS: float = 1.0
|
||||
"""The delay between consecutive HTTP requests, in seconds."""
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None) -> argparse.Namespace:
|
||||
"""Parse the command-line arguments.
|
||||
|
||||
:param argv: The command-line arguments, or None for
|
||||
``sys.argv``.
|
||||
:return: The parsed arguments.
|
||||
"""
|
||||
parser: argparse.ArgumentParser = argparse.ArgumentParser(
|
||||
description="Fetch the missing song lyrics from the"
|
||||
" public lyrics APIs into the capture layer.")
|
||||
parser.add_argument(
|
||||
"lyrics_dir", type=Path,
|
||||
help="the lyrics cache directory")
|
||||
parser.add_argument(
|
||||
"provenance_csv", type=Path,
|
||||
help="the lyrics provenance CSV file")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
class LyricsFetcher:
|
||||
"""A fetcher of song lyrics from the public lyrics APIs."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Construct the fetcher."""
|
||||
self.__sent: int = 0
|
||||
"""The number of the HTTP requests already sent."""
|
||||
|
||||
def fetch(self, artist: str, title: str) \
|
||||
-> tuple[str, str] | None:
|
||||
"""Fetch the lyrics of a song.
|
||||
|
||||
The APIs are tried in order, Lyrics.ovh and then LRCLIB,
|
||||
stopping at the first hit. An HTTP or network error on
|
||||
an API counts as a miss on that API.
|
||||
|
||||
:param artist: The artist name to query with.
|
||||
:param title: The song title to query with.
|
||||
:return: The lyrics text and the source name, or None
|
||||
when every API misses.
|
||||
"""
|
||||
lyrics: str | None = self.__fetch_ovh(artist, title)
|
||||
if lyrics is not None:
|
||||
return lyrics, "lyrics.ovh"
|
||||
lyrics = self.__fetch_lrclib(artist, title)
|
||||
if lyrics is not None:
|
||||
return lyrics, "lrclib"
|
||||
return None
|
||||
|
||||
def __fetch_ovh(self, artist: str, title: str) -> str | None:
|
||||
"""Fetch the lyrics of a song from Lyrics.ovh.
|
||||
|
||||
:param artist: The artist name to query with.
|
||||
:param title: The song title to query with.
|
||||
:return: The lyrics text, or None on a miss.
|
||||
"""
|
||||
url: str = ("https://api.lyrics.ovh/v1/"
|
||||
f"{urllib.parse.quote(artist, safe='')}/"
|
||||
f"{urllib.parse.quote(title, safe='')}")
|
||||
data: Any = self.__get_json(url)
|
||||
if isinstance(data, dict) \
|
||||
and isinstance(data.get("lyrics"), str) \
|
||||
and data["lyrics"] != "":
|
||||
return data["lyrics"]
|
||||
return None
|
||||
|
||||
def __fetch_lrclib(self, artist: str, title: str) \
|
||||
-> str | None:
|
||||
"""Fetch the lyrics of a song from LRCLIB.
|
||||
|
||||
:param artist: The artist name to query with.
|
||||
:param title: The song title to query with.
|
||||
:return: The plain lyrics text, or None on a miss.
|
||||
"""
|
||||
query: str = urllib.parse.urlencode(
|
||||
{"artist_name": artist, "track_name": title})
|
||||
url: str = f"https://lrclib.net/api/get?{query}"
|
||||
data: Any = self.__get_json(url)
|
||||
if isinstance(data, dict) \
|
||||
and isinstance(data.get("plainLyrics"), str) \
|
||||
and data["plainLyrics"] != "":
|
||||
return data["plainLyrics"]
|
||||
return None
|
||||
|
||||
def __get_json(self, url: str) -> Any:
|
||||
"""Send a GET request and return the parsed JSON body.
|
||||
|
||||
Consecutive requests are separated by a fixed delay.
|
||||
|
||||
:param url: The URL to request.
|
||||
:return: The parsed JSON body, or None on any HTTP,
|
||||
network, or decoding error.
|
||||
"""
|
||||
if self.__sent > 0:
|
||||
time.sleep(SLEEP_SECONDS)
|
||||
self.__sent += 1
|
||||
request: urllib.request.Request = urllib.request.Request(
|
||||
url, headers={"User-Agent": USER_AGENT})
|
||||
try:
|
||||
with urllib.request.urlopen(
|
||||
request, timeout=TIMEOUT) as response:
|
||||
return json.load(response)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def query_artist(session: Session, song_id: int) -> str:
|
||||
"""Find the artist name to query the APIs with.
|
||||
|
||||
:param session: The database session.
|
||||
:param song_id: The song ID.
|
||||
:return: The name of the primary-role artist with the lowest
|
||||
position.
|
||||
"""
|
||||
name: str | None = session.scalar(
|
||||
sa.select(Artist.name)
|
||||
.join(SongArtist, SongArtist.artist_id == Artist.id)
|
||||
.where(SongArtist.song_id == song_id,
|
||||
SongArtist.role == Role.PRIMARY)
|
||||
.order_by(SongArtist.position)
|
||||
.limit(1))
|
||||
assert name is not None
|
||||
return name
|
||||
|
||||
|
||||
def save_lyrics(lyrics_dir: Path, song_id: int,
|
||||
lyrics: str) -> None:
|
||||
"""Write the lyrics of a song into the cache directory.
|
||||
|
||||
The cache directory is created when missing.
|
||||
|
||||
:param lyrics_dir: The lyrics cache directory.
|
||||
:param song_id: The song ID.
|
||||
:param lyrics: The lyrics text.
|
||||
:return: None.
|
||||
:raises OSError: When the file cannot be written.
|
||||
"""
|
||||
lyrics_dir.mkdir(parents=True, exist_ok=True)
|
||||
(lyrics_dir / f"{song_id}.txt").write_text(
|
||||
lyrics, encoding="utf-8")
|
||||
|
||||
|
||||
def append_provenance(path: Path, song_id: int,
|
||||
source: str) -> None:
|
||||
"""Append a provenance row for a fetched lyrics file.
|
||||
|
||||
The CSV file is created with the header row when missing.
|
||||
|
||||
:param path: The lyrics provenance CSV file.
|
||||
:param song_id: The song ID.
|
||||
:param source: The source name of the fetched lyrics.
|
||||
:return: None.
|
||||
:raises OSError: When the file cannot be written.
|
||||
"""
|
||||
is_new: bool = not path.exists()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "a", encoding="utf-8",
|
||||
newline="") as file:
|
||||
writer: Any = csv.writer(file)
|
||||
if is_new:
|
||||
writer.writerow(PROVENANCE_FIELDS)
|
||||
writer.writerow([song_id, source, "api-fetch",
|
||||
datetime.date.today().isoformat(), ""])
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""Fetch the missing song lyrics from the public APIs.
|
||||
|
||||
:param argv: The command-line arguments, or None for
|
||||
``sys.argv``.
|
||||
:return: The exit status: 0 on success, misses included,
|
||||
non-zero on a setup error.
|
||||
"""
|
||||
started: float = time.monotonic()
|
||||
args: argparse.Namespace = parse_args(argv)
|
||||
fetcher: LyricsFetcher = LyricsFetcher()
|
||||
fetched: int = 0
|
||||
missed: int = 0
|
||||
session: Session = ds.get_db()
|
||||
try:
|
||||
song: Song
|
||||
for song in session.scalars(
|
||||
sa.select(Song).order_by(Song.id)):
|
||||
if (args.lyrics_dir / f"{song.id}.txt").exists():
|
||||
continue
|
||||
artist: str = query_artist(session, song.id)
|
||||
result: tuple[str, str] | None = fetcher.fetch(
|
||||
artist, song.title)
|
||||
if result is None and song.artist_credit != artist:
|
||||
result = fetcher.fetch(
|
||||
song.artist_credit, song.title)
|
||||
if result is None:
|
||||
missed += 1
|
||||
print(f"song {song.id} \"{song.title}\": miss",
|
||||
file=sys.stderr)
|
||||
continue
|
||||
lyrics: str
|
||||
source: str
|
||||
lyrics, source = result
|
||||
save_lyrics(args.lyrics_dir, song.id, lyrics)
|
||||
append_provenance(args.provenance_csv, song.id,
|
||||
source)
|
||||
fetched += 1
|
||||
print(f"song {song.id} \"{song.title}\": {source}",
|
||||
file=sys.stderr)
|
||||
except (OSError, sa.exc.SQLAlchemyError) as error:
|
||||
print(f"error: {error}", file=sys.stderr)
|
||||
return 1
|
||||
finally:
|
||||
session.close()
|
||||
attempted: int = fetched + missed
|
||||
elapsed: str = format_duration(time.monotonic() - started)
|
||||
print(f"Done. Fetched lyrics for {fetched}/{attempted}"
|
||||
f" songs. {elapsed} elapsed.",
|
||||
file=sys.stderr)
|
||||
return 0
|
||||
@@ -0,0 +1,653 @@
|
||||
#!/usr/bin/env python3
|
||||
# Tools for A Feminist Audit of Pop Music.
|
||||
# Copyright 2026 imacat. All rights reserved.
|
||||
# Authors:
|
||||
# imacat@mail.imacat.idv.tw (imacat), 2026/7/30
|
||||
# AI assistance: Claude Code (Anthropic)
|
||||
"""The generic batch runner for one LLM analysis step.
|
||||
|
||||
Sends every input item to the Anthropic Messages Batch API twice with
|
||||
the same system prompt, reconciles the disagreeing items with a third
|
||||
arbitration batch, and archives every artifact self-contained under
|
||||
``<runs_dir>/<phase>/<YYYYMMDD-HHMM>-<prompt-stem>/``, where the
|
||||
base directory of the run archives is given as the positional
|
||||
command-line argument.
|
||||
"""
|
||||
import argparse
|
||||
import enum
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Self
|
||||
|
||||
import anthropic
|
||||
|
||||
from ..config import get_settings
|
||||
|
||||
MODEL: str = "claude-sonnet-4-6"
|
||||
TEMPERATURE: float = 0.0
|
||||
THINKING: dict[str, str] = {"type": "disabled"}
|
||||
SCRIPT_VERSION: str = "run_llm.py 1.0.0"
|
||||
POLL_INTERVAL_SECONDS: float = 60.0
|
||||
ARBITRATION_TEMPLATE: str = (
|
||||
"<item>\n{content}\n</item>\n"
|
||||
"<run1>\n{run1}\n</run1>\n"
|
||||
"<run2>\n{run2}\n</run2>")
|
||||
|
||||
|
||||
class Source(enum.StrEnum):
|
||||
"""The source of a final record."""
|
||||
|
||||
AGREED = "agreed"
|
||||
"""The record takes the run text the two runs agreed on."""
|
||||
ARBITRATION = "arbitration"
|
||||
"""The record takes the arbitration output."""
|
||||
|
||||
|
||||
class InputFormatError(Exception):
|
||||
"""An error in the JSONL input file."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InputItem:
|
||||
"""An input item of the LLM step."""
|
||||
|
||||
id: str
|
||||
"""The item ID."""
|
||||
content: str
|
||||
"""The item content."""
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls, data: Any, path: Path,
|
||||
number: int) -> Self:
|
||||
"""Validate one parsed JSONL record as an input item.
|
||||
|
||||
:param data: The parsed JSON value of the line.
|
||||
:param path: The path of the JSONL input file, for the
|
||||
messages.
|
||||
:param number: The line number, for the messages.
|
||||
:return: The validated input item.
|
||||
:raises InputFormatError: When the record is malformed.
|
||||
"""
|
||||
if not isinstance(data, dict):
|
||||
raise InputFormatError(
|
||||
f"{path}: line {number}: not a JSON object")
|
||||
if set(data.keys()) != {"id", "content"}:
|
||||
raise InputFormatError(
|
||||
f"{path}: line {number}: keys must be exactly"
|
||||
" \"id\" and \"content\"")
|
||||
if not isinstance(data["id"], str) or data["id"] == "":
|
||||
raise InputFormatError(
|
||||
f"{path}: line {number}: \"id\" must be a"
|
||||
" non-empty string")
|
||||
if not isinstance(data["content"], str):
|
||||
raise InputFormatError(
|
||||
f"{path}: line {number}: \"content\" must be a"
|
||||
" string")
|
||||
return cls(id=data["id"], content=data["content"])
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchResult:
|
||||
"""One batch result record."""
|
||||
|
||||
id: str
|
||||
"""The item ID."""
|
||||
text: str | None = None
|
||||
"""The output text of a succeeded result."""
|
||||
stop_reason: str | None = None
|
||||
"""The stop reason of a succeeded result."""
|
||||
usage: dict[str, Any] | None = None
|
||||
"""The token usage of a succeeded result."""
|
||||
error: str | None = None
|
||||
"""The error code of a failed result."""
|
||||
|
||||
@property
|
||||
def is_failure(self) -> bool:
|
||||
"""Whether this result is a failure.
|
||||
|
||||
:return: True when the result carries an error, or False
|
||||
when it succeeded.
|
||||
"""
|
||||
return self.error is not None
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls, entry: Any) -> Self:
|
||||
"""Create the result record of a batch result entry.
|
||||
|
||||
A succeeded entry yields the text, the stop reason, and
|
||||
the usage; any other entry yields the error code.
|
||||
|
||||
:param entry: The batch result entry.
|
||||
:return: The result record.
|
||||
"""
|
||||
result: Any = entry.result
|
||||
match result.type:
|
||||
case "succeeded":
|
||||
message: Any = result.message
|
||||
text: str = "".join(
|
||||
x.text for x in message.content
|
||||
if x.type == "text")
|
||||
return cls(id=entry.custom_id, text=text,
|
||||
stop_reason=message.stop_reason,
|
||||
usage=usage_to_dict(message.usage))
|
||||
case "errored":
|
||||
return cls(id=entry.custom_id,
|
||||
error=result.error.error.type)
|
||||
case other:
|
||||
return cls(id=entry.custom_id, error=str(other))
|
||||
|
||||
def to_record(self) -> dict[str, Any]:
|
||||
"""Return this result as an archive JSONL record.
|
||||
|
||||
:return: The record, with the None fields omitted.
|
||||
"""
|
||||
return {k: v for k, v in asdict(self).items()
|
||||
if v is not None}
|
||||
|
||||
|
||||
type Results = dict[str, BatchResult]
|
||||
"""The batch result records, keyed by item ID."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchInfo:
|
||||
"""The bookkeeping of one submitted message batch."""
|
||||
|
||||
batch_id: str
|
||||
"""The batch ID."""
|
||||
submitted_at: str
|
||||
"""The submission time, in ISO 8601 format."""
|
||||
ended_at: str | None = None
|
||||
"""The end time, in ISO 8601 format, or None while the batch
|
||||
is still processing."""
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None) -> argparse.Namespace:
|
||||
"""Parse the command-line arguments.
|
||||
|
||||
:param argv: The command-line arguments, or None for ``sys.argv``.
|
||||
:return: The parsed arguments.
|
||||
"""
|
||||
parser: argparse.ArgumentParser = argparse.ArgumentParser(
|
||||
description="Run one LLM step: 2 runs + 1 arbitration.")
|
||||
parser.add_argument(
|
||||
"runs_dir", type=Path,
|
||||
help="the base directory of the run archives")
|
||||
parser.add_argument(
|
||||
"--prompt", required=True, type=Path,
|
||||
help="the prompt definition file, used as the system prompt")
|
||||
parser.add_argument(
|
||||
"--arbitration-prompt", required=True, type=Path,
|
||||
help="the arbitration prompt definition file")
|
||||
parser.add_argument(
|
||||
"--input", required=True, type=Path,
|
||||
help="the JSONL input file with \"id\" and \"content\"")
|
||||
parser.add_argument(
|
||||
"--phase", required=True,
|
||||
help="the phase name for the archive directory")
|
||||
parser.add_argument(
|
||||
"--max-tokens", type=int, default=2048,
|
||||
help="the maximum output tokens per request (default 2048)")
|
||||
parser.add_argument(
|
||||
"--dry-run", action="store_true",
|
||||
help="validate and archive without calling the API")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def load_items(path: Path) -> list[InputItem]:
|
||||
"""Load and validate the JSONL input items.
|
||||
|
||||
:param path: The path of the JSONL input file.
|
||||
:return: The input items, in file order.
|
||||
:raises InputFormatError: When a line is malformed, an ID is
|
||||
duplicated, or the file contains no item.
|
||||
:raises OSError: When the file cannot be read.
|
||||
"""
|
||||
items: list[InputItem] = []
|
||||
seen: set[str] = set()
|
||||
with open(path, encoding="utf-8") as file:
|
||||
for number, line in enumerate(file, start=1):
|
||||
if line.strip() == "":
|
||||
continue
|
||||
try:
|
||||
data: Any = json.loads(line)
|
||||
except json.JSONDecodeError as error:
|
||||
raise InputFormatError(
|
||||
f"{path}: line {number}: malformed JSON: {error}")
|
||||
item: InputItem = InputItem.get_instance(
|
||||
data, path, number)
|
||||
if item.id in seen:
|
||||
raise InputFormatError(
|
||||
f"{path}: line {number}: duplicated ID"
|
||||
f" \"{item.id}\"")
|
||||
seen.add(item.id)
|
||||
items.append(item)
|
||||
if len(items) == 0:
|
||||
raise InputFormatError(f"{path}: no input items")
|
||||
return items
|
||||
|
||||
|
||||
def build_request(item: InputItem, system_prompt: str,
|
||||
max_tokens: int) -> dict[str, Any]:
|
||||
"""Build one Message Batches request for an input item.
|
||||
|
||||
:param item: The input item.
|
||||
:param system_prompt: The system prompt text.
|
||||
:param max_tokens: The maximum output tokens.
|
||||
:return: The batch request with "custom_id" and "params".
|
||||
"""
|
||||
return {
|
||||
"custom_id": item.id,
|
||||
"params": {
|
||||
"model": MODEL,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": TEMPERATURE,
|
||||
"thinking": THINKING,
|
||||
"system": system_prompt,
|
||||
"messages": [
|
||||
{"role": "user", "content": item.content},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_arbitration_content(content: str, run1_text: str,
|
||||
run2_text: str) -> str:
|
||||
"""Build the arbitration user message for one item.
|
||||
|
||||
:param content: The original item content.
|
||||
:param run1_text: The run-1 output text.
|
||||
:param run2_text: The run-2 output text.
|
||||
:return: The user message text.
|
||||
"""
|
||||
return ARBITRATION_TEMPLATE.format(
|
||||
content=content, run1=run1_text, run2=run2_text)
|
||||
|
||||
|
||||
def submit_batch(client: anthropic.Anthropic,
|
||||
requests: list[dict[str, Any]]) -> str:
|
||||
"""Submit one message batch.
|
||||
|
||||
:param client: The Anthropic client.
|
||||
:param requests: The batch requests.
|
||||
:return: The batch ID.
|
||||
"""
|
||||
return client.messages.batches.create(requests=requests).id
|
||||
|
||||
|
||||
def poll_batches(client: anthropic.Anthropic,
|
||||
batch_ids: list[str]) -> dict[str, Any]:
|
||||
"""Poll the batches until every one of them has ended.
|
||||
|
||||
Progress is printed to the standard error every poll.
|
||||
|
||||
:param client: The Anthropic client.
|
||||
:param batch_ids: The batch IDs to poll.
|
||||
:return: The final batch object of each batch, keyed by batch ID.
|
||||
"""
|
||||
while True:
|
||||
batches: dict[str, Any] = {
|
||||
x: client.messages.batches.retrieve(x) for x in batch_ids}
|
||||
pending: list[str] = [
|
||||
x for x in batch_ids
|
||||
if batches[x].processing_status != "ended"]
|
||||
for batch_id in batch_ids:
|
||||
status: str = batches[batch_id].processing_status
|
||||
print(f"batch {batch_id}: {status}", file=sys.stderr)
|
||||
if len(pending) == 0:
|
||||
return batches
|
||||
time.sleep(POLL_INTERVAL_SECONDS)
|
||||
|
||||
|
||||
def usage_to_dict(usage: Any) -> dict[str, Any]:
|
||||
"""Convert a usage object to a plain dictionary.
|
||||
|
||||
:param usage: The usage object of a message.
|
||||
:return: The usage as a dictionary, without null entries.
|
||||
"""
|
||||
return {k: v for k, v in usage.model_dump().items()
|
||||
if v is not None}
|
||||
|
||||
|
||||
def collect_results(client: anthropic.Anthropic,
|
||||
batch_id: str) -> Results:
|
||||
"""Collect the results of an ended batch.
|
||||
|
||||
:param client: The Anthropic client.
|
||||
:param batch_id: The batch ID.
|
||||
:return: The result records, keyed by custom ID.
|
||||
"""
|
||||
results: Results = {}
|
||||
for entry in client.messages.batches.results(batch_id):
|
||||
results[entry.custom_id] = BatchResult.get_instance(entry)
|
||||
return results
|
||||
|
||||
|
||||
def find_failures(item_ids: list[str],
|
||||
results: Results) -> list[str]:
|
||||
"""Find the item IDs that failed in a result set.
|
||||
|
||||
An item failed when it is missing from the results or when its
|
||||
record is a failure.
|
||||
|
||||
:param item_ids: The item IDs to check, in order.
|
||||
:param results: The result records, keyed by item ID.
|
||||
:return: The failed item IDs, in the given order.
|
||||
"""
|
||||
return [x for x in item_ids
|
||||
if x not in results or results[x].is_failure]
|
||||
|
||||
|
||||
def split_by_agreement(
|
||||
items: list[InputItem], run1: Results, run2: Results,
|
||||
) -> tuple[list[str], list[str]]:
|
||||
"""Split the item IDs into agreed and disagreeing ones.
|
||||
|
||||
Two outputs agree when their texts are identical after strip().
|
||||
|
||||
:param items: The input items.
|
||||
:param run1: The run-1 results, keyed by item ID.
|
||||
:param run2: The run-2 results, keyed by item ID.
|
||||
:return: A tuple of the agreed item IDs and the disagreeing item
|
||||
IDs, both in input order.
|
||||
"""
|
||||
agreed: list[str] = []
|
||||
disagreed: list[str] = []
|
||||
for item in items:
|
||||
text1: str | None = run1[item.id].text
|
||||
text2: str | None = run2[item.id].text
|
||||
assert text1 is not None and text2 is not None
|
||||
if text1.strip() == text2.strip():
|
||||
agreed.append(item.id)
|
||||
else:
|
||||
disagreed.append(item.id)
|
||||
return agreed, disagreed
|
||||
|
||||
|
||||
def build_final_records(
|
||||
items: list[InputItem], run1: Results, arbitration: Results,
|
||||
) -> list[dict[str, str]]:
|
||||
"""Assemble the final records, one per item, in input order.
|
||||
|
||||
An arbitrated item takes the arbitration output; an agreed item
|
||||
takes the agreed (stripped) run text.
|
||||
|
||||
:param items: The input items.
|
||||
:param run1: The run-1 results, keyed by item ID.
|
||||
:param arbitration: The arbitration results, keyed by item ID.
|
||||
:return: The final records with "id", "text", and "source".
|
||||
"""
|
||||
records: list[dict[str, str]] = []
|
||||
for item in items:
|
||||
text: str | None
|
||||
if item.id in arbitration:
|
||||
text = arbitration[item.id].text
|
||||
assert text is not None
|
||||
records.append({"id": item.id, "text": text,
|
||||
"source": Source.ARBITRATION})
|
||||
else:
|
||||
text = run1[item.id].text
|
||||
assert text is not None
|
||||
records.append({"id": item.id, "text": text.strip(),
|
||||
"source": Source.AGREED})
|
||||
return records
|
||||
|
||||
|
||||
def create_archive_dir(runs_root: Path, phase: str, prompt_path: Path,
|
||||
now: datetime) -> Path:
|
||||
"""Create the archive directory for this execution.
|
||||
|
||||
:param runs_root: The root directory of the run archives.
|
||||
:param phase: The phase name.
|
||||
:param prompt_path: The path of the prompt definition file.
|
||||
:param now: The local timestamp of this execution.
|
||||
:return: The created archive directory.
|
||||
:raises FileExistsError: When the directory already exists.
|
||||
"""
|
||||
stem: str = prompt_path.stem
|
||||
name: str = f"{now.strftime('%Y%m%d-%H%M')}-{stem}"
|
||||
directory: Path = runs_root / phase / name
|
||||
if directory.exists():
|
||||
raise FileExistsError(
|
||||
f"archive directory {directory} already exists")
|
||||
directory.mkdir(parents=True)
|
||||
return directory
|
||||
|
||||
|
||||
def write_jsonl(path: Path, records: list[dict[str, Any]]) -> None:
|
||||
"""Write records to a file as JSON Lines.
|
||||
|
||||
:param path: The path of the file to write.
|
||||
:param records: The records, one per line.
|
||||
:return: None.
|
||||
"""
|
||||
with open(path, "w", encoding="utf-8") as file:
|
||||
for record in records:
|
||||
file.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict[str, Any]) -> None:
|
||||
"""Write data to a file as pretty-printed JSON.
|
||||
|
||||
:param path: The path of the file to write.
|
||||
:param data: The data to write.
|
||||
:return: None.
|
||||
"""
|
||||
path.write_text(
|
||||
json.dumps(data, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8")
|
||||
|
||||
|
||||
def write_meta(path: Path, meta: dict[str, Any]) -> None:
|
||||
"""Write the metadata to the ``meta.json`` file.
|
||||
|
||||
The ``BatchInfo`` values under ``batches`` are written as
|
||||
plain JSON objects.
|
||||
|
||||
:param path: The path of the ``meta.json`` file.
|
||||
:param meta: The metadata to write.
|
||||
:return: None.
|
||||
"""
|
||||
write_json(path, {
|
||||
**meta,
|
||||
"batches": {k: asdict(v)
|
||||
for k, v in meta["batches"].items()}})
|
||||
|
||||
|
||||
def sha256_of(path: Path) -> str:
|
||||
"""Calculate the SHA-256 digest of a file.
|
||||
|
||||
:param path: The path of the file.
|
||||
:return: The hexadecimal SHA-256 digest.
|
||||
"""
|
||||
with open(path, "rb") as file:
|
||||
return hashlib.file_digest(file, "sha256").hexdigest()
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
"""Return the current local time in ISO 8601 format.
|
||||
|
||||
:return: The current local time with the timezone offset.
|
||||
"""
|
||||
return datetime.now().astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def execute_runs(
|
||||
client: anthropic.Anthropic, items: list[InputItem],
|
||||
system_prompt: str, max_tokens: int, meta: dict[str, Any],
|
||||
) -> tuple[Results, Results]:
|
||||
"""Submit the two identical runs and await their results.
|
||||
|
||||
The batch IDs and timestamps are recorded into the metadata as an
|
||||
observable side effect.
|
||||
|
||||
:param client: The Anthropic client.
|
||||
:param items: The input items.
|
||||
:param system_prompt: The system prompt text.
|
||||
:param max_tokens: The maximum output tokens per request.
|
||||
:param meta: The metadata to record the batch bookkeeping into.
|
||||
:return: The results of run 1 and run 2, keyed by item ID.
|
||||
"""
|
||||
requests: list[dict[str, Any]] = [
|
||||
build_request(x, system_prompt, max_tokens) for x in items]
|
||||
infos: dict[str, BatchInfo] = {}
|
||||
for run_name in ("run1", "run2"):
|
||||
info: BatchInfo = BatchInfo(
|
||||
batch_id=submit_batch(client, requests),
|
||||
submitted_at=now_iso())
|
||||
infos[run_name] = info
|
||||
meta["batches"][run_name] = info
|
||||
print(f"{run_name}: submitted batch {info.batch_id}",
|
||||
file=sys.stderr)
|
||||
batches: dict[str, Any] = poll_batches(
|
||||
client, [x.batch_id for x in infos.values()])
|
||||
for info in infos.values():
|
||||
info.ended_at = batches[info.batch_id].ended_at.isoformat()
|
||||
return (collect_results(client, infos["run1"].batch_id),
|
||||
collect_results(client, infos["run2"].batch_id))
|
||||
|
||||
|
||||
def execute_arbitration(
|
||||
client: anthropic.Anthropic, items: list[InputItem],
|
||||
disagreed: list[str], run1: Results, run2: Results,
|
||||
system_prompt: str, max_tokens: int, meta: dict[str, Any],
|
||||
) -> Results:
|
||||
"""Submit the arbitration batch and await its results.
|
||||
|
||||
The batch ID and timestamps are recorded into the metadata as an
|
||||
observable side effect.
|
||||
|
||||
:param client: The Anthropic client.
|
||||
:param items: The input items.
|
||||
:param disagreed: The disagreeing item IDs.
|
||||
:param run1: The run-1 results, keyed by item ID.
|
||||
:param run2: The run-2 results, keyed by item ID.
|
||||
:param system_prompt: The arbitration system prompt text.
|
||||
:param max_tokens: The maximum output tokens per request.
|
||||
:param meta: The metadata to record the batch bookkeeping into.
|
||||
:return: The arbitration results, keyed by item ID.
|
||||
"""
|
||||
content_by_id: dict[str, str] = {
|
||||
x.id: x.content for x in items}
|
||||
requests: list[dict[str, Any]] = []
|
||||
for item_id in disagreed:
|
||||
text1: str | None = run1[item_id].text
|
||||
text2: str | None = run2[item_id].text
|
||||
assert text1 is not None and text2 is not None
|
||||
requests.append(build_request(
|
||||
InputItem(id=item_id,
|
||||
content=build_arbitration_content(
|
||||
content_by_id[item_id], text1, text2)),
|
||||
system_prompt, max_tokens))
|
||||
info: BatchInfo = BatchInfo(
|
||||
batch_id=submit_batch(client, requests),
|
||||
submitted_at=now_iso())
|
||||
meta["batches"]["arbitration"] = info
|
||||
print(f"arbitration: submitted batch {info.batch_id}",
|
||||
file=sys.stderr)
|
||||
batches: dict[str, Any] = poll_batches(client, [info.batch_id])
|
||||
info.ended_at = batches[info.batch_id].ended_at.isoformat()
|
||||
return collect_results(client, info.batch_id)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""Run one LLM step end-to-end.
|
||||
|
||||
:param argv: The command-line arguments, or None for ``sys.argv``.
|
||||
:return: The exit status: 0 on success, non-zero on failure.
|
||||
"""
|
||||
args: argparse.Namespace = parse_args(argv)
|
||||
try:
|
||||
items: list[InputItem] = load_items(args.input)
|
||||
prompt_text: str = args.prompt.read_text(encoding="utf-8")
|
||||
arbitration_text: str = args.arbitration_prompt.read_text(
|
||||
encoding="utf-8")
|
||||
except (OSError, InputFormatError) as error:
|
||||
print(f"error: {error}", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
run_dir: Path = create_archive_dir(
|
||||
args.runs_dir, args.phase, args.prompt,
|
||||
datetime.now())
|
||||
except FileExistsError as error:
|
||||
print(f"error: {error}", file=sys.stderr)
|
||||
return 1
|
||||
meta_path: Path = run_dir / "meta.json"
|
||||
(run_dir / "prompt.md").write_bytes(args.prompt.read_bytes())
|
||||
(run_dir / "arbitration_prompt.md").write_bytes(
|
||||
args.arbitration_prompt.read_bytes())
|
||||
meta: dict[str, Any] = {
|
||||
"model": MODEL,
|
||||
"temperature": TEMPERATURE,
|
||||
"max_tokens": args.max_tokens,
|
||||
"thinking": THINKING,
|
||||
"prompt_path": str(args.prompt),
|
||||
"prompt_sha256": sha256_of(args.prompt),
|
||||
"arbitration_prompt_path": str(args.arbitration_prompt),
|
||||
"arbitration_prompt_sha256": sha256_of(
|
||||
args.arbitration_prompt),
|
||||
"batches": {},
|
||||
"item_count": len(items),
|
||||
"agreed_count": None,
|
||||
"agreement_rate": None,
|
||||
"dry_run": args.dry_run,
|
||||
"script_version": SCRIPT_VERSION,
|
||||
}
|
||||
if args.dry_run:
|
||||
write_meta(meta_path, meta)
|
||||
print(json.dumps(
|
||||
build_request(items[0], prompt_text, args.max_tokens),
|
||||
ensure_ascii=False, indent=2))
|
||||
print(f"dry run: archive created at {run_dir}",
|
||||
file=sys.stderr)
|
||||
return 0
|
||||
client: anthropic.Anthropic = anthropic.Anthropic(
|
||||
api_key=get_settings().ANTHROPIC_API_KEY)
|
||||
run1: Results
|
||||
run2: Results
|
||||
run1, run2 = execute_runs(
|
||||
client, items, prompt_text, args.max_tokens, meta)
|
||||
item_ids: list[str] = [x.id for x in items]
|
||||
write_jsonl(run_dir / "run1.jsonl",
|
||||
[run1[x].to_record() for x in item_ids if x in run1])
|
||||
write_jsonl(run_dir / "run2.jsonl",
|
||||
[run2[x].to_record() for x in item_ids if x in run2])
|
||||
failed: set[str] = (set(find_failures(item_ids, run1))
|
||||
| set(find_failures(item_ids, run2)))
|
||||
if len(failed) > 0:
|
||||
write_meta(meta_path, meta)
|
||||
names: str = ", ".join(x for x in item_ids if x in failed)
|
||||
print(f"error: failed items: {names}", file=sys.stderr)
|
||||
return 1
|
||||
agreed: list[str]
|
||||
disagreed: list[str]
|
||||
agreed, disagreed = split_by_agreement(items, run1, run2)
|
||||
meta["agreed_count"] = len(agreed)
|
||||
meta["agreement_rate"] = len(agreed) / len(items)
|
||||
arbitration: Results = {}
|
||||
if len(disagreed) > 0:
|
||||
arbitration = execute_arbitration(
|
||||
client, items, disagreed, run1, run2, arbitration_text,
|
||||
args.max_tokens, meta)
|
||||
write_jsonl(run_dir / "arbitration.jsonl",
|
||||
[arbitration[x].to_record() for x in disagreed
|
||||
if x in arbitration])
|
||||
arb_failed: list[str] = find_failures(disagreed, arbitration)
|
||||
if len(arb_failed) > 0:
|
||||
write_meta(meta_path, meta)
|
||||
names = ", ".join(arb_failed)
|
||||
print(f"error: failed arbitration items: {names}",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
write_jsonl(run_dir / "final.jsonl",
|
||||
build_final_records(items, run1, arbitration))
|
||||
write_meta(meta_path, meta)
|
||||
print(f"done: {len(items)} items, {len(agreed)} agreed,"
|
||||
f" {len(disagreed)} arbitrated; archived to {run_dir}",
|
||||
file=sys.stderr)
|
||||
return 0
|
||||
Reference in New Issue
Block a user