imacat/pop-fem-audit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Export human-readable songs and artists reports to the derived data layer

Browse Source
This commit is contained in:
imacat
2026-08-04 15:15:49 +08:00
parent ae0e9d0a08
commit 1f95f0231d
9 changed files with 1711 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/src/pop_fem_audit_tools/build_db.py
+168 -8
View File
@@ -5,13 +5,13 @@
"""The builder of the SQLite working store.
Rebuilds the working store from scratch out of the committed
inputs: the year-end chart CSV, given as the positional
command-line argument, and the optional capture inputs, each
given as an option: the lyrics cache directory, the Wikidata
artist snapshot CSV, and the manual artist overrides CSV. An
omitted option leaves its capture layer unloaded; a given
option whose path does not exist fails the build. Missing
tables
inputs: the year-end chart CSV and the output directory for the
review CSV files, given as the two positional command-line
arguments, and the optional capture inputs, each given as an
option: the lyrics cache directory, the Wikidata artist snapshot
CSV, and the manual artist overrides CSV. An omitted option
leaves its capture layer unloaded; a given option whose path does
not exist fails the build. Missing tables
are created on a fresh store; existing tables are never altered,
as the schema lifecycle belongs to the migrations. Every rebuild
deletes all the rows, loads the data, and validates it in one
@@ -38,6 +38,12 @@ collapse onto a single artist row. The stored artist name is the
first-seen spelling, except for the names listed in
``CANONICAL_ARTIST_NAMES``, which always store the canonical
spelling regardless of which variant is seen first.
On a successful build, two review CSV files, ``songs.csv`` and
``artists.csv``, are (re)written under the given output directory,
mirroring the stored songs and artists without their IDs; see
`CSVExporter`. A failed build leaves any existing review CSV
files untouched, matching the store rollback.
"""
import argparse
import csv
@@ -46,7 +52,7 @@ import sys
from collections.abc import Iterable, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Self
from typing import Any, Self
import sqlalchemy as sa
from sqlalchemy.orm import Session
@@ -156,6 +162,9 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
parser.add_argument(
"chart_csv", type=Path,
help="the year-end chart CSV file")
parser.add_argument(
"derived_dir", type=Path,
help="the output directory for the review CSV files")
parser.add_argument(
"--lyrics-dir", type=Path, default=None,
help="the lyrics cache directory to load")
@@ -527,6 +536,156 @@ def reset_store(session: Session) -> None:
session.execute(sa.delete(model))
class CSVExporter:
"""Writes the review CSV files mirroring the working store."""
__SONGS_HEADER: tuple[str, ...] = ("Title", "Artists", "Positions")
"""The header row of ``songs.csv``, for human readers."""
__ARTISTS_HEADER: tuple[str, ...] = (
"Name", "Wikidata QID", "Gender", "Type", "Genre", "Country",
"Songs")
"""The header row of ``artists.csv``, for human readers."""
def __init__(self, session: Session, derived_dir: Path) -> None:
"""Initialize the exporter.
:param session: The database session with the loaded data
flushed.
:param derived_dir: The output directory for the review CSV
files.
"""
self.__session: Session = session
self.__derived_dir: Path = derived_dir
def write(self) -> None:
"""Write the review CSV files mirroring the loaded data.
Fully overwrites ``songs.csv`` and ``artists.csv`` under the
output directory, creating it when missing, with normal
minimal CSV quoting. Neither file carries a song or an
artist ID; a multi-valued field is a plain joined string,
itself CSV-quoted as a whole only when its content requires
it: "/" joins the chart appearances of one song, and "|"
joins the distinct songs credited to one artist.
:return: None.
:raises OSError: When a CSV file cannot be written.
"""
self.__derived_dir.mkdir(parents=True, exist_ok=True)
self.__write_csv(
self.__derived_dir / "songs.csv", self.__SONGS_HEADER,
self.__songs_rows())
self.__write_csv(
self.__derived_dir / "artists.csv", self.__ARTISTS_HEADER,
self.__artists_rows())
@staticmethod
def __write_csv(path: Path, header: Sequence[str],
rows: Iterable[Sequence[str]]) -> None:
"""Write a CSV file with LF line endings, fully overwritten.
:param path: The output CSV file.
:param header: The header row.
:param rows: The data rows, in the given order.
:return: None.
:raises OSError: When the file cannot be written.
"""
with open(path, "w", encoding="utf-8", newline="") as file:
writer: Any = csv.writer(file)
writer.writerow(header)
writer.writerows(rows)
@staticmethod
def __sorted_chart_entries(song: Song) -> list[ChartEntry]:
"""Sort the chart entries of a song by year then rank.
:param song: The song with its chart entries loaded.
:return: The chart entries, ordered by year then rank.
"""
return sorted(
song.chart_entries, key=lambda x: (x.year, x.rank))
@classmethod
def __song_positions(cls, song: Song) -> str:
"""Format the chart positions of a song for the songs.csv
value.
:param song: The song with its chart entries loaded.
:return: The "YEAR#RANK" tokens, ordered by year then rank,
joined by "/".
"""
entries: list[ChartEntry] = cls.__sorted_chart_entries(song)
return "/".join(f"{x.year}#{x.rank}" for x in entries)
@classmethod
def __formatted_song_positions(cls, song: Song) -> str:
"""Format the chart positions of a song for the artists.csv
value.
:param song: The song with its chart entries loaded.
:return: The "YEAR#RANK" tokens, ordered by year then rank,
joined by "/".
"""
entries: list[ChartEntry] = cls.__sorted_chart_entries(song)
return "/".join(f"{x.year}#{x.rank}" for x in entries)
def __songs_rows(self) -> list[list[str]]:
"""Build the sorted data rows of ``songs.csv``.
:return: The rows, sorted by the case-folded title, then
the case-folded artist credit.
"""
songs: list[Song] = sorted(
self.__session.scalars(sa.select(Song)),
key=lambda x: (x.title.casefold(),
x.artist_credit.casefold()))
rows: list[list[str]] = []
song: Song
for song in songs:
row: list[str] = [
song.title, song.artist_credit,
self.__song_positions(song)]
rows.append(row)
return rows
@classmethod
def __artist_songs(cls, artist: Artist) -> str:
"""Format the credited songs for the artists.csv value.
:param artist: The artist with its song credits loaded.
:return: The credited songs, each formatted as
"TITLE (YEAR#RANK[/YEAR#RANK...])" with its chart
appearances, sorted alphabetically case-folded by
title, joined by "|".
"""
songs: list[Song] = sorted(
(x.song for x in artist.song_artists),
key=lambda x: x.title.casefold())
entries: list[str] = [
f"{song.title} ({cls.__formatted_song_positions(song)})"
for song in songs]
return "|".join(entries)
def __artists_rows(self) -> list[list[str]]:
"""Build the sorted data rows of ``artists.csv``.
:return: The rows, sorted by the case-folded name.
"""
artists: list[Artist] = sorted(
self.__session.scalars(sa.select(Artist)),
key=lambda x: x.name.casefold())
rows: list[list[str]] = []
artist: Artist
for artist in artists:
row: list[str] = [
artist.name, artist.wikidata_qid or "",
artist.gender or "", artist.type or "",
artist.genre or "", artist.country or "",
self.__artist_songs(artist)]
rows.append(row)
return rows
def main(argv: list[str] | None = None) -> int:
"""Rebuild the SQLite working store from the inputs.
@@ -562,6 +721,7 @@ def main(argv: list[str] | None = None) -> int:
print(f"error: {violation}", file=sys.stderr)
return 1
counts = StoreCounts.get_instance(session)
CSVExporter(session, args.derived_dir).write()
session.commit()
except (OSError, BuildError) as error:
session.rollback()
tools/tests/test_build_db.py
+162 -1
View File
@@ -3,6 +3,7 @@
# Authors:
# imacat@mail.imacat.idv.tw (imacat), 2026/7/31
"""Unit tests for the working store builder module."""
import csv
import io
import tempfile
import unittest
@@ -253,6 +254,7 @@ class TestBuildDB(unittest.TestCase):
self.addCleanup(tmp.cleanup)
self.__dir: Path = Path(tmp.name)
self.__chart: Path = self.__dir / "chart.csv"
self.__derived: Path = self.__dir / "derived"
self.__lyrics: Path = self.__dir / "lyrics"
self.__wikidata: Path = \
self.__dir / "artists_wikidata.csv"
@@ -290,9 +292,41 @@ class TestBuildDB(unittest.TestCase):
stderr: io.StringIO = io.StringIO()
with redirect_stderr(stderr):
status: int = build_db.main(
[str(self.__chart), *options])
[str(self.__chart), str(self.__derived), *options])
return status, stderr.getvalue()
def __read_csv_rows(self, name: str) -> list[list[str]]:
"""Read the data rows of a derived CSV file.
:param name: The CSV file name under the derived directory.
:return: The data rows, the header row excluded.
"""
with open(self.__derived / name, encoding="utf-8",
newline="") as file:
rows: list[list[str]] = list(csv.reader(file))
return rows[1:]
def __read_csv_header(self, name: str) -> list[str]:
"""Read the header row of a derived CSV file.
:param name: The CSV file name under the derived directory.
:return: The header row.
"""
with open(self.__derived / name, encoding="utf-8",
newline="") as file:
return next(csv.reader(file))
@staticmethod
def __split_joined_field(value: str,
separator: str = "|") -> list[str]:
"""Split a joined field into its entries.
:param value: The joined field value.
:param separator: The join separator.
:return: The entries, in the given order.
"""
return value.split(separator)
def __session(self) -> Session:
"""Open a database session closed on test cleanup.
@@ -594,3 +628,130 @@ class TestBuildDB(unittest.TestCase):
self.assertIn(str(self.__overrides), stderr)
session: Session = self.__session()
self.assertEqual(list(session.scalars(sa.select(Song))), [])
REVIEW_CHART_CSV: str = (
"year,rank,title,artist\n"
"2016,1,banana,Artist B\n"
"2016,2,\"Me, Myself & I\",Billie\n"
"2017,1,\"Me, Myself & I\",Billie\n"
"2017,2,Apple,Artist A\n")
"""The chart CSV fixture exercising the review CSV ordering,
positions, and the outer-field comma quoting."""
def test_derived_dir_created_when_missing(self) -> None:
"""Test that the derived directory is created when
missing."""
self.assertFalse(self.__derived.exists())
self.assertEqual(self.__run_build()[0], 0)
self.assertTrue(self.__derived.is_dir())
self.assertTrue((self.__derived / "songs.csv").is_file())
self.assertTrue((self.__derived / "artists.csv").is_file())
def test_songs_csv_written(self) -> None:
"""Test the songs.csv header, the rows, the title
ordering, and the "/"-joined "positions" value."""
self.__write_chart(self.REVIEW_CHART_CSV)
self.assertEqual(self.__run_build()[0], 0)
self.assertEqual(
self.__read_csv_header("songs.csv"),
["Title", "Artists", "Positions"])
rows: list[list[str]] = self.__read_csv_rows("songs.csv")
self.assertEqual(
[row[:2] for row in rows],
[["Apple", "Artist A"],
["banana", "Artist B"],
["Me, Myself & I", "Billie"]])
self.assertEqual(
[self.__split_joined_field(row[2], "/") for row in rows],
[["2017#2"], ["2016#1"], ["2016#2", "2017#1"]])
def test_songs_csv_uses_minimal_quoting(self) -> None:
"""Test that songs.csv uses normal minimal CSV quoting: a
numeric-looking title is written unquoted, like every other
field not otherwise requiring quoting."""
self.__write_chart(
"year,rank,title,artist\n"
"2016,1,679,Artist A\n"
"2016,2,filler,Filler Artist\n"
"2017,1,filler2,Filler Artist Two\n"
"2017,2,filler3,Filler Artist Three\n")
self.assertEqual(self.__run_build()[0], 0)
content: str = (self.__derived / "songs.csv").read_text(
encoding="utf-8")
self.assertIn("679,Artist A,2016#1", content)
def test_artists_csv_written(self) -> None:
"""Test the artists.csv header, the rows, the name
ordering, and the "|"-joined "TITLE (YEAR#RANK...)" song
encoding."""
self.__write_chart(self.REVIEW_CHART_CSV)
self.assertEqual(self.__run_build()[0], 0)
self.assertEqual(
self.__read_csv_header("artists.csv"),
["Name", "Wikidata QID", "Gender", "Type", "Genre",
"Country", "Songs"])
rows: list[list[str]] = self.__read_csv_rows("artists.csv")
self.assertEqual(
[row[0] for row in rows],
["Artist A", "Artist B", "Billie"])
self.assertEqual(
[self.__split_joined_field(row[6]) for row in rows],
[["Apple (2017#2)"],
["banana (2016#1)"],
["Me, Myself & I (2016#2/2017#1)"]])
def test_artists_csv_songs_field_quoted_for_comma_title(
self) -> None:
"""Test that the outer "Songs" field is CSV-quoted as a
whole when the joined value contains a comma, from a
comma-bearing credited title."""
self.__write_chart(self.REVIEW_CHART_CSV)
self.assertEqual(self.__run_build()[0], 0)
content: str = (self.__derived / "artists.csv").read_text(
encoding="utf-8")
self.assertIn(
'"Me, Myself & I (2016#2/2017#1)"', content)
self.assertIn("Apple (2017#2)\n", content)
def test_derived_csvs_have_no_ids(self) -> None:
"""Test that neither derived CSV file exposes a song or an
artist ID column."""
self.assertEqual(self.__run_build()[0], 0)
self.assertNotIn(
"id", self.__read_csv_header("songs.csv"))
self.assertNotIn(
"id", self.__read_csv_header("artists.csv"))
def test_derived_csvs_have_crlf_line_endings(self) -> None:
"""Test that the derived CSV files use CRLF line
endings."""
self.assertEqual(self.__run_build()[0], 0)
raw: bytes = (self.__derived / "songs.csv").read_bytes()
self.assertGreater(raw.count(b"\r\n"), 0)
self.assertEqual(raw.count(b"\r"), raw.count(b"\r\n"))
self.assertEqual(raw.count(b"\n"), raw.count(b"\r\n"))
def test_failed_build_leaves_derived_csvs_untouched(self) -> None:
"""Test that a failed build does not touch the existing
derived CSV files."""
self.assertEqual(self.__run_build()[0], 0)
songs_before: str = (self.__derived / "songs.csv").read_text(
encoding="utf-8")
artists_before: str = \
(self.__derived / "artists.csv").read_text(
encoding="utf-8")
self.__write_chart(
"year,rank,title,artist\n"
"2016,1,Hello,Adele\n")
status: int
stderr: str
status, stderr = self.__run_build()
self.assertNotEqual(status, 0)
self.assertEqual(
(self.__derived / "songs.csv").read_text(
encoding="utf-8"),
songs_before)
self.assertEqual(
(self.__derived / "artists.csv").read_text(
encoding="utf-8"),
artists_before)