Add the data layer with the build-db, fetch-lyrics, and fetch-artists subcommands
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5
@@ -0,0 +1,294 @@
|
||||
# Tools for A Feminist Audit of Pop Music.
|
||||
# Copyright 2026 imacat. All rights reserved.
|
||||
# Authors:
|
||||
# imacat@mail.imacat.idv.tw (imacat), 2026/7/31
|
||||
"""Unit tests for the working store builder module."""
|
||||
import io
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import redirect_stderr
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest import mock
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from pop_fem_audit_tools import build_db, config
|
||||
from pop_fem_audit_tools.database import DataSource
|
||||
from pop_fem_audit_tools.models import (
|
||||
Artist,
|
||||
ChartEntry,
|
||||
Song,
|
||||
)
|
||||
|
||||
|
||||
class TestParseArtistCredit(unittest.TestCase):
|
||||
"""Test cases for the artist credit parser."""
|
||||
|
||||
def test_plain_solo(self) -> None:
|
||||
"""Test a plain solo artist credit."""
|
||||
self.assertEqual(build_db.parse_artist_credit("Adele"),
|
||||
[("Adele", "primary")])
|
||||
|
||||
def test_featuring_with_and(self) -> None:
|
||||
"""Test a featuring credit with an "and" delimiter."""
|
||||
self.assertEqual(
|
||||
build_db.parse_artist_credit(
|
||||
"Drake featuring Wizkid and Kyla"),
|
||||
[("Drake", "primary"),
|
||||
("Wizkid", "featured"),
|
||||
("Kyla", "featured")])
|
||||
|
||||
def test_comma_and_ampersand(self) -> None:
|
||||
"""Test a credit with comma and ampersand delimiters."""
|
||||
self.assertEqual(
|
||||
build_db.parse_artist_credit(
|
||||
"Lady Gaga, Bradley Cooper & BloodPop"),
|
||||
[("Lady Gaga", "primary"),
|
||||
("Bradley Cooper", "primary"),
|
||||
("BloodPop", "primary")])
|
||||
|
||||
def test_x_delimiter(self) -> None:
|
||||
"""Test the "x" delimiter."""
|
||||
self.assertEqual(
|
||||
build_db.parse_artist_credit("KAROL G x Nicki Minaj"),
|
||||
[("KAROL G", "primary"),
|
||||
("Nicki Minaj", "primary")])
|
||||
|
||||
def test_plus_delimiter(self) -> None:
|
||||
"""Test the "+" delimiter."""
|
||||
self.assertEqual(
|
||||
build_db.parse_artist_credit("Marshmello + Halsey"),
|
||||
[("Marshmello", "primary"),
|
||||
("Halsey", "primary")])
|
||||
|
||||
def test_with_delimiter(self) -> None:
|
||||
"""Test the "with" delimiter."""
|
||||
self.assertEqual(
|
||||
build_db.parse_artist_credit(
|
||||
"Kane Brown with Lauren Alaina"),
|
||||
[("Kane Brown", "primary"),
|
||||
("Lauren Alaina", "primary")])
|
||||
|
||||
def test_feat_abbreviation(self) -> None:
|
||||
"""Test that "Feat." splits the featured side."""
|
||||
self.assertEqual(
|
||||
build_db.parse_artist_credit(
|
||||
"Ariana Grande Feat. Doja Cat"
|
||||
" & Megan Thee Stallion"),
|
||||
[("Ariana Grande", "primary"),
|
||||
("Doja Cat", "featured"),
|
||||
("Megan Thee Stallion", "featured")])
|
||||
|
||||
def test_case_insensitive_featuring(self) -> None:
|
||||
"""Test that "Featuring" splits case-insensitively."""
|
||||
self.assertEqual(
|
||||
build_db.parse_artist_credit(
|
||||
"24kGoldn Featuring iann dior"),
|
||||
[("24kGoldn", "primary"),
|
||||
("iann dior", "featured")])
|
||||
|
||||
|
||||
class TestBuildDB(unittest.TestCase):
|
||||
"""Test cases for the working store build."""
|
||||
|
||||
CHART_CSV: str = (
|
||||
"year,rank,title,artist\n"
|
||||
"2016,1,Hello,Adele\n"
|
||||
"2016,2,One Dance,Drake featuring Wizkid\n"
|
||||
"2017,1,One Dance,Drake featuring Wizkid\n"
|
||||
"2017,2,Shape of You,Ed Sheeran\n")
|
||||
"""The default chart CSV fixture: 2 years with 2 ranks each."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
"""Create a temporary working directory with the fixtures."""
|
||||
tmp: tempfile.TemporaryDirectory[str] \
|
||||
= tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.__dir: Path = Path(tmp.name)
|
||||
old_cwd: str = os.getcwd()
|
||||
self.addCleanup(os.chdir, old_cwd)
|
||||
os.chdir(self.__dir)
|
||||
Path("data").mkdir()
|
||||
self.__write_chart(self.CHART_CSV)
|
||||
url: str = f"sqlite:///{self.__dir}/store.sqlite3"
|
||||
config.set_settings(config.Settings(
|
||||
SQLALCHEMY_DATABASE_URL=url,
|
||||
ANTHROPIC_API_KEY="test-key"))
|
||||
self.__ds: DataSource = DataSource()
|
||||
patchers: list[Any] = [
|
||||
mock.patch.object(build_db, "ds", self.__ds),
|
||||
mock.patch.object(build_db, "YEARS", [2016, 2017]),
|
||||
mock.patch.object(build_db, "RANKS_PER_YEAR", 2)]
|
||||
for patcher in patchers:
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
@staticmethod
|
||||
def __write_chart(content: str) -> None:
|
||||
"""Write the chart CSV fixture.
|
||||
|
||||
:param content: The CSV content.
|
||||
:return: None.
|
||||
"""
|
||||
Path("data/yearend_hot100_2016_2025.csv").write_text(
|
||||
content, encoding="utf-8")
|
||||
|
||||
@staticmethod
|
||||
def __run_build() -> tuple[int, str]:
|
||||
"""Run the build with the standard error captured.
|
||||
|
||||
:return: A tuple of the exit status and the standard
|
||||
error.
|
||||
"""
|
||||
stderr: io.StringIO = io.StringIO()
|
||||
with redirect_stderr(stderr):
|
||||
status: int = build_db.main([])
|
||||
return status, stderr.getvalue()
|
||||
|
||||
def __session(self) -> Session:
|
||||
"""Open a database session closed on test cleanup.
|
||||
|
||||
:return: The database session.
|
||||
"""
|
||||
session: Session = self.__ds.get_db()
|
||||
self.addCleanup(session.close)
|
||||
return session
|
||||
|
||||
def __song_titles(self) -> dict[int, str]:
|
||||
"""Read the song titles keyed by their IDs.
|
||||
|
||||
:return: The song titles, keyed by the song IDs.
|
||||
"""
|
||||
session: Session = self.__session()
|
||||
return {x.id: x.title
|
||||
for x in session.scalars(sa.select(Song))}
|
||||
|
||||
def test_dedup_repeated_song(self) -> None:
|
||||
"""Test that a song repeated across years is stored once."""
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_build()
|
||||
self.assertEqual(status, 0)
|
||||
session: Session = self.__session()
|
||||
self.assertEqual(
|
||||
len(list(session.scalars(sa.select(Song)))), 3)
|
||||
song: Song | None = session.scalar(
|
||||
sa.select(Song).where(Song.title == "One Dance"))
|
||||
assert song is not None
|
||||
self.assertEqual(sorted((x.year, x.rank)
|
||||
for x in song.chart_entries),
|
||||
[(2016, 2), (2017, 1)])
|
||||
self.assertEqual([(x.artist.name, x.role, x.position)
|
||||
for x in song.song_artists],
|
||||
[("Drake", "primary", 0),
|
||||
("Wizkid", "featured", 1)])
|
||||
self.assertIn("3 songs", stderr)
|
||||
self.assertIn("4 chart entries", stderr)
|
||||
self.assertIn("4 artists", stderr)
|
||||
self.assertIn("4 credits", stderr)
|
||||
|
||||
def test_first_run_on_fresh_store(self) -> None:
|
||||
"""Test that a build on a fresh store creates the tables."""
|
||||
self.assertFalse((self.__dir / "store.sqlite3").exists())
|
||||
self.assertEqual(self.__run_build()[0], 0)
|
||||
session: Session = self.__session()
|
||||
self.assertEqual(
|
||||
len(list(session.scalars(sa.select(Song)))), 3)
|
||||
|
||||
def test_deterministic_ids(self) -> None:
|
||||
"""Test that two rebuilds assign the same song IDs."""
|
||||
self.assertEqual(self.__run_build()[0], 0)
|
||||
titles: dict[int, str] = self.__song_titles()
|
||||
self.assertEqual(titles, {1: "Hello", 2: "One Dance",
|
||||
3: "Shape of You"})
|
||||
self.assertEqual(self.__run_build()[0], 0)
|
||||
self.assertEqual(self.__song_titles(), titles)
|
||||
|
||||
def test_failed_build_keeps_previous(self) -> None:
|
||||
"""Test that a failed build keeps the previous contents."""
|
||||
self.assertEqual(self.__run_build()[0], 0)
|
||||
titles: dict[int, str] = self.__song_titles()
|
||||
self.__write_chart(
|
||||
"year,rank,title,artist\n"
|
||||
"2016,1,Hello,Adele\n")
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_build()
|
||||
self.assertNotEqual(status, 0)
|
||||
self.assertIn("year 2016 rank 2", stderr)
|
||||
self.assertEqual(self.__song_titles(), titles)
|
||||
session: Session = self.__session()
|
||||
self.assertEqual(
|
||||
len(list(session.scalars(sa.select(ChartEntry)))), 4)
|
||||
|
||||
def test_missing_rank_fails(self) -> None:
|
||||
"""Test that a missing rank fails without partial data."""
|
||||
self.__write_chart(
|
||||
"year,rank,title,artist\n"
|
||||
"2016,1,Hello,Adele\n"
|
||||
"2016,2,One Dance,Drake featuring Wizkid\n"
|
||||
"2017,1,One Dance,Drake featuring Wizkid\n")
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_build()
|
||||
self.assertNotEqual(status, 0)
|
||||
self.assertIn("year 2017 rank 2", stderr)
|
||||
session: Session = self.__session()
|
||||
self.assertEqual(
|
||||
list(session.scalars(sa.select(ChartEntry))), [])
|
||||
self.assertEqual(list(session.scalars(sa.select(Song))), [])
|
||||
|
||||
def test_overrides_apply_over_wikidata(self) -> None:
|
||||
"""Test that the overrides win over the Wikidata snapshot."""
|
||||
Path("data/artists_wikidata.csv").write_text(
|
||||
"name,qid,gender,artist_type,genre,country,note\n"
|
||||
"Adele,Q2831,female,solo,pop,GB,\n",
|
||||
encoding="utf-8")
|
||||
Path("data/artists_overrides.csv").write_text(
|
||||
"name,qid,gender,artist_type,genre,country,note\n"
|
||||
"Adele,,,,soul,,manually checked\n",
|
||||
encoding="utf-8")
|
||||
self.assertEqual(self.__run_build()[0], 0)
|
||||
session: Session = self.__session()
|
||||
artist: Artist | None = session.scalar(
|
||||
sa.select(Artist).where(Artist.name == "Adele"))
|
||||
assert artist is not None
|
||||
self.assertEqual(artist.genre, "soul")
|
||||
self.assertEqual(artist.gender, "female")
|
||||
self.assertEqual(artist.wikidata_qid, "Q2831")
|
||||
self.assertEqual(artist.country, "GB")
|
||||
|
||||
def test_unknown_override_name_fails(self) -> None:
|
||||
"""Test that an unknown override name fails the build."""
|
||||
Path("data/artists_overrides.csv").write_text(
|
||||
"name,qid,gender,artist_type,genre,country,note\n"
|
||||
"Adel,,female,,,,typo\n", encoding="utf-8")
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_build()
|
||||
self.assertNotEqual(status, 0)
|
||||
self.assertIn("Adel", stderr)
|
||||
session: Session = self.__session()
|
||||
self.assertEqual(list(session.scalars(sa.select(Song))), [])
|
||||
|
||||
def test_lyrics_loaded(self) -> None:
|
||||
"""Test loading the lyrics cache into the songs."""
|
||||
Path("data/lyrics").mkdir()
|
||||
Path("data/lyrics/1.txt").write_text(
|
||||
"Hello, it's me\n", encoding="utf-8")
|
||||
Path("data/lyrics/999.txt").write_text(
|
||||
"orphan\n", encoding="utf-8")
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_build()
|
||||
self.assertEqual(status, 0)
|
||||
self.assertIn("999", stderr)
|
||||
self.assertIn("1 songs with lyrics", stderr)
|
||||
session: Session = self.__session()
|
||||
song: Song | None = session.get(Song, 1)
|
||||
assert song is not None
|
||||
self.assertEqual(song.title, "Hello")
|
||||
self.assertEqual(song.lyrics, "Hello, it's me\n")
|
||||
@@ -0,0 +1,310 @@
|
||||
# Tools for A Feminist Audit of Pop Music.
|
||||
# Copyright 2026 imacat. All rights reserved.
|
||||
# Authors:
|
||||
# imacat@mail.imacat.idv.tw (imacat), 2026/7/31
|
||||
"""Unit tests for the artist metadata fetcher module."""
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
import urllib.error
|
||||
from contextlib import redirect_stderr
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest import mock
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from pop_fem_audit_tools import config, fetch_artists
|
||||
from pop_fem_audit_tools.database import Base, DataSource
|
||||
from pop_fem_audit_tools.models import Artist
|
||||
|
||||
|
||||
class TestFetchArtists(unittest.TestCase):
|
||||
"""Test cases for the artist metadata fetcher."""
|
||||
|
||||
HEADER: list[str] = [
|
||||
"name", "qid", "gender", "artist_type", "genre",
|
||||
"country", "note"]
|
||||
"""The expected header row of the snapshot CSV file."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
"""Create a temporary working directory with the store."""
|
||||
tmp: tempfile.TemporaryDirectory[str] \
|
||||
= tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.__dir: Path = Path(tmp.name)
|
||||
old_cwd: str = os.getcwd()
|
||||
self.addCleanup(os.chdir, old_cwd)
|
||||
os.chdir(self.__dir)
|
||||
Path("data").mkdir()
|
||||
url: str = f"sqlite:///{self.__dir}/store.sqlite3"
|
||||
config.set_settings(config.Settings(
|
||||
SQLALCHEMY_DATABASE_URL=url,
|
||||
ANTHROPIC_API_KEY="test-key"))
|
||||
self.__ds: DataSource = DataSource()
|
||||
patchers: list[Any] = [
|
||||
mock.patch.object(fetch_artists, "ds", self.__ds),
|
||||
mock.patch.object(fetch_artists, "SLEEP_SECONDS",
|
||||
0.0)]
|
||||
for patcher in patchers:
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
def __seed(self, names: list[str]) -> None:
|
||||
"""Create the schema and the fixture artists.
|
||||
|
||||
The artist IDs are assigned in list order starting from
|
||||
1.
|
||||
|
||||
:param names: The artist names.
|
||||
:return: None.
|
||||
"""
|
||||
Base.metadata.create_all(self.__ds.engine)
|
||||
session: Session = self.__ds.get_db()
|
||||
try:
|
||||
name: str
|
||||
for name in names:
|
||||
session.add(Artist(name=name))
|
||||
session.commit()
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@staticmethod
|
||||
def __response(payload: dict[str, Any]) -> mock.MagicMock:
|
||||
"""Build a fake HTTP response with a JSON body.
|
||||
|
||||
:param payload: The JSON payload of the response body.
|
||||
:return: The fake response, usable as a context manager.
|
||||
"""
|
||||
response: mock.MagicMock = mock.MagicMock()
|
||||
response.__enter__.return_value = response
|
||||
response.read.return_value \
|
||||
= json.dumps(payload).encode("utf-8")
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
def __server_error() -> urllib.error.HTTPError:
|
||||
"""Build an HTTP 500 error.
|
||||
|
||||
:return: The HTTP 500 error.
|
||||
"""
|
||||
return urllib.error.HTTPError(
|
||||
"https://example.com/", 500,
|
||||
"Internal Server Error", None, None)
|
||||
|
||||
@staticmethod
|
||||
def __claim(qid: str) -> dict[str, Any]:
|
||||
"""Build a claim statement with an item-ID target.
|
||||
|
||||
:param qid: The item ID of the statement target.
|
||||
:return: The claim statement.
|
||||
"""
|
||||
return {"mainsnak": {"snaktype": "value",
|
||||
"datavalue": {"value": {"id": qid}}}}
|
||||
|
||||
@staticmethod
|
||||
def __labels(labels: dict[str, str]) -> dict[str, Any]:
|
||||
"""Build a label query response payload.
|
||||
|
||||
:param labels: The English labels, keyed by the item ID.
|
||||
:return: The response payload.
|
||||
"""
|
||||
return {"entities": {
|
||||
x: {"labels": {"en": {"value": y}}}
|
||||
for x, y in labels.items()}}
|
||||
|
||||
@staticmethod
|
||||
def __run_fetch() -> tuple[int, str]:
|
||||
"""Run the fetcher with the standard error captured.
|
||||
|
||||
:return: A tuple of the exit status and the standard
|
||||
error.
|
||||
"""
|
||||
stderr: io.StringIO = io.StringIO()
|
||||
with redirect_stderr(stderr):
|
||||
status: int = fetch_artists.main([])
|
||||
return status, stderr.getvalue()
|
||||
|
||||
@staticmethod
|
||||
def __read_rows(path: Path) -> list[list[str]]:
|
||||
"""Read the rows of a CSV file.
|
||||
|
||||
:param path: The CSV file.
|
||||
:return: The rows, the header included.
|
||||
"""
|
||||
with open(path, encoding="utf-8", newline="") as file:
|
||||
return list(csv.reader(file))
|
||||
|
||||
def test_human_artist(self) -> None:
|
||||
"""Test a human artist resolving the full metadata."""
|
||||
self.__seed(["Adele"])
|
||||
search: dict[str, Any] = {"search": [
|
||||
{"id": "Q1", "description": "English singer"}]}
|
||||
claims: dict[str, Any] = {"entities": {"Q1": {"claims": {
|
||||
"P21": [self.__claim("Q2")],
|
||||
"P31": [self.__claim("Q5")],
|
||||
"P136": [self.__claim("Q3"), self.__claim("Q4")],
|
||||
"P27": [self.__claim("Q6")]}}}}
|
||||
labels: dict[str, Any] = self.__labels({
|
||||
"Q2": "female", "Q5": "human", "Q3": "pop",
|
||||
"Q4": "soul music", "Q6": "United Kingdom"})
|
||||
urlopen: mock.Mock
|
||||
with mock.patch(
|
||||
"urllib.request.urlopen",
|
||||
side_effect=[self.__response(search),
|
||||
self.__response(claims),
|
||||
self.__response(labels)]) as urlopen:
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_fetch()
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(urlopen.call_count, 3)
|
||||
request: Any = urlopen.call_args_list[0][0][0]
|
||||
self.assertEqual(request.get_header("User-agent"),
|
||||
fetch_artists.USER_AGENT)
|
||||
urls: list[str] = [x[0][0].full_url
|
||||
for x in urlopen.call_args_list]
|
||||
self.assertEqual(
|
||||
urls[0],
|
||||
"https://www.wikidata.org/w/api.php"
|
||||
"?action=wbsearchentities&search=Adele&language=en"
|
||||
"&type=item&format=json")
|
||||
self.assertEqual(
|
||||
urls[1],
|
||||
"https://www.wikidata.org/w/api.php"
|
||||
"?action=wbgetentities&ids=Q1&props=claims"
|
||||
"&format=json")
|
||||
self.assertEqual(
|
||||
urls[2],
|
||||
"https://www.wikidata.org/w/api.php"
|
||||
"?action=wbgetentities&ids=Q2%7CQ5%7CQ3%7CQ4%7CQ6"
|
||||
"&props=labels&languages=en&format=json")
|
||||
rows: list[list[str]] = self.__read_rows(
|
||||
Path("data/artists_wikidata.csv"))
|
||||
self.assertEqual(len(rows), 2)
|
||||
self.assertEqual(rows[0], self.HEADER)
|
||||
self.assertEqual(rows[1], [
|
||||
"Adele", "Q1", "female", "solo", "pop; soul music",
|
||||
"United Kingdom", "English singer"])
|
||||
self.assertIn(
|
||||
"1 fetched, 0 not found, 0 errors, 0 skipped",
|
||||
stderr)
|
||||
|
||||
def test_band(self) -> None:
|
||||
"""Test a band resolving the group type and the origin."""
|
||||
self.__seed(["BTS"])
|
||||
search: dict[str, Any] = {"search": [
|
||||
{"id": "Q10",
|
||||
"description": "South Korean boy band"}]}
|
||||
claims: dict[str, Any] = {"entities": {"Q10": {"claims": {
|
||||
"P31": [self.__claim("Q11")],
|
||||
"P136": [self.__claim("Q12")],
|
||||
"P495": [self.__claim("Q13")]}}}}
|
||||
labels: dict[str, Any] = self.__labels({
|
||||
"Q11": "boy band", "Q12": "K-pop",
|
||||
"Q13": "South Korea"})
|
||||
with mock.patch(
|
||||
"urllib.request.urlopen",
|
||||
side_effect=[self.__response(search),
|
||||
self.__response(claims),
|
||||
self.__response(labels)]):
|
||||
status: int = self.__run_fetch()[0]
|
||||
self.assertEqual(status, 0)
|
||||
rows: list[list[str]] = self.__read_rows(
|
||||
Path("data/artists_wikidata.csv"))
|
||||
self.assertEqual(len(rows), 2)
|
||||
self.assertEqual(rows[1], [
|
||||
"BTS", "Q10", "", "group", "K-pop", "South Korea",
|
||||
"South Korean boy band"])
|
||||
|
||||
def test_not_found(self) -> None:
|
||||
"""Test that a search miss writes a not-found row."""
|
||||
self.__seed(["Nobody"])
|
||||
urlopen: mock.Mock
|
||||
with mock.patch(
|
||||
"urllib.request.urlopen",
|
||||
side_effect=[self.__response({"search": []})]
|
||||
) as urlopen:
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_fetch()
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(urlopen.call_count, 1)
|
||||
rows: list[list[str]] = self.__read_rows(
|
||||
Path("data/artists_wikidata.csv"))
|
||||
self.assertEqual(len(rows), 2)
|
||||
self.assertEqual(rows[0], self.HEADER)
|
||||
self.assertEqual(rows[1], [
|
||||
"Nobody", "", "", "", "", "", "not found"])
|
||||
self.assertIn(
|
||||
"0 fetched, 1 not found, 0 errors, 0 skipped",
|
||||
stderr)
|
||||
|
||||
def test_http_error_continues(self) -> None:
|
||||
"""Test that an HTTP error is noted and the run goes on."""
|
||||
self.__seed(["Broken", "Nobody"])
|
||||
with mock.patch(
|
||||
"urllib.request.urlopen",
|
||||
side_effect=[self.__server_error(),
|
||||
self.__response({"search": []})]):
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_fetch()
|
||||
self.assertEqual(status, 0)
|
||||
rows: list[list[str]] = self.__read_rows(
|
||||
Path("data/artists_wikidata.csv"))
|
||||
self.assertEqual(len(rows), 3)
|
||||
self.assertEqual(rows[1][:2], ["Broken", ""])
|
||||
self.assertTrue(rows[1][6].startswith("error: "))
|
||||
self.assertEqual(rows[2], [
|
||||
"Nobody", "", "", "", "", "", "not found"])
|
||||
self.assertIn(
|
||||
"0 fetched, 1 not found, 1 errors, 0 skipped",
|
||||
stderr)
|
||||
|
||||
def test_rerun_skips_existing(self) -> None:
|
||||
"""Test that the snapshot rows are skipped and preserved."""
|
||||
self.__seed(["Adele", "Nobody"])
|
||||
snapshot: Path = Path("data/artists_wikidata.csv")
|
||||
old_row: list[str] = [
|
||||
"Adele", "Q1", "female", "solo", "pop",
|
||||
"United Kingdom", "English singer"]
|
||||
with open(snapshot, "w", encoding="utf-8",
|
||||
newline="") as file:
|
||||
writer: Any = csv.writer(file)
|
||||
writer.writerow(self.HEADER)
|
||||
writer.writerow(old_row)
|
||||
urlopen: mock.Mock
|
||||
with mock.patch(
|
||||
"urllib.request.urlopen",
|
||||
side_effect=[self.__response({"search": []})]
|
||||
) as urlopen:
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_fetch()
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(urlopen.call_count, 1)
|
||||
rows: list[list[str]] = self.__read_rows(snapshot)
|
||||
self.assertEqual(len(rows), 3)
|
||||
self.assertEqual(rows[0], self.HEADER)
|
||||
self.assertNotIn(self.HEADER, rows[1:])
|
||||
self.assertEqual(rows[1], old_row)
|
||||
self.assertEqual(rows[2], [
|
||||
"Nobody", "", "", "", "", "", "not found"])
|
||||
self.assertIn(
|
||||
"0 fetched, 1 not found, 0 errors, 1 skipped",
|
||||
stderr)
|
||||
|
||||
def test_no_store_fails(self) -> None:
|
||||
"""Test that a missing working store fails the run."""
|
||||
urlopen: mock.Mock
|
||||
with mock.patch("urllib.request.urlopen") as urlopen:
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_fetch()
|
||||
self.assertNotEqual(status, 0)
|
||||
urlopen.assert_not_called()
|
||||
self.assertIn("error:", stderr)
|
||||
@@ -0,0 +1,280 @@
|
||||
# Tools for A Feminist Audit of Pop Music.
|
||||
# Copyright 2026 imacat. All rights reserved.
|
||||
# Authors:
|
||||
# imacat@mail.imacat.idv.tw (imacat), 2026/7/31
|
||||
"""Unit tests for the lyrics fetcher module."""
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
import urllib.error
|
||||
from contextlib import redirect_stderr
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest import mock
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from pop_fem_audit_tools import config, fetch_lyrics
|
||||
from pop_fem_audit_tools.database import Base, DataSource
|
||||
from pop_fem_audit_tools.models import (
|
||||
Artist,
|
||||
Song,
|
||||
SongArtist,
|
||||
)
|
||||
|
||||
|
||||
class TestFetchLyrics(unittest.TestCase):
|
||||
"""Test cases for the lyrics fetcher."""
|
||||
|
||||
PROVENANCE_HEADER: list[str] = [
|
||||
"song_id", "source", "method", "acquired_at", "note"]
|
||||
"""The expected header row of the provenance CSV file."""
|
||||
MISSING_HEADER: list[str] = [
|
||||
"song_id", "title", "artist_credit", "reason"]
|
||||
"""The expected header row of the missing report CSV file."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
"""Create a temporary working directory with the store."""
|
||||
tmp: tempfile.TemporaryDirectory[str] \
|
||||
= tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.__dir: Path = Path(tmp.name)
|
||||
old_cwd: str = os.getcwd()
|
||||
self.addCleanup(os.chdir, old_cwd)
|
||||
os.chdir(self.__dir)
|
||||
Path("data").mkdir()
|
||||
url: str = f"sqlite:///{self.__dir}/store.sqlite3"
|
||||
config.set_settings(config.Settings(
|
||||
SQLALCHEMY_DATABASE_URL=url,
|
||||
ANTHROPIC_API_KEY="test-key"))
|
||||
self.__ds: DataSource = DataSource()
|
||||
patchers: list[Any] = [
|
||||
mock.patch.object(fetch_lyrics, "ds", self.__ds),
|
||||
mock.patch.object(fetch_lyrics, "SLEEP_SECONDS", 0.0)]
|
||||
for patcher in patchers:
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
def __seed(self, songs: list[tuple[str, str]]) -> None:
|
||||
"""Create the schema and the fixture songs.
|
||||
|
||||
Each song gets a single primary artist at position 0;
|
||||
the song IDs are assigned in list order starting from 1.
|
||||
|
||||
:param songs: The (title, artist) pairs.
|
||||
:return: None.
|
||||
"""
|
||||
Base.metadata.create_all(self.__ds.engine)
|
||||
session: Session = self.__ds.get_db()
|
||||
try:
|
||||
artists: dict[str, Artist] = {}
|
||||
title: str
|
||||
artist: str
|
||||
for title, artist in songs:
|
||||
if artist not in artists:
|
||||
artists[artist] = Artist(name=artist)
|
||||
song: Song = Song(title=title,
|
||||
artist_credit=artist)
|
||||
session.add(song)
|
||||
session.add(SongArtist(song=song,
|
||||
artist=artists[artist],
|
||||
role="primary",
|
||||
position=0))
|
||||
session.commit()
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@staticmethod
|
||||
def __response(payload: dict[str, Any]) -> mock.MagicMock:
|
||||
"""Build a fake HTTP response with a JSON body.
|
||||
|
||||
:param payload: The JSON payload of the response body.
|
||||
:return: The fake response, usable as a context manager.
|
||||
"""
|
||||
response: mock.MagicMock = mock.MagicMock()
|
||||
response.__enter__.return_value = response
|
||||
response.read.return_value \
|
||||
= json.dumps(payload).encode("utf-8")
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
def __not_found() -> urllib.error.HTTPError:
|
||||
"""Build an HTTP 404 error.
|
||||
|
||||
:return: The HTTP 404 error.
|
||||
"""
|
||||
return urllib.error.HTTPError(
|
||||
"https://example.com/", 404, "Not Found", None, None)
|
||||
|
||||
@staticmethod
|
||||
def __run_fetch() -> tuple[int, str]:
|
||||
"""Run the fetcher with the standard error captured.
|
||||
|
||||
:return: A tuple of the exit status and the standard
|
||||
error.
|
||||
"""
|
||||
stderr: io.StringIO = io.StringIO()
|
||||
with redirect_stderr(stderr):
|
||||
status: int = fetch_lyrics.main([])
|
||||
return status, stderr.getvalue()
|
||||
|
||||
@staticmethod
|
||||
def __read_rows(path: Path) -> list[list[str]]:
|
||||
"""Read the rows of a CSV file.
|
||||
|
||||
:param path: The CSV file.
|
||||
:return: The rows, the header included.
|
||||
"""
|
||||
with open(path, encoding="utf-8", newline="") as file:
|
||||
return list(csv.reader(file))
|
||||
|
||||
def test_ovh_hit(self) -> None:
|
||||
"""Test that a Lyrics.ovh hit writes the cache files."""
|
||||
self.__seed([("Hello", "Adele")])
|
||||
urlopen: mock.Mock
|
||||
with mock.patch(
|
||||
"urllib.request.urlopen",
|
||||
side_effect=[self.__response(
|
||||
{"lyrics": "Hello, it's me\n"})]) as urlopen:
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_fetch()
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(urlopen.call_count, 1)
|
||||
request: Any = urlopen.call_args[0][0]
|
||||
self.assertEqual(request.get_header("User-agent"),
|
||||
fetch_lyrics.USER_AGENT)
|
||||
self.assertEqual(
|
||||
Path("data/lyrics/1.txt").read_text(encoding="utf-8"),
|
||||
"Hello, it's me\n")
|
||||
rows: list[list[str]] = self.__read_rows(
|
||||
Path("data/lyrics_provenance.csv"))
|
||||
self.assertEqual(len(rows), 2)
|
||||
self.assertEqual(rows[0], self.PROVENANCE_HEADER)
|
||||
self.assertEqual(rows[1][:3],
|
||||
["1", "lyrics.ovh", "api-fetch"])
|
||||
self.assertNotEqual(rows[1][3], "")
|
||||
self.assertEqual(rows[1][4], "")
|
||||
self.assertIn("1 fetched, 0 missed", stderr)
|
||||
|
||||
def test_lrclib_fallback(self) -> None:
|
||||
"""Test that an ovh miss falls back to an LRCLIB hit."""
|
||||
self.__seed([("Hello", "Adele")])
|
||||
urlopen: mock.Mock
|
||||
with mock.patch(
|
||||
"urllib.request.urlopen",
|
||||
side_effect=[
|
||||
self.__not_found(),
|
||||
self.__response({"plainLyrics": "Hello\n"})]
|
||||
) as urlopen:
|
||||
status: int = self.__run_fetch()[0]
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(urlopen.call_count, 2)
|
||||
self.assertEqual(
|
||||
Path("data/lyrics/1.txt").read_text(encoding="utf-8"),
|
||||
"Hello\n")
|
||||
rows: list[list[str]] = self.__read_rows(
|
||||
Path("data/lyrics_provenance.csv"))
|
||||
self.assertEqual(rows[1][:2], ["1", "lrclib"])
|
||||
|
||||
def test_both_miss(self) -> None:
|
||||
"""Test that a double miss reports the song as missing."""
|
||||
self.__seed([("Hello", "Adele")])
|
||||
with mock.patch(
|
||||
"urllib.request.urlopen",
|
||||
side_effect=[self.__not_found(),
|
||||
self.__not_found()]):
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_fetch()
|
||||
self.assertEqual(status, 0)
|
||||
self.assertFalse(Path("data/lyrics/1.txt").exists())
|
||||
self.assertFalse(
|
||||
Path("data/lyrics_provenance.csv").exists())
|
||||
rows: list[list[str]] = self.__read_rows(
|
||||
Path("data/lyrics_missing.csv"))
|
||||
self.assertEqual(len(rows), 2)
|
||||
self.assertEqual(rows[0], self.MISSING_HEADER)
|
||||
self.assertEqual(rows[1][:3], ["1", "Hello", "Adele"])
|
||||
self.assertIn("0 fetched, 1 missed", stderr)
|
||||
|
||||
def test_cached_song_skipped(self) -> None:
|
||||
"""Test that a cached song triggers no HTTP request."""
|
||||
self.__seed([("Hello", "Adele")])
|
||||
Path("data/lyrics").mkdir()
|
||||
Path("data/lyrics/1.txt").write_text(
|
||||
"cached\n", encoding="utf-8")
|
||||
urlopen: mock.Mock
|
||||
with mock.patch("urllib.request.urlopen") as urlopen:
|
||||
status: int = self.__run_fetch()[0]
|
||||
self.assertEqual(status, 0)
|
||||
urlopen.assert_not_called()
|
||||
self.assertEqual(
|
||||
Path("data/lyrics/1.txt").read_text(encoding="utf-8"),
|
||||
"cached\n")
|
||||
rows: list[list[str]] = self.__read_rows(
|
||||
Path("data/lyrics_missing.csv"))
|
||||
self.assertEqual(rows, [self.MISSING_HEADER])
|
||||
|
||||
def test_url_encoding(self) -> None:
|
||||
"""Test the percent-encoding of the artist and title."""
|
||||
self.__seed([("What's Up? / Down", "AC/DC & Friends")])
|
||||
urlopen: mock.Mock
|
||||
with mock.patch(
|
||||
"urllib.request.urlopen",
|
||||
side_effect=[self.__not_found(),
|
||||
self.__not_found()]) as urlopen:
|
||||
self.__run_fetch()
|
||||
self.assertEqual(urlopen.call_count, 2)
|
||||
urls: list[str] = [x[0][0].full_url
|
||||
for x in urlopen.call_args_list]
|
||||
self.assertEqual(
|
||||
urls[0],
|
||||
"https://api.lyrics.ovh/v1/AC%2FDC%20%26%20Friends/"
|
||||
"What%27s%20Up%3F%20%2F%20Down")
|
||||
self.assertEqual(
|
||||
urls[1],
|
||||
"https://lrclib.net/api/get"
|
||||
"?artist_name=AC%2FDC+%26+Friends"
|
||||
"&track_name=What%27s+Up%3F+%2F+Down")
|
||||
|
||||
def test_provenance_single_header(self) -> None:
|
||||
"""Test that the provenance keeps one header across runs."""
|
||||
self.__seed([("Hello", "Adele"),
|
||||
("Umbrella", "Rihanna")])
|
||||
with mock.patch(
|
||||
"urllib.request.urlopen",
|
||||
side_effect=[
|
||||
self.__response({"lyrics": "one\n"}),
|
||||
self.__response({"lyrics": "two\n"})]):
|
||||
self.assertEqual(self.__run_fetch()[0], 0)
|
||||
provenance: Path = Path("data/lyrics_provenance.csv")
|
||||
rows: list[list[str]] = self.__read_rows(provenance)
|
||||
self.assertEqual(len(rows), 3)
|
||||
self.assertEqual(rows[0], self.PROVENANCE_HEADER)
|
||||
Path("data/lyrics/2.txt").unlink()
|
||||
with mock.patch(
|
||||
"urllib.request.urlopen",
|
||||
side_effect=[
|
||||
self.__response({"lyrics": "two again\n"})]):
|
||||
self.assertEqual(self.__run_fetch()[0], 0)
|
||||
rows = self.__read_rows(provenance)
|
||||
self.assertEqual(len(rows), 4)
|
||||
self.assertEqual(rows[0], self.PROVENANCE_HEADER)
|
||||
self.assertNotIn(self.PROVENANCE_HEADER, rows[1:])
|
||||
self.assertEqual([x[0] for x in rows[1:]],
|
||||
["1", "2", "2"])
|
||||
|
||||
def test_no_store_fails(self) -> None:
|
||||
"""Test that a missing working store fails the run."""
|
||||
urlopen: mock.Mock
|
||||
with mock.patch("urllib.request.urlopen") as urlopen:
|
||||
status: int
|
||||
stderr: str
|
||||
status, stderr = self.__run_fetch()
|
||||
self.assertNotEqual(status, 0)
|
||||
urlopen.assert_not_called()
|
||||
self.assertIn("error:", stderr)
|
||||
@@ -0,0 +1,95 @@
|
||||
# Tools for A Feminist Audit of Pop Music.
|
||||
# Copyright 2026 imacat. All rights reserved.
|
||||
# Authors:
|
||||
# imacat@mail.imacat.idv.tw (imacat), 2026/7/31
|
||||
# AI assistance: Claude Code (Anthropic)
|
||||
"""Unit tests for the data models."""
|
||||
import unittest
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from pop_fem_audit_tools import config
|
||||
from pop_fem_audit_tools.database import Base, DataSource
|
||||
from pop_fem_audit_tools.models import (
|
||||
Artist,
|
||||
ChartEntry,
|
||||
Song,
|
||||
SongArtist,
|
||||
)
|
||||
|
||||
|
||||
class TestModels(unittest.TestCase):
|
||||
"""Test cases for the data models."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
"""Create the schema on an in-memory SQLite database."""
|
||||
config.set_settings(config.Settings(
|
||||
SQLALCHEMY_DATABASE_URL="sqlite://",
|
||||
ANTHROPIC_API_KEY="test-key"))
|
||||
self.__ds: DataSource = DataSource()
|
||||
Base.metadata.create_all(self.__ds.engine)
|
||||
self.__session: Session = self.__ds.get_db()
|
||||
self.addCleanup(self.__session.close)
|
||||
|
||||
def __add_song(self) -> None:
|
||||
"""Add a song with chart entries, artists, and lyrics.
|
||||
|
||||
:return: None.
|
||||
"""
|
||||
song: Song = Song(title="One Dance",
|
||||
artist_credit="Drake featuring Wizkid")
|
||||
song.chart_entries = [ChartEntry(year=2016, rank=4)]
|
||||
song.song_artists = [
|
||||
SongArtist(artist=Artist(name="Drake"),
|
||||
role="primary", position=0),
|
||||
SongArtist(artist=Artist(name="Wizkid"),
|
||||
role="featured", position=1)]
|
||||
song.lyrics = "Baby, I like your style"
|
||||
self.__session.add(song)
|
||||
self.__session.commit()
|
||||
|
||||
def test_song_graph(self) -> None:
|
||||
"""Test reading a song graph back through relationships."""
|
||||
self.__add_song()
|
||||
self.__session.expunge_all()
|
||||
song: Song | None = self.__session.scalar(
|
||||
sa.select(Song).where(Song.title == "One Dance"))
|
||||
assert song is not None
|
||||
self.assertEqual(song.artist_credit,
|
||||
"Drake featuring Wizkid")
|
||||
self.assertEqual([(x.year, x.rank)
|
||||
for x in song.chart_entries],
|
||||
[(2016, 4)])
|
||||
self.assertEqual([(x.artist.name, x.role, x.position)
|
||||
for x in song.song_artists],
|
||||
[("Drake", "primary", 0),
|
||||
("Wizkid", "featured", 1)])
|
||||
self.assertEqual(song.lyrics,
|
||||
"Baby, I like your style")
|
||||
artist: Artist | None = self.__session.scalar(
|
||||
sa.select(Artist).where(Artist.name == "Wizkid"))
|
||||
assert artist is not None
|
||||
self.assertEqual([x.song.title for x in artist.song_artists],
|
||||
["One Dance"])
|
||||
|
||||
def test_duplicated_song_rejected(self) -> None:
|
||||
"""Test that a duplicated title and artist credit fails."""
|
||||
self.__add_song()
|
||||
self.__session.add(
|
||||
Song(title="One Dance",
|
||||
artist_credit="Drake featuring Wizkid"))
|
||||
with self.assertRaises(sa.exc.IntegrityError):
|
||||
self.__session.commit()
|
||||
|
||||
def test_invalid_role_rejected(self) -> None:
|
||||
"""Test that an invalid song-artist role fails."""
|
||||
self.__add_song()
|
||||
song: Song | None = self.__session.scalar(
|
||||
sa.select(Song).where(Song.title == "One Dance"))
|
||||
assert song is not None
|
||||
self.__session.add(
|
||||
SongArtist(song=song, artist=Artist(name="Kyla"),
|
||||
role="cover", position=2))
|
||||
with self.assertRaises(sa.exc.IntegrityError):
|
||||
self.__session.commit()
|
||||
Reference in New Issue
Block a user