imacat/pop-fem-audit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Split artist credits with documented exceptions and normalize artist names

Browse Source
This commit is contained in:
imacat
2026-08-04 15:15:49 +08:00
parent 6a91bc73eb
commit ae0e9d0a08
3 changed files with 369 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/src/pop_fem_audit_tools/build_db.py
+149 -21
View File
@@ -29,7 +29,15 @@ credit, the credit canonicalized through
``CANONICAL_ARTIST_CREDITS``; a credit listed there collapses onto
the same song as its canonical form, and the stored artist credit
is always the canonical form. Artist deduplication is by the
exact parsed artist name, as printed on the chart.
identity key resolved from the parsed artist name (see
`resolve_artist_identity`): the case-folded name, or, when that
case-folded name is listed in ``CANONICAL_ARTIST_NAMES``, the
case-folded canonical spelling, so letter-case variants and
alternate spellings mapped to the same canonical name all
collapse onto a single artist row. The stored artist name is the
first-seen spelling, except for the names listed in
``CANONICAL_ARTIST_NAMES``, which always store the canonical
spelling regardless of which variant is seen first.
"""
import argparse
import csv
@@ -68,13 +76,67 @@ FEATURING_PATTERN: re.Pattern[str] = re.compile(
r" featuring | feat\. ", re.IGNORECASE)
"""The pattern splitting the primary and featured sides."""
DELIMITER_PATTERN: re.Pattern[str] = re.compile(
r", | & | \+ |(?i: and | x | with )")
r", | & | \+ | / |(?i: and | x | with )")
"""The pattern splitting the artist names within a side."""
COLON_PATTERN: re.Pattern[str] = re.compile(r": ")
"""The pattern separating a group prefix from its members in a
"<group>: <members>" credit."""
PAREN_MEMBERS_PATTERN: re.Pattern[str] = re.compile(
r"^.+ \((?P<members>.+)\)$")
"""The pattern separating a group name from its members in a
"<group> (<members>)" credit spanning the whole credit."""
DUET_WITH_PATTERN: re.Pattern[str] = re.compile(
r" Duet With ", re.IGNORECASE)
"""The pattern normalizing the "Duet With" co-billing connector
to the plain "with" delimiter."""
PROTECTED_ARTIST_NAMES: tuple[str, ...] = (
"Tyler, The Creator",
"Lil Nas X",
"Tones And I",
)
"""The exact artist names guarded from the delimiter splitting,
because each contains a delimiter word or punctuation as part of
the name itself."""
EXCEPTION_CREDITS: dict[str, list[tuple[str, Role]]] = {
"SpotemGottem Featuring Pooh Shiesty Or DaBaby": [
("SpotemGottem", Role.PRIMARY),
("Pooh Shiesty", Role.FEATURED),
("DaBaby", Role.FEATURED),
],
"THE SCOTTS, Travis Scott & Kid Cudi": [
("Travis Scott", Role.PRIMARY),
("Kid Cudi", Role.PRIMARY),
],
"Drake Featuring The Throne": [
("Drake", Role.PRIMARY),
("Jay Z", Role.FEATURED),
("Kanye West", Role.FEATURED),
],
}
"""The single-credit exceptions parsed by an explicit lookup
rather than by the general rules, because the credit text alone
does not spell out the correct member split."""
CANONICAL_ARTIST_CREDITS: dict[str, str] = {
"benny blanco, Halsey & Khalid": "Benny Blanco, Halsey & Khalid",
}
"""The canonical artist credit spellings, keyed by a variant
credit string."""
CANONICAL_ARTIST_NAMES: dict[str, str] = {
"beyonce": "Beyoncé",
"5 seconds of summer": "5 Seconds of Summer",
"a boogie wit da hoodie": "A Boogie wit da Hoodie",
"benny blanco": "benny blanco",
"blackbear": "blackbear",
"chance the rapper": "Chance the Rapper",
"xxxtentacion": "XXXTENTACION",
"maneskin": "Måneskin",
"rose": "ROSÉ",
"mo": "MØ",
"wizkid": "Wizkid",
"ye": "Kanye West",
}
"""The canonical artist spellings, keyed by the case-folded
identity."""
class BuildError(Exception):
@@ -109,22 +171,55 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
def parse_artist_credit(credit: str) -> list[tuple[str, Role]]:
"""Parse a combined artist credit into artists and roles.
The credit splits into a primary side and a featured side on
the word "featuring" or "feat.", case-insensitively; without
them, every artist is primary. Each side splits into artist
names on the delimiters ", ", " & ", " + " (literally) and
" and ", " x ", " with " (case-insensitively).
A credit listed in ``EXCEPTION_CREDITS`` is looked up verbatim,
because its correct split is not derivable from the credit
text alone. Otherwise the credit first reduces to an
effective credit: a "<group>: <members>" prefix (split at the
first ": ") drops the group and keeps the members; failing
that, a "<group> (<members>)" suffix spanning the whole credit
drops the group and keeps the members. The "Duet With"
connector, case-insensitively, then normalizes to "with". The
effective credit splits into a primary side and a featured
side on the word "featuring" or "feat.", case-insensitively;
without them, every artist is primary. Each side splits into
artist names on the delimiters ", ", " & ", " + ", " / "
(literally) and " and ", " x ", " with " (case-insensitively),
except for the names listed in ``PROTECTED_ARTIST_NAMES``,
which are never split even though each contains a delimiter
word or punctuation.
Known limitation: a compound act name that contains one of the
delimiters is over-split; such cases are corrected later via
the human override layer.
delimiters, other than the protected names, is over-split;
such cases are corrected later via the human override layer.
:param credit: The combined artist credit string.
:return: The (name, role) pairs in credit order, primary side
first, with the role ``Role.PRIMARY`` or
``Role.FEATURED``.
"""
sides: list[str] = FEATURING_PATTERN.split(credit, maxsplit=1)
if credit in EXCEPTION_CREDITS:
return list(EXCEPTION_CREDITS[credit])
effective: str = credit
colon_match: re.Match[str] | None = COLON_PATTERN.search(
effective)
if colon_match is not None:
effective = effective[colon_match.end():]
else:
paren_match: re.Match[str] | None = \
PAREN_MEMBERS_PATTERN.match(effective)
if paren_match is not None:
effective = paren_match.group("members")
effective = DUET_WITH_PATTERN.sub(" with ", effective)
placeholders: dict[str, str] = {}
index: int
protected: str
for index, protected in enumerate(PROTECTED_ARTIST_NAMES):
if protected in effective:
placeholder: str = f"{index}"
placeholders[placeholder] = protected
effective = effective.replace(protected, placeholder)
sides: list[str] = FEATURING_PATTERN.split(
effective, maxsplit=1)
pairs: list[tuple[str, Role]] = []
role: Role
side: str
@@ -132,6 +227,10 @@ def parse_artist_credit(credit: str) -> list[tuple[str, Role]]:
token: str
for token in DELIMITER_PATTERN.split(side):
name: str = token.strip()
placeholder = ""
original: str
for placeholder, original in placeholders.items():
name = name.replace(placeholder, original)
if name != "":
pairs.append((name, role))
return pairs
@@ -153,21 +252,46 @@ def song_identity(title: str, credit: str) -> tuple[str, str]:
return title, CANONICAL_ARTIST_CREDITS.get(credit, credit)
def resolve_artist_identity(name: str) -> tuple[str, str]:
"""Resolve the dedup key and the stored spelling of a name.
The name's case-folded form is looked up in
``CANONICAL_ARTIST_NAMES`` first; when it is listed there, the
dedup key is the canonical spelling case-folded and the stored
spelling is the canonical spelling, so every variant of the
name, canonical or not, resolves to the same identity.
Otherwise the dedup key is the name case-folded and the stored
spelling is the given name.
:param name: An artist name, as parsed from a credit.
:return: A tuple of the dedup key and the stored spelling.
"""
folded: str = name.casefold()
canonical: str | None = CANONICAL_ARTIST_NAMES.get(folded)
if canonical is not None:
return canonical.casefold(), canonical
return folded, name
def create_song(session: Session, song_id: int, title: str,
credit: str, artists: dict[str, Artist]) -> Song:
"""Create a song with its parsed artist credits.
The song takes the given ID. A newly created artist takes
the ID following the known artists, in credit order. An
artist duplicated within the credit is kept only at its
first occurrence, with a warning to the standard error.
The song takes the given ID. An artist parsed out of the
credit is matched against the known artists by its identity
key (see `resolve_artist_identity`); a newly seen one takes
the ID following the known artists, keyed by its identity key,
and its stored name is the resolved stored spelling. An
artist duplicated within the credit, by its identity key, is
kept only at its first occurrence, with a warning to the
standard error.
:param session: The database session.
:param song_id: The song ID to assign.
:param title: The song title.
:param credit: The combined artist credit string.
:param artists: The known artists by name, updated with the
newly created ones as an observable side effect.
:param artists: The known artists by identity key, updated
with the newly created ones as an observable side effect.
:return: The created song, added to the session.
"""
song: Song = Song(id=song_id, title=title,
@@ -178,14 +302,18 @@ def create_song(session: Session, song_id: int, title: str,
name: str
role: Role
for name, role in parse_artist_credit(credit):
if name in seen:
key: str
stored_name: str
key, stored_name = resolve_artist_identity(name)
if key in seen:
print(f"warning: {credit}: duplicated artist"
f" \"{name}\"", file=sys.stderr)
continue
seen.add(name)
if name not in artists:
artists[name] = Artist(id=len(artists) + 1, name=name)
session.add(SongArtist(song=song, artist=artists[name],
seen.add(key)
if key not in artists:
artists[key] = Artist(id=len(artists) + 1,
name=stored_name)
session.add(SongArtist(song=song, artist=artists[key],
role=role, position=position))
position += 1
return song
tools/tests/test_build_db.py
+212 -2
View File
@@ -90,6 +90,150 @@ class TestParseArtistCredit(unittest.TestCase):
[("24kGoldn", Role.PRIMARY),
("iann dior", Role.FEATURED)])
def test_colon_prefix_group(self) -> None:
"""Test that a colon-prefixed group name is dropped."""
self.assertEqual(
build_db.parse_artist_credit(
"¥$: Ye & Ty Dolla $ign Featuring Rich The Kid"
" & Playboi Carti"),
[("Ye", Role.PRIMARY),
("Ty Dolla $ign", Role.PRIMARY),
("Rich The Kid", Role.FEATURED),
("Playboi Carti", Role.FEATURED)])
def test_colon_prefix_with_ampersand_in_prefix(self) -> None:
"""Test a colon prefix that itself contains "&"."""
self.assertEqual(
build_db.parse_artist_credit(
"Rumi & JINU: EJAE & Andrew Choi"),
[("EJAE", Role.PRIMARY),
("Andrew Choi", Role.PRIMARY)])
def test_colon_prefix_comma_list(self) -> None:
"""Test a colon-prefixed comma-separated member list."""
self.assertEqual(
build_db.parse_artist_credit(
"HUNTR/X: EJAE, Audrey Nuna & REI AMI"),
[("EJAE", Role.PRIMARY),
("Audrey Nuna", Role.PRIMARY),
("REI AMI", Role.PRIMARY)])
def test_colon_prefix_five_members(self) -> None:
"""Test a colon-prefixed five-member list."""
self.assertEqual(
build_db.parse_artist_credit(
"Saja Boys: Andrew Choi, Neckwav, Danny Chung,"
" Kevin Woo & samUIL Lee"),
[("Andrew Choi", Role.PRIMARY),
("Neckwav", Role.PRIMARY),
("Danny Chung", Role.PRIMARY),
("Kevin Woo", Role.PRIMARY),
("samUIL Lee", Role.PRIMARY)])
def test_colon_prefix_duo(self) -> None:
"""Test a colon-prefixed two-member list."""
self.assertEqual(
build_db.parse_artist_credit(
"THE ANXIETY: WILLOW & Tyler Cole"),
[("WILLOW", Role.PRIMARY),
("Tyler Cole", Role.PRIMARY)])
def test_parenthesized_members(self) -> None:
"""Test that a parenthesized member list replaces the
group name spanning the whole credit."""
self.assertEqual(
build_db.parse_artist_credit(
"Silk Sonic (Bruno Mars & Anderson .Paak)"),
[("Bruno Mars", Role.PRIMARY),
("Anderson .Paak", Role.PRIMARY)])
def test_duet_with_connector(self) -> None:
"""Test that "Duet With" is a co-billing connector like
"with", dropping the word "Duet" entirely."""
self.assertEqual(
build_db.parse_artist_credit(
"Blake Shelton Duet With Gwen Stefani"),
[("Blake Shelton", Role.PRIMARY),
("Gwen Stefani", Role.PRIMARY)])
self.assertEqual(
build_db.parse_artist_credit(
"Keith Urban Duet With P!nk"),
[("Keith Urban", Role.PRIMARY),
("P!nk", Role.PRIMARY)])
def test_slash_delimiter(self) -> None:
"""Test the " / " co-billing delimiter."""
self.assertEqual(
build_db.parse_artist_credit(
"Cole Swindell / Lainey Wilson"),
[("Cole Swindell", Role.PRIMARY),
("Lainey Wilson", Role.PRIMARY)])
self.assertEqual(
build_db.parse_artist_credit("Zayn / Taylor Swift"),
[("Zayn", Role.PRIMARY),
("Taylor Swift", Role.PRIMARY)])
def test_protected_name_lil_nas_x(self) -> None:
"""Test that "Lil Nas X" is guarded from the " x "
delimiter split."""
self.assertEqual(
build_db.parse_artist_credit("Lil Nas X & Jack Harlow"),
[("Lil Nas X", Role.PRIMARY),
("Jack Harlow", Role.PRIMARY)])
def test_protected_name_tyler_the_creator(self) -> None:
"""Test that "Tyler, The Creator" is guarded from the
comma delimiter split."""
self.assertEqual(
build_db.parse_artist_credit(
"Tyler, The Creator Featuring GloRilla, Sexyy Red"
" & Lil Wayne"),
[("Tyler, The Creator", Role.PRIMARY),
("GloRilla", Role.FEATURED),
("Sexyy Red", Role.FEATURED),
("Lil Wayne", Role.FEATURED)])
def test_protected_name_tones_and_i(self) -> None:
"""Test that "Tones And I" is guarded from the " and "
delimiter split."""
self.assertEqual(
build_db.parse_artist_credit("Tones And I"),
[("Tones And I", Role.PRIMARY)])
def test_exception_spotemgottem(self) -> None:
"""Test the SpotemGottem exception-table credit."""
self.assertEqual(
build_db.parse_artist_credit(
"SpotemGottem Featuring Pooh Shiesty Or DaBaby"),
[("SpotemGottem", Role.PRIMARY),
("Pooh Shiesty", Role.FEATURED),
("DaBaby", Role.FEATURED)])
def test_exception_the_scotts(self) -> None:
"""Test the THE SCOTTS exception-table credit."""
self.assertEqual(
build_db.parse_artist_credit(
"THE SCOTTS, Travis Scott & Kid Cudi"),
[("Travis Scott", Role.PRIMARY),
("Kid Cudi", Role.PRIMARY)])
def test_exception_drake_featuring_the_throne(self) -> None:
"""Test the Drake Featuring The Throne exception-table
credit."""
self.assertEqual(
build_db.parse_artist_credit(
"Drake Featuring The Throne"),
[("Drake", Role.PRIMARY),
("Jay Z", Role.FEATURED),
("Kanye West", Role.FEATURED)])
def test_plus_delimiter_unaffected(self) -> None:
"""Test that the existing "+" delimiter split is
unaffected by the new rules."""
self.assertEqual(
build_db.parse_artist_credit("Dan + Shay"),
[("Dan", Role.PRIMARY), ("Shay", Role.PRIMARY)])
class TestBuildDB(unittest.TestCase):
"""Test cases for the working store build."""
@@ -194,7 +338,8 @@ class TestBuildDB(unittest.TestCase):
def test_dedup_credit_variant(self) -> None:
"""Test that a credit variant listed in
``CANONICAL_ARTIST_CREDITS`` merges into one song, storing
the canonical credit."""
the canonical credit, with the canonical artist spelling
applied to the artist listed in ``CANONICAL_ARTIST_NAMES``."""
self.__write_chart(
"year,rank,title,artist\n"
"2016,1,Eastside,\"benny blanco, Halsey & Khalid\"\n"
@@ -217,7 +362,72 @@ class TestBuildDB(unittest.TestCase):
[(2016, 1), (2017, 1)])
self.assertEqual(
[x.artist.name for x in songs[0].song_artists],
["Benny Blanco", "Halsey", "Khalid"])
["benny blanco", "Halsey", "Khalid"])
def test_dedup_artist_name_case_variant(self) -> None:
"""Test that an artist name case variant merges into one
artist, keeping the first-seen spelling."""
self.__write_chart(
"year,rank,title,artist\n"
"2016,1,Song One,Marshmello\n"
"2016,2,filler,Filler Artist\n"
"2017,1,Song Two,marshmello\n"
"2017,2,filler,Filler Artist\n")
status: int
stderr: str
status, stderr = self.__run_build()
self.assertEqual(status, 0)
session: Session = self.__session()
artists: list[Artist] = list(session.scalars(
sa.select(Artist).where(Artist.name.ilike("marsh%"))))
self.assertEqual(len(artists), 1)
self.assertEqual(artists[0].name, "Marshmello")
self.assertEqual(
{x.title for x in
{y.song for y in artists[0].song_artists}},
{"Song One", "Song Two"})
def test_canonical_artist_name_restored(self) -> None:
"""Test that a canonical spelling is stored even for a
single, non-canonically spelled occurrence."""
self.__write_chart(
"year,rank,title,artist\n"
"2016,1,Beggin',Maneskin\n"
"2016,2,filler,Filler Artist\n"
"2017,1,filler2,Filler Artist Two\n"
"2017,2,filler3,Filler Artist Three\n")
status: int
stderr: str
status, stderr = self.__run_build()
self.assertEqual(status, 0)
session: Session = self.__session()
artist: Artist | None = session.scalar(
sa.select(Artist).where(Artist.name.ilike("m%nesk%")))
assert artist is not None
self.assertEqual(artist.name, "Måneskin")
def test_canonical_ye_resolves_to_kanye_west(self) -> None:
"""Test that a "Ye" credit and a "Kanye West" credit merge
into a single artist row named "Kanye West", credited on
both songs."""
self.__write_chart(
"year,rank,title,artist\n"
"2016,1,Song One,Ye\n"
"2016,2,Song Two,Kanye West\n"
"2017,1,filler2,Filler Artist Two\n"
"2017,2,filler3,Filler Artist Three\n")
status: int
stderr: str
status, stderr = self.__run_build()
self.assertEqual(status, 0)
session: Session = self.__session()
artists: list[Artist] = list(session.scalars(
sa.select(Artist).where(Artist.name == "Kanye West")))
self.assertEqual(len(artists), 1)
self.assertEqual(
{x.title for x in
{y.song for y in artists[0].song_artists}},
{"Song One", "Song Two"})
def test_first_run_on_fresh_store(self) -> None:
"""Test that a build on a fresh store creates the tables."""