From ae0e9d0a082ce91e275e61fda9bf7f3dcbbc7589 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BE=9D=E7=91=AA=E8=B2=93?= Date: Sun, 2 Aug 2026 13:33:30 +0800 Subject: [PATCH] Split artist credits with documented exceptions and normalize artist names --- docs/decision_log.md | 8 + tools/src/pop_fem_audit_tools/build_db.py | 170 ++++++++++++++--- tools/tests/test_build_db.py | 214 +++++++++++++++++++++- 3 files changed, 369 insertions(+), 23 deletions(-) diff --git a/docs/decision_log.md b/docs/decision_log.md index 4e9c08e..e1f6fb6 100644 --- a/docs/decision_log.md +++ b/docs/decision_log.md @@ -133,3 +133,11 @@ 因署名字串寫法不同而分裂者，以明示對照表合併；歌曲身分＝ （原始歌名，對照後署名字串），不解析署名——拆解機器不進 歌曲身分判定，對照表即完整可稽核清單。 +- **歌手名單拆解規則定案**：署名列出成員的團體只留成員 + （冒號與括號兩型）；名字內含連接詞的單一藝人入保護名單； + 規則無法表達的個案入例外表；人名連寫的二人組拆成個人； + 僅以團名掛名的團體維持單一實體，屬性留待 fetch-artists + 與人工 overrides。完整清單見 `build_db.py`。 +- **歌手名正規化**：歌手名大小寫不敏感歸併、存首見拼法； + 正名表（casefold 鍵→官方拼法）存官方拼法，含跨拼法合併 + ye→Kanye West。 diff --git a/tools/src/pop_fem_audit_tools/build_db.py b/tools/src/pop_fem_audit_tools/build_db.py index 58c424f..75fa1e7 100644 --- a/tools/src/pop_fem_audit_tools/build_db.py +++ b/tools/src/pop_fem_audit_tools/build_db.py @@ -29,7 +29,15 @@ credit, the credit canonicalized through ``CANONICAL_ARTIST_CREDITS``; a credit listed there collapses onto the same song as its canonical form, and the stored artist credit is always the canonical form. Artist deduplication is by the -exact parsed artist name, as printed on the chart. +identity key resolved from the parsed artist name (see +`resolve_artist_identity`): the case-folded name, or, when that +case-folded name is listed in ``CANONICAL_ARTIST_NAMES``, the +case-folded canonical spelling, so letter-case variants and +alternate spellings mapped to the same canonical name all +collapse onto a single artist row. The stored artist name is the +first-seen spelling, except for the names listed in +``CANONICAL_ARTIST_NAMES``, which always store the canonical +spelling regardless of which variant is seen first. """ import argparse import csv @@ -68,13 +76,67 @@ FEATURING_PATTERN: re.Pattern[str] = re.compile( r" featuring | feat\. ", re.IGNORECASE) """The pattern splitting the primary and featured sides.""" DELIMITER_PATTERN: re.Pattern[str] = re.compile( - r", | & | \+ |(?i: and | x | with )") + r", | & | \+ | / |(?i: and | x | with )") """The pattern splitting the artist names within a side.""" +COLON_PATTERN: re.Pattern[str] = re.compile(r": ") +"""The pattern separating a group prefix from its members in a +": " credit.""" +PAREN_MEMBERS_PATTERN: re.Pattern[str] = re.compile( + r"^.+ \((?P.+)\)$") +"""The pattern separating a group name from its members in a +" ()" credit spanning the whole credit.""" +DUET_WITH_PATTERN: re.Pattern[str] = re.compile( + r" Duet With ", re.IGNORECASE) +"""The pattern normalizing the "Duet With" co-billing connector +to the plain "with" delimiter.""" +PROTECTED_ARTIST_NAMES: tuple[str, ...] = ( + "Tyler, The Creator", + "Lil Nas X", + "Tones And I", +) +"""The exact artist names guarded from the delimiter splitting, +because each contains a delimiter word or punctuation as part of +the name itself.""" +EXCEPTION_CREDITS: dict[str, list[tuple[str, Role]]] = { + "SpotemGottem Featuring Pooh Shiesty Or DaBaby": [ + ("SpotemGottem", Role.PRIMARY), + ("Pooh Shiesty", Role.FEATURED), + ("DaBaby", Role.FEATURED), + ], + "THE SCOTTS, Travis Scott & Kid Cudi": [ + ("Travis Scott", Role.PRIMARY), + ("Kid Cudi", Role.PRIMARY), + ], + "Drake Featuring The Throne": [ + ("Drake", Role.PRIMARY), + ("Jay Z", Role.FEATURED), + ("Kanye West", Role.FEATURED), + ], +} +"""The single-credit exceptions parsed by an explicit lookup +rather than by the general rules, because the credit text alone +does not spell out the correct member split.""" CANONICAL_ARTIST_CREDITS: dict[str, str] = { "benny blanco, Halsey & Khalid": "Benny Blanco, Halsey & Khalid", } """The canonical artist credit spellings, keyed by a variant credit string.""" +CANONICAL_ARTIST_NAMES: dict[str, str] = { + "beyonce": "Beyoncé", + "5 seconds of summer": "5 Seconds of Summer", + "a boogie wit da hoodie": "A Boogie wit da Hoodie", + "benny blanco": "benny blanco", + "blackbear": "blackbear", + "chance the rapper": "Chance the Rapper", + "xxxtentacion": "XXXTENTACION", + "maneskin": "Måneskin", + "rose": "ROSÉ", + "mo": "MØ", + "wizkid": "Wizkid", + "ye": "Kanye West", +} +"""The canonical artist spellings, keyed by the case-folded +identity.""" class BuildError(Exception): @@ -109,22 +171,55 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: def parse_artist_credit(credit: str) -> list[tuple[str, Role]]: """Parse a combined artist credit into artists and roles. - The credit splits into a primary side and a featured side on - the word "featuring" or "feat.", case-insensitively; without - them, every artist is primary. Each side splits into artist - names on the delimiters ", ", " & ", " + " (literally) and - " and ", " x ", " with " (case-insensitively). + A credit listed in ``EXCEPTION_CREDITS`` is looked up verbatim, + because its correct split is not derivable from the credit + text alone. Otherwise the credit first reduces to an + effective credit: a ": " prefix (split at the + first ": ") drops the group and keeps the members; failing + that, a " ()" suffix spanning the whole credit + drops the group and keeps the members. The "Duet With" + connector, case-insensitively, then normalizes to "with". The + effective credit splits into a primary side and a featured + side on the word "featuring" or "feat.", case-insensitively; + without them, every artist is primary. Each side splits into + artist names on the delimiters ", ", " & ", " + ", " / " + (literally) and " and ", " x ", " with " (case-insensitively), + except for the names listed in ``PROTECTED_ARTIST_NAMES``, + which are never split even though each contains a delimiter + word or punctuation. Known limitation: a compound act name that contains one of the - delimiters is over-split; such cases are corrected later via - the human override layer. + delimiters, other than the protected names, is over-split; + such cases are corrected later via the human override layer. :param credit: The combined artist credit string. :return: The (name, role) pairs in credit order, primary side first, with the role ``Role.PRIMARY`` or ``Role.FEATURED``. """ - sides: list[str] = FEATURING_PATTERN.split(credit, maxsplit=1) + if credit in EXCEPTION_CREDITS: + return list(EXCEPTION_CREDITS[credit]) + effective: str = credit + colon_match: re.Match[str] | None = COLON_PATTERN.search( + effective) + if colon_match is not None: + effective = effective[colon_match.end():] + else: + paren_match: re.Match[str] | None = \ + PAREN_MEMBERS_PATTERN.match(effective) + if paren_match is not None: + effective = paren_match.group("members") + effective = DUET_WITH_PATTERN.sub(" with ", effective) + placeholders: dict[str, str] = {} + index: int + protected: str + for index, protected in enumerate(PROTECTED_ARTIST_NAMES): + if protected in effective: + placeholder: str = f"{index}" + placeholders[placeholder] = protected + effective = effective.replace(protected, placeholder) + sides: list[str] = FEATURING_PATTERN.split( + effective, maxsplit=1) pairs: list[tuple[str, Role]] = [] role: Role side: str @@ -132,6 +227,10 @@ def parse_artist_credit(credit: str) -> list[tuple[str, Role]]: token: str for token in DELIMITER_PATTERN.split(side): name: str = token.strip() + placeholder = "" + original: str + for placeholder, original in placeholders.items(): + name = name.replace(placeholder, original) if name != "": pairs.append((name, role)) return pairs @@ -153,21 +252,46 @@ def song_identity(title: str, credit: str) -> tuple[str, str]: return title, CANONICAL_ARTIST_CREDITS.get(credit, credit) +def resolve_artist_identity(name: str) -> tuple[str, str]: + """Resolve the dedup key and the stored spelling of a name. + + The name's case-folded form is looked up in + ``CANONICAL_ARTIST_NAMES`` first; when it is listed there, the + dedup key is the canonical spelling case-folded and the stored + spelling is the canonical spelling, so every variant of the + name, canonical or not, resolves to the same identity. + Otherwise the dedup key is the name case-folded and the stored + spelling is the given name. + + :param name: An artist name, as parsed from a credit. + :return: A tuple of the dedup key and the stored spelling. + """ + folded: str = name.casefold() + canonical: str | None = CANONICAL_ARTIST_NAMES.get(folded) + if canonical is not None: + return canonical.casefold(), canonical + return folded, name + + def create_song(session: Session, song_id: int, title: str, credit: str, artists: dict[str, Artist]) -> Song: """Create a song with its parsed artist credits. - The song takes the given ID. A newly created artist takes - the ID following the known artists, in credit order. An - artist duplicated within the credit is kept only at its - first occurrence, with a warning to the standard error. + The song takes the given ID. An artist parsed out of the + credit is matched against the known artists by its identity + key (see `resolve_artist_identity`); a newly seen one takes + the ID following the known artists, keyed by its identity key, + and its stored name is the resolved stored spelling. An + artist duplicated within the credit, by its identity key, is + kept only at its first occurrence, with a warning to the + standard error. :param session: The database session. :param song_id: The song ID to assign. :param title: The song title. :param credit: The combined artist credit string. - :param artists: The known artists by name, updated with the - newly created ones as an observable side effect. + :param artists: The known artists by identity key, updated + with the newly created ones as an observable side effect. :return: The created song, added to the session. """ song: Song = Song(id=song_id, title=title, @@ -178,14 +302,18 @@ def create_song(session: Session, song_id: int, title: str, name: str role: Role for name, role in parse_artist_credit(credit): - if name in seen: + key: str + stored_name: str + key, stored_name = resolve_artist_identity(name) + if key in seen: print(f"warning: {credit}: duplicated artist" f" \"{name}\"", file=sys.stderr) continue - seen.add(name) - if name not in artists: - artists[name] = Artist(id=len(artists) + 1, name=name) - session.add(SongArtist(song=song, artist=artists[name], + seen.add(key) + if key not in artists: + artists[key] = Artist(id=len(artists) + 1, + name=stored_name) + session.add(SongArtist(song=song, artist=artists[key], role=role, position=position)) position += 1 return song diff --git a/tools/tests/test_build_db.py b/tools/tests/test_build_db.py index e219d62..fc66831 100644 --- a/tools/tests/test_build_db.py +++ b/tools/tests/test_build_db.py @@ -90,6 +90,150 @@ class TestParseArtistCredit(unittest.TestCase): [("24kGoldn", Role.PRIMARY), ("iann dior", Role.FEATURED)]) + def test_colon_prefix_group(self) -> None: + """Test that a colon-prefixed group name is dropped.""" + self.assertEqual( + build_db.parse_artist_credit( + "¥$: Ye & Ty Dolla $ign Featuring Rich The Kid" + " & Playboi Carti"), + [("Ye", Role.PRIMARY), + ("Ty Dolla $ign", Role.PRIMARY), + ("Rich The Kid", Role.FEATURED), + ("Playboi Carti", Role.FEATURED)]) + + def test_colon_prefix_with_ampersand_in_prefix(self) -> None: + """Test a colon prefix that itself contains "&".""" + self.assertEqual( + build_db.parse_artist_credit( + "Rumi & JINU: EJAE & Andrew Choi"), + [("EJAE", Role.PRIMARY), + ("Andrew Choi", Role.PRIMARY)]) + + def test_colon_prefix_comma_list(self) -> None: + """Test a colon-prefixed comma-separated member list.""" + self.assertEqual( + build_db.parse_artist_credit( + "HUNTR/X: EJAE, Audrey Nuna & REI AMI"), + [("EJAE", Role.PRIMARY), + ("Audrey Nuna", Role.PRIMARY), + ("REI AMI", Role.PRIMARY)]) + + def test_colon_prefix_five_members(self) -> None: + """Test a colon-prefixed five-member list.""" + self.assertEqual( + build_db.parse_artist_credit( + "Saja Boys: Andrew Choi, Neckwav, Danny Chung," + " Kevin Woo & samUIL Lee"), + [("Andrew Choi", Role.PRIMARY), + ("Neckwav", Role.PRIMARY), + ("Danny Chung", Role.PRIMARY), + ("Kevin Woo", Role.PRIMARY), + ("samUIL Lee", Role.PRIMARY)]) + + def test_colon_prefix_duo(self) -> None: + """Test a colon-prefixed two-member list.""" + self.assertEqual( + build_db.parse_artist_credit( + "THE ANXIETY: WILLOW & Tyler Cole"), + [("WILLOW", Role.PRIMARY), + ("Tyler Cole", Role.PRIMARY)]) + + def test_parenthesized_members(self) -> None: + """Test that a parenthesized member list replaces the + group name spanning the whole credit.""" + self.assertEqual( + build_db.parse_artist_credit( + "Silk Sonic (Bruno Mars & Anderson .Paak)"), + [("Bruno Mars", Role.PRIMARY), + ("Anderson .Paak", Role.PRIMARY)]) + + def test_duet_with_connector(self) -> None: + """Test that "Duet With" is a co-billing connector like + "with", dropping the word "Duet" entirely.""" + self.assertEqual( + build_db.parse_artist_credit( + "Blake Shelton Duet With Gwen Stefani"), + [("Blake Shelton", Role.PRIMARY), + ("Gwen Stefani", Role.PRIMARY)]) + self.assertEqual( + build_db.parse_artist_credit( + "Keith Urban Duet With P!nk"), + [("Keith Urban", Role.PRIMARY), + ("P!nk", Role.PRIMARY)]) + + def test_slash_delimiter(self) -> None: + """Test the " / " co-billing delimiter.""" + self.assertEqual( + build_db.parse_artist_credit( + "Cole Swindell / Lainey Wilson"), + [("Cole Swindell", Role.PRIMARY), + ("Lainey Wilson", Role.PRIMARY)]) + self.assertEqual( + build_db.parse_artist_credit("Zayn / Taylor Swift"), + [("Zayn", Role.PRIMARY), + ("Taylor Swift", Role.PRIMARY)]) + + def test_protected_name_lil_nas_x(self) -> None: + """Test that "Lil Nas X" is guarded from the " x " + delimiter split.""" + self.assertEqual( + build_db.parse_artist_credit("Lil Nas X & Jack Harlow"), + [("Lil Nas X", Role.PRIMARY), + ("Jack Harlow", Role.PRIMARY)]) + + def test_protected_name_tyler_the_creator(self) -> None: + """Test that "Tyler, The Creator" is guarded from the + comma delimiter split.""" + self.assertEqual( + build_db.parse_artist_credit( + "Tyler, The Creator Featuring GloRilla, Sexyy Red" + " & Lil Wayne"), + [("Tyler, The Creator", Role.PRIMARY), + ("GloRilla", Role.FEATURED), + ("Sexyy Red", Role.FEATURED), + ("Lil Wayne", Role.FEATURED)]) + + def test_protected_name_tones_and_i(self) -> None: + """Test that "Tones And I" is guarded from the " and " + delimiter split.""" + self.assertEqual( + build_db.parse_artist_credit("Tones And I"), + [("Tones And I", Role.PRIMARY)]) + + def test_exception_spotemgottem(self) -> None: + """Test the SpotemGottem exception-table credit.""" + self.assertEqual( + build_db.parse_artist_credit( + "SpotemGottem Featuring Pooh Shiesty Or DaBaby"), + [("SpotemGottem", Role.PRIMARY), + ("Pooh Shiesty", Role.FEATURED), + ("DaBaby", Role.FEATURED)]) + + def test_exception_the_scotts(self) -> None: + """Test the THE SCOTTS exception-table credit.""" + self.assertEqual( + build_db.parse_artist_credit( + "THE SCOTTS, Travis Scott & Kid Cudi"), + [("Travis Scott", Role.PRIMARY), + ("Kid Cudi", Role.PRIMARY)]) + + def test_exception_drake_featuring_the_throne(self) -> None: + """Test the Drake Featuring The Throne exception-table + credit.""" + self.assertEqual( + build_db.parse_artist_credit( + "Drake Featuring The Throne"), + [("Drake", Role.PRIMARY), + ("Jay Z", Role.FEATURED), + ("Kanye West", Role.FEATURED)]) + + def test_plus_delimiter_unaffected(self) -> None: + """Test that the existing "+" delimiter split is + unaffected by the new rules.""" + self.assertEqual( + build_db.parse_artist_credit("Dan + Shay"), + [("Dan", Role.PRIMARY), ("Shay", Role.PRIMARY)]) + class TestBuildDB(unittest.TestCase): """Test cases for the working store build.""" @@ -194,7 +338,8 @@ class TestBuildDB(unittest.TestCase): def test_dedup_credit_variant(self) -> None: """Test that a credit variant listed in ``CANONICAL_ARTIST_CREDITS`` merges into one song, storing - the canonical credit.""" + the canonical credit, with the canonical artist spelling + applied to the artist listed in ``CANONICAL_ARTIST_NAMES``.""" self.__write_chart( "year,rank,title,artist\n" "2016,1,Eastside,\"benny blanco, Halsey & Khalid\"\n" @@ -217,7 +362,72 @@ class TestBuildDB(unittest.TestCase): [(2016, 1), (2017, 1)]) self.assertEqual( [x.artist.name for x in songs[0].song_artists], - ["Benny Blanco", "Halsey", "Khalid"]) + ["benny blanco", "Halsey", "Khalid"]) + + def test_dedup_artist_name_case_variant(self) -> None: + """Test that an artist name case variant merges into one + artist, keeping the first-seen spelling.""" + self.__write_chart( + "year,rank,title,artist\n" + "2016,1,Song One,Marshmello\n" + "2016,2,filler,Filler Artist\n" + "2017,1,Song Two,marshmello\n" + "2017,2,filler,Filler Artist\n") + status: int + stderr: str + status, stderr = self.__run_build() + self.assertEqual(status, 0) + session: Session = self.__session() + artists: list[Artist] = list(session.scalars( + sa.select(Artist).where(Artist.name.ilike("marsh%")))) + self.assertEqual(len(artists), 1) + self.assertEqual(artists[0].name, "Marshmello") + self.assertEqual( + {x.title for x in + {y.song for y in artists[0].song_artists}}, + {"Song One", "Song Two"}) + + def test_canonical_artist_name_restored(self) -> None: + """Test that a canonical spelling is stored even for a + single, non-canonically spelled occurrence.""" + self.__write_chart( + "year,rank,title,artist\n" + "2016,1,Beggin',Maneskin\n" + "2016,2,filler,Filler Artist\n" + "2017,1,filler2,Filler Artist Two\n" + "2017,2,filler3,Filler Artist Three\n") + status: int + stderr: str + status, stderr = self.__run_build() + self.assertEqual(status, 0) + session: Session = self.__session() + artist: Artist | None = session.scalar( + sa.select(Artist).where(Artist.name.ilike("m%nesk%"))) + assert artist is not None + self.assertEqual(artist.name, "Måneskin") + + def test_canonical_ye_resolves_to_kanye_west(self) -> None: + """Test that a "Ye" credit and a "Kanye West" credit merge + into a single artist row named "Kanye West", credited on + both songs.""" + self.__write_chart( + "year,rank,title,artist\n" + "2016,1,Song One,Ye\n" + "2016,2,Song Two,Kanye West\n" + "2017,1,filler2,Filler Artist Two\n" + "2017,2,filler3,Filler Artist Three\n") + status: int + stderr: str + status, stderr = self.__run_build() + self.assertEqual(status, 0) + session: Session = self.__session() + artists: list[Artist] = list(session.scalars( + sa.select(Artist).where(Artist.name == "Kanye West"))) + self.assertEqual(len(artists), 1) + self.assertEqual( + {x.title for x in + {y.song for y in artists[0].song_artists}}, + {"Song One", "Song Two"}) def test_first_run_on_fresh_store(self) -> None: """Test that a build on a fresh store creates the tables."""