Split artist credits with documented exceptions and normalize artist names

docs/decision_log.md

@@ -133,3 +133,11 @@
   因署名字串寫法不同而分裂者，以明示對照表合併；歌曲身分＝
   （原始歌名，對照後署名字串），不解析署名——拆解機器不進
   歌曲身分判定，對照表即完整可稽核清單。
+- **歌手名單拆解規則定案**：署名列出成員的團體只留成員
+  （冒號與括號兩型）；名字內含連接詞的單一藝人入保護名單；
+  規則無法表達的個案入例外表；人名連寫的二人組拆成個人；
+  僅以團名掛名的團體維持單一實體，屬性留待 fetch-artists
+  與人工 overrides。完整清單見 `build_db.py`。
+- **歌手名正規化**：歌手名大小寫不敏感歸併、存首見拼法；
+  正名表（casefold 鍵→官方拼法）存官方拼法，含跨拼法合併
+  ye→Kanye West。

tools/src/pop_fem_audit_tools/build_db.py

@@ -29,7 +29,15 @@ credit, the credit canonicalized through
 ``CANONICAL_ARTIST_CREDITS``; a credit listed there collapses onto
 the same song as its canonical form, and the stored artist credit
 is always the canonical form.  Artist deduplication is by the
-exact parsed artist name, as printed on the chart.
+identity key resolved from the parsed artist name (see
+`resolve_artist_identity`): the case-folded name, or, when that
+case-folded name is listed in ``CANONICAL_ARTIST_NAMES``, the
+case-folded canonical spelling, so letter-case variants and
+alternate spellings mapped to the same canonical name all
+collapse onto a single artist row.  The stored artist name is the
+first-seen spelling, except for the names listed in
+``CANONICAL_ARTIST_NAMES``, which always store the canonical
+spelling regardless of which variant is seen first.
 """
 import argparse
 import csv
@@ -68,13 +76,67 @@ FEATURING_PATTERN: re.Pattern[str] = re.compile(
     r" featuring | feat\. ", re.IGNORECASE)
 """The pattern splitting the primary and featured sides."""
 DELIMITER_PATTERN: re.Pattern[str] = re.compile(
-    r", | & | \+ |(?i: and | x | with )")
+    r", | & | \+ | / |(?i: and | x | with )")
 """The pattern splitting the artist names within a side."""
+COLON_PATTERN: re.Pattern[str] = re.compile(r": ")
+"""The pattern separating a group prefix from its members in a
+"<group>: <members>" credit."""
+PAREN_MEMBERS_PATTERN: re.Pattern[str] = re.compile(
+    r"^.+ \((?P<members>.+)\)$")
+"""The pattern separating a group name from its members in a
+"<group> (<members>)" credit spanning the whole credit."""
+DUET_WITH_PATTERN: re.Pattern[str] = re.compile(
+    r" Duet With ", re.IGNORECASE)
+"""The pattern normalizing the "Duet With" co-billing connector
+to the plain "with" delimiter."""
+PROTECTED_ARTIST_NAMES: tuple[str, ...] = (
+    "Tyler, The Creator",
+    "Lil Nas X",
+    "Tones And I",
+)
+"""The exact artist names guarded from the delimiter splitting,
+because each contains a delimiter word or punctuation as part of
+the name itself."""
+EXCEPTION_CREDITS: dict[str, list[tuple[str, Role]]] = {
+    "SpotemGottem Featuring Pooh Shiesty Or DaBaby": [
+        ("SpotemGottem", Role.PRIMARY),
+        ("Pooh Shiesty", Role.FEATURED),
+        ("DaBaby", Role.FEATURED),
+    ],
+    "THE SCOTTS, Travis Scott & Kid Cudi": [
+        ("Travis Scott", Role.PRIMARY),
+        ("Kid Cudi", Role.PRIMARY),
+    ],
+    "Drake Featuring The Throne": [
+        ("Drake", Role.PRIMARY),
+        ("Jay Z", Role.FEATURED),
+        ("Kanye West", Role.FEATURED),
+    ],
+}
+"""The single-credit exceptions parsed by an explicit lookup
+rather than by the general rules, because the credit text alone
+does not spell out the correct member split."""
 CANONICAL_ARTIST_CREDITS: dict[str, str] = {
     "benny blanco, Halsey & Khalid": "Benny Blanco, Halsey & Khalid",
 }
 """The canonical artist credit spellings, keyed by a variant
 credit string."""
+CANONICAL_ARTIST_NAMES: dict[str, str] = {
+    "beyonce": "Beyoncé",
+    "5 seconds of summer": "5 Seconds of Summer",
+    "a boogie wit da hoodie": "A Boogie wit da Hoodie",
+    "benny blanco": "benny blanco",
+    "blackbear": "blackbear",
+    "chance the rapper": "Chance the Rapper",
+    "xxxtentacion": "XXXTENTACION",
+    "maneskin": "Måneskin",
+    "rose": "ROSÉ",
+    "mo": "MØ",
+    "wizkid": "Wizkid",
+    "ye": "Kanye West",
+}
+"""The canonical artist spellings, keyed by the case-folded
+identity."""
 
 
 class BuildError(Exception):
@@ -109,22 +171,55 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
 def parse_artist_credit(credit: str) -> list[tuple[str, Role]]:
     """Parse a combined artist credit into artists and roles.
 
-    The credit splits into a primary side and a featured side on
+    A credit listed in ``EXCEPTION_CREDITS`` is looked up verbatim,
-    the word "featuring" or "feat.", case-insensitively; without
+    because its correct split is not derivable from the credit
-    them, every artist is primary.  Each side splits into artist
+    text alone.  Otherwise the credit first reduces to an
-    names on the delimiters ", ", " & ", " + " (literally) and
+    effective credit: a "<group>: <members>" prefix (split at the
-    " and ", " x ", " with " (case-insensitively).
+    first ": ") drops the group and keeps the members; failing
+    that, a "<group> (<members>)" suffix spanning the whole credit
+    drops the group and keeps the members.  The "Duet With"
+    connector, case-insensitively, then normalizes to "with".  The
+    effective credit splits into a primary side and a featured
+    side on the word "featuring" or "feat.", case-insensitively;
+    without them, every artist is primary.  Each side splits into
+    artist names on the delimiters ", ", " & ", " + ", " / "
+    (literally) and " and ", " x ", " with " (case-insensitively),
+    except for the names listed in ``PROTECTED_ARTIST_NAMES``,
+    which are never split even though each contains a delimiter
+    word or punctuation.
 
     Known limitation: a compound act name that contains one of the
-    delimiters is over-split; such cases are corrected later via
+    delimiters, other than the protected names, is over-split;
-    the human override layer.
+    such cases are corrected later via the human override layer.
 
     :param credit: The combined artist credit string.
     :return: The (name, role) pairs in credit order, primary side
         first, with the role ``Role.PRIMARY`` or
         ``Role.FEATURED``.
     """
-    sides: list[str] = FEATURING_PATTERN.split(credit, maxsplit=1)
+    if credit in EXCEPTION_CREDITS:
+        return list(EXCEPTION_CREDITS[credit])
+    effective: str = credit
+    colon_match: re.Match[str] | None = COLON_PATTERN.search(
+        effective)
+    if colon_match is not None:
+        effective = effective[colon_match.end():]
+    else:
+        paren_match: re.Match[str] | None = \
+            PAREN_MEMBERS_PATTERN.match(effective)
+        if paren_match is not None:
+            effective = paren_match.group("members")
+    effective = DUET_WITH_PATTERN.sub(" with ", effective)
+    placeholders: dict[str, str] = {}
+    index: int
+    protected: str
+    for index, protected in enumerate(PROTECTED_ARTIST_NAMES):
+        if protected in effective:
+            placeholder: str = f"{index}"
+            placeholders[placeholder] = protected
+            effective = effective.replace(protected, placeholder)
+    sides: list[str] = FEATURING_PATTERN.split(
+        effective, maxsplit=1)
     pairs: list[tuple[str, Role]] = []
     role: Role
     side: str
@@ -132,6 +227,10 @@ def parse_artist_credit(credit: str) -> list[tuple[str, Role]]:
         token: str
         for token in DELIMITER_PATTERN.split(side):
             name: str = token.strip()
+            placeholder = ""
+            original: str
+            for placeholder, original in placeholders.items():
+                name = name.replace(placeholder, original)
             if name != "":
                 pairs.append((name, role))
     return pairs
@@ -153,21 +252,46 @@ def song_identity(title: str, credit: str) -> tuple[str, str]:
     return title, CANONICAL_ARTIST_CREDITS.get(credit, credit)
 
 
+def resolve_artist_identity(name: str) -> tuple[str, str]:
+    """Resolve the dedup key and the stored spelling of a name.
+
+    The name's case-folded form is looked up in
+    ``CANONICAL_ARTIST_NAMES`` first; when it is listed there, the
+    dedup key is the canonical spelling case-folded and the stored
+    spelling is the canonical spelling, so every variant of the
+    name, canonical or not, resolves to the same identity.
+    Otherwise the dedup key is the name case-folded and the stored
+    spelling is the given name.
+
+    :param name: An artist name, as parsed from a credit.
+    :return: A tuple of the dedup key and the stored spelling.
+    """
+    folded: str = name.casefold()
+    canonical: str | None = CANONICAL_ARTIST_NAMES.get(folded)
+    if canonical is not None:
+        return canonical.casefold(), canonical
+    return folded, name
+
+
 def create_song(session: Session, song_id: int, title: str,
                 credit: str, artists: dict[str, Artist]) -> Song:
     """Create a song with its parsed artist credits.
 
-    The song takes the given ID.  A newly created artist takes
+    The song takes the given ID.  An artist parsed out of the
-    the ID following the known artists, in credit order.  An
+    credit is matched against the known artists by its identity
-    artist duplicated within the credit is kept only at its
+    key (see `resolve_artist_identity`); a newly seen one takes
-    first occurrence, with a warning to the standard error.
+    the ID following the known artists, keyed by its identity key,
+    and its stored name is the resolved stored spelling.  An
+    artist duplicated within the credit, by its identity key, is
+    kept only at its first occurrence, with a warning to the
+    standard error.
 
     :param session: The database session.
     :param song_id: The song ID to assign.
     :param title: The song title.
     :param credit: The combined artist credit string.
-    :param artists: The known artists by name, updated with the
+    :param artists: The known artists by identity key, updated
-        newly created ones as an observable side effect.
+        with the newly created ones as an observable side effect.
     :return: The created song, added to the session.
     """
     song: Song = Song(id=song_id, title=title,
@@ -178,14 +302,18 @@ def create_song(session: Session, song_id: int, title: str,
     name: str
     role: Role
     for name, role in parse_artist_credit(credit):
-        if name in seen:
+        key: str
+        stored_name: str
+        key, stored_name = resolve_artist_identity(name)
+        if key in seen:
             print(f"warning: {credit}: duplicated artist"
                   f" \"{name}\"", file=sys.stderr)
             continue
-        seen.add(name)
+        seen.add(key)
-        if name not in artists:
+        if key not in artists:
-            artists[name] = Artist(id=len(artists) + 1, name=name)
+            artists[key] = Artist(id=len(artists) + 1,
-        session.add(SongArtist(song=song, artist=artists[name],
+                                  name=stored_name)
+        session.add(SongArtist(song=song, artist=artists[key],
                                role=role, position=position))
         position += 1
     return song

tools/tests/test_build_db.py

@@ -90,6 +90,150 @@ class TestParseArtistCredit(unittest.TestCase):
             [("24kGoldn", Role.PRIMARY),
              ("iann dior", Role.FEATURED)])
 
+    def test_colon_prefix_group(self) -> None:
+        """Test that a colon-prefixed group name is dropped."""
+        self.assertEqual(
+            build_db.parse_artist_credit(
+                "¥$: Ye & Ty Dolla $ign Featuring Rich The Kid"
+                " & Playboi Carti"),
+            [("Ye", Role.PRIMARY),
+             ("Ty Dolla $ign", Role.PRIMARY),
+             ("Rich The Kid", Role.FEATURED),
+             ("Playboi Carti", Role.FEATURED)])
+
+    def test_colon_prefix_with_ampersand_in_prefix(self) -> None:
+        """Test a colon prefix that itself contains "&"."""
+        self.assertEqual(
+            build_db.parse_artist_credit(
+                "Rumi & JINU: EJAE & Andrew Choi"),
+            [("EJAE", Role.PRIMARY),
+             ("Andrew Choi", Role.PRIMARY)])
+
+    def test_colon_prefix_comma_list(self) -> None:
+        """Test a colon-prefixed comma-separated member list."""
+        self.assertEqual(
+            build_db.parse_artist_credit(
+                "HUNTR/X: EJAE, Audrey Nuna & REI AMI"),
+            [("EJAE", Role.PRIMARY),
+             ("Audrey Nuna", Role.PRIMARY),
+             ("REI AMI", Role.PRIMARY)])
+
+    def test_colon_prefix_five_members(self) -> None:
+        """Test a colon-prefixed five-member list."""
+        self.assertEqual(
+            build_db.parse_artist_credit(
+                "Saja Boys: Andrew Choi, Neckwav, Danny Chung,"
+                " Kevin Woo & samUIL Lee"),
+            [("Andrew Choi", Role.PRIMARY),
+             ("Neckwav", Role.PRIMARY),
+             ("Danny Chung", Role.PRIMARY),
+             ("Kevin Woo", Role.PRIMARY),
+             ("samUIL Lee", Role.PRIMARY)])
+
+    def test_colon_prefix_duo(self) -> None:
+        """Test a colon-prefixed two-member list."""
+        self.assertEqual(
+            build_db.parse_artist_credit(
+                "THE ANXIETY: WILLOW & Tyler Cole"),
+            [("WILLOW", Role.PRIMARY),
+             ("Tyler Cole", Role.PRIMARY)])
+
+    def test_parenthesized_members(self) -> None:
+        """Test that a parenthesized member list replaces the
+        group name spanning the whole credit."""
+        self.assertEqual(
+            build_db.parse_artist_credit(
+                "Silk Sonic (Bruno Mars & Anderson .Paak)"),
+            [("Bruno Mars", Role.PRIMARY),
+             ("Anderson .Paak", Role.PRIMARY)])
+
+    def test_duet_with_connector(self) -> None:
+        """Test that "Duet With" is a co-billing connector like
+        "with", dropping the word "Duet" entirely."""
+        self.assertEqual(
+            build_db.parse_artist_credit(
+                "Blake Shelton Duet With Gwen Stefani"),
+            [("Blake Shelton", Role.PRIMARY),
+             ("Gwen Stefani", Role.PRIMARY)])
+        self.assertEqual(
+            build_db.parse_artist_credit(
+                "Keith Urban Duet With P!nk"),
+            [("Keith Urban", Role.PRIMARY),
+             ("P!nk", Role.PRIMARY)])
+
+    def test_slash_delimiter(self) -> None:
+        """Test the " / " co-billing delimiter."""
+        self.assertEqual(
+            build_db.parse_artist_credit(
+                "Cole Swindell / Lainey Wilson"),
+            [("Cole Swindell", Role.PRIMARY),
+             ("Lainey Wilson", Role.PRIMARY)])
+        self.assertEqual(
+            build_db.parse_artist_credit("Zayn / Taylor Swift"),
+            [("Zayn", Role.PRIMARY),
+             ("Taylor Swift", Role.PRIMARY)])
+
+    def test_protected_name_lil_nas_x(self) -> None:
+        """Test that "Lil Nas X" is guarded from the " x "
+        delimiter split."""
+        self.assertEqual(
+            build_db.parse_artist_credit("Lil Nas X & Jack Harlow"),
+            [("Lil Nas X", Role.PRIMARY),
+             ("Jack Harlow", Role.PRIMARY)])
+
+    def test_protected_name_tyler_the_creator(self) -> None:
+        """Test that "Tyler, The Creator" is guarded from the
+        comma delimiter split."""
+        self.assertEqual(
+            build_db.parse_artist_credit(
+                "Tyler, The Creator Featuring GloRilla, Sexyy Red"
+                " & Lil Wayne"),
+            [("Tyler, The Creator", Role.PRIMARY),
+             ("GloRilla", Role.FEATURED),
+             ("Sexyy Red", Role.FEATURED),
+             ("Lil Wayne", Role.FEATURED)])
+
+    def test_protected_name_tones_and_i(self) -> None:
+        """Test that "Tones And I" is guarded from the " and "
+        delimiter split."""
+        self.assertEqual(
+            build_db.parse_artist_credit("Tones And I"),
+            [("Tones And I", Role.PRIMARY)])
+
+    def test_exception_spotemgottem(self) -> None:
+        """Test the SpotemGottem exception-table credit."""
+        self.assertEqual(
+            build_db.parse_artist_credit(
+                "SpotemGottem Featuring Pooh Shiesty Or DaBaby"),
+            [("SpotemGottem", Role.PRIMARY),
+             ("Pooh Shiesty", Role.FEATURED),
+             ("DaBaby", Role.FEATURED)])
+
+    def test_exception_the_scotts(self) -> None:
+        """Test the THE SCOTTS exception-table credit."""
+        self.assertEqual(
+            build_db.parse_artist_credit(
+                "THE SCOTTS, Travis Scott & Kid Cudi"),
+            [("Travis Scott", Role.PRIMARY),
+             ("Kid Cudi", Role.PRIMARY)])
+
+    def test_exception_drake_featuring_the_throne(self) -> None:
+        """Test the Drake Featuring The Throne exception-table
+        credit."""
+        self.assertEqual(
+            build_db.parse_artist_credit(
+                "Drake Featuring The Throne"),
+            [("Drake", Role.PRIMARY),
+             ("Jay Z", Role.FEATURED),
+             ("Kanye West", Role.FEATURED)])
+
+    def test_plus_delimiter_unaffected(self) -> None:
+        """Test that the existing "+" delimiter split is
+        unaffected by the new rules."""
+        self.assertEqual(
+            build_db.parse_artist_credit("Dan + Shay"),
+            [("Dan", Role.PRIMARY), ("Shay", Role.PRIMARY)])
+
 
 class TestBuildDB(unittest.TestCase):
     """Test cases for the working store build."""
@@ -194,7 +338,8 @@ class TestBuildDB(unittest.TestCase):
     def test_dedup_credit_variant(self) -> None:
         """Test that a credit variant listed in
         ``CANONICAL_ARTIST_CREDITS`` merges into one song, storing
-        the canonical credit."""
+        the canonical credit, with the canonical artist spelling
+        applied to the artist listed in ``CANONICAL_ARTIST_NAMES``."""
         self.__write_chart(
             "year,rank,title,artist\n"
             "2016,1,Eastside,\"benny blanco, Halsey & Khalid\"\n"
@@ -217,7 +362,72 @@ class TestBuildDB(unittest.TestCase):
             [(2016, 1), (2017, 1)])
         self.assertEqual(
             [x.artist.name for x in songs[0].song_artists],
-            ["Benny Blanco", "Halsey", "Khalid"])
+            ["benny blanco", "Halsey", "Khalid"])
+
+    def test_dedup_artist_name_case_variant(self) -> None:
+        """Test that an artist name case variant merges into one
+        artist, keeping the first-seen spelling."""
+        self.__write_chart(
+            "year,rank,title,artist\n"
+            "2016,1,Song One,Marshmello\n"
+            "2016,2,filler,Filler Artist\n"
+            "2017,1,Song Two,marshmello\n"
+            "2017,2,filler,Filler Artist\n")
+        status: int
+        stderr: str
+        status, stderr = self.__run_build()
+        self.assertEqual(status, 0)
+        session: Session = self.__session()
+        artists: list[Artist] = list(session.scalars(
+            sa.select(Artist).where(Artist.name.ilike("marsh%"))))
+        self.assertEqual(len(artists), 1)
+        self.assertEqual(artists[0].name, "Marshmello")
+        self.assertEqual(
+            {x.title for x in
+             {y.song for y in artists[0].song_artists}},
+            {"Song One", "Song Two"})
+
+    def test_canonical_artist_name_restored(self) -> None:
+        """Test that a canonical spelling is stored even for a
+        single, non-canonically spelled occurrence."""
+        self.__write_chart(
+            "year,rank,title,artist\n"
+            "2016,1,Beggin',Maneskin\n"
+            "2016,2,filler,Filler Artist\n"
+            "2017,1,filler2,Filler Artist Two\n"
+            "2017,2,filler3,Filler Artist Three\n")
+        status: int
+        stderr: str
+        status, stderr = self.__run_build()
+        self.assertEqual(status, 0)
+        session: Session = self.__session()
+        artist: Artist | None = session.scalar(
+            sa.select(Artist).where(Artist.name.ilike("m%nesk%")))
+        assert artist is not None
+        self.assertEqual(artist.name, "Måneskin")
+
+    def test_canonical_ye_resolves_to_kanye_west(self) -> None:
+        """Test that a "Ye" credit and a "Kanye West" credit merge
+        into a single artist row named "Kanye West", credited on
+        both songs."""
+        self.__write_chart(
+            "year,rank,title,artist\n"
+            "2016,1,Song One,Ye\n"
+            "2016,2,Song Two,Kanye West\n"
+            "2017,1,filler2,Filler Artist Two\n"
+            "2017,2,filler3,Filler Artist Three\n")
+        status: int
+        stderr: str
+        status, stderr = self.__run_build()
+        self.assertEqual(status, 0)
+        session: Session = self.__session()
+        artists: list[Artist] = list(session.scalars(
+            sa.select(Artist).where(Artist.name == "Kanye West")))
+        self.assertEqual(len(artists), 1)
+        self.assertEqual(
+            {x.title for x in
+             {y.song for y in artists[0].song_artists}},
+            {"Song One", "Song Two"})
 
     def test_first_run_on_fresh_store(self) -> None:
         """Test that a build on a fresh store creates the tables."""