From 00b5c208ec13f8753612afaf768289a4222638e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BE=9D=E7=91=AA=E8=B2=93?= Date: Mon, 3 Aug 2026 23:49:14 +0800 Subject: [PATCH] Resolve artists on Wikidata via exact-match SPARQL with song corroboration --- README.md | 2 +- data/captures/artists-wikidata.csv | 470 ++++++++++++++++ docs/decision_log.md | 14 + docs/project_structure.md | 2 +- .../src/pop_fem_audit_tools/fetch_artists.py | 521 +++++++++++++++--- tools/tests/test_fetch_artists.py | 374 ++++++++++--- 6 files changed, 1231 insertions(+), 152 deletions(-) create mode 100644 data/captures/artists-wikidata.csv diff --git a/README.md b/README.md index 35dc7ed..eccda66 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Hot 100（2016–2025）為例的內容分析。 ../data/source/yearend_hot100_2016_2025.csv ../data/derived --lyrics-dir ../data/captures/lyrics - --wikidata-csv ../data/captures/artists_wikidata.csv + --wikidata-csv ../data/captures/artists-wikidata.csv --overrides-csv ../data/manual/artists_overrides.csv`）。 LLM 步驟使用 `claude-sonnet-4-6`、temperature=0、 thinking 關閉；每步驟獨立執行兩次後由仲裁步驟合併 diff --git a/data/captures/artists-wikidata.csv b/data/captures/artists-wikidata.csv new file mode 100644 index 0000000..ec84b2b --- /dev/null +++ b/data/captures/artists-wikidata.csv @@ -0,0 +1,470 @@ +name,qid,gender,type,genre,country,note +Justin Bieber,Q34086,male,solo,pop music; contemporary R&B; tropical house; reggaeton; contemporary country,Canada,Canadian singer-songwriter (born 1994) +Drake,Q33240,male,solo,hip-hop; contemporary R&B; pop music; trap music,Canada,Canadian-American rapper and singer +Wizkid,Q16233663,male,solo,Afrobeats; dancehall; rhythm and blues,Nigeria,Nigerian singer and songwriter (born 1990) +Kyla,Q25190133,female,solo,UK funky,United Kingdom,British singer +Rihanna,Q36844,female,solo,pop music; dancehall; contemporary R&B; electronic dance music; adult contemporary music,Barbados,Barbadian singer (born 1988) +twenty one pilots,Q7857806,,group,reggae; alternative rock; electropop; alternative hip-hop; rap rock; indie pop; pop rock,United States,American musical duo +Desiigner,Q23091498,male,solo,hip-hop; East Coast hip-hop; trap music; mumble rap; hardcore hip-hop,United States,American rapper and singer (born 1997) +Adele,Q23215,female,solo,pop music; blue-eyed soul; rhythm and blues; soul; jazz; rock music,United Kingdom,English singer-songwriter (born 1988) +The Chainsmokers,Q7721993,,group,electronic dance music,United States,American electronic music duo +Daya,Q21005558,female,solo,pop music,United States,American singer +Justin Timberlake,Q43432,male,solo,pop music; contemporary R&B,United States,American singer and actor (born 1981) +Halsey,Q18534249,female,solo,pop music; electropop; synth-pop; art pop; alternative pop; alternative rock; alternative R&B; indie pop; contemporary R&B,United States,American singer-songwriter (born 1994) +Sia,Q181484,female,solo,electropop; indie pop,Australia,Australian singer and songwriter (born 1975) +Sean Paul,Q200577,male,solo,dancehall; pop music,Jamaica,Jamaican dancehall singer and rapper (born 1973) +Lukas Graham,Q300935,,group,soul; pop music; pop rock; funk,Denmark,Danish band +Flo Rida,Q213538,male,solo,hip-hop; pop music; dance music; electronic dance music; Southern hip-hop; contemporary R&B; house music; electro hop; electro house; electropop; electro; EDM trap music; trap music; bass music; crunk; pop rap; dirty south; dance-pop; hip house; Miami bass; electronic music; techno; trance,United States,American rapper and singer (born 1979) +Mike Posner,Q268284,male,solo,pop music,United States,American musician +Fifth Harmony,Q3087893,,group,pop music,United States,American girl group +Ty Dolla $ign,Q7859785,male,solo,contemporary R&B; West Coast hip-hop; trap music; hip-hop,United States,American singer and rapper (born 1982) +Calvin Harris,Q81637,male,solo,electronic dance music; electronic pop; acid house,United Kingdom,Scottish electronic music producer and DJ (born 1984) +DNCE,Q21648421,,group,pop rock,United States,American pop band +G-Eazy,Q5511893,male,solo,hip-hop,United States,American rapper +Bebe Rexha,Q16185856,female,solo,pop music; contemporary R&B,United States,American singer +Zayn,Q3626950,male,solo,pop music; contemporary R&B; dance-pop; alternative R&B,United Kingdom,English singer +Shawn Mendes,Q17198340,male,solo,folk-pop; pop music; pop rock,Canada,Canadian singer +Major Lazer,Q2604214,,group,dancehall; moombahton; reggae fusion; trap music; electronic dance music; hip-hop; electro house; rocksteady,United States,Jamaican-American electronic music band +MØ,Q10670047,female,solo,indie pop; electronic music,Kingdom of Denmark,Danish singer and songwriter +Rozes,Q22955580,female,solo,pop music; dance music; electronic dance music,United States,American singer +Future,Q3445057,male,solo,Southern hip-hop; trap music; mumble rap; contemporary R&B,United States,American rapper (born 1983) +The Weeknd,Q2121062,male,solo,alternative R&B; pop music; electropop,Canada,Canadian singer and songwriter (born 1990) +P!nk,Q160009,female,solo,pop music; pop rock; contemporary R&B,United States,"American singer, songwriter, and actress (born 1979)" +D.R.A.M.,Q25138504,male,solo,Southern hip-hop; contemporary R&B; trap music,United States,American rapper and singer (born 1988) +Lil Yachty,Q23772141,male,solo,hip-hop; trap music; pop rap; mumble rap; cloud rap; Southern hip-hop,United States,American rapper and singer +Bryson Tiller,Q21066641,male,solo,contemporary R&B; hip-hop; trap music; soul,United States,American singer and rapper (born 1993) +Ariana Grande,Q151892,female,solo,pop music; contemporary R&B; electronic dance music,United States,"American singer, songwriter, and actress (born 1993)" +gnash,Q23978226,male,solo,pop music; hip-hop; alternative R&B; indie folk,United States,American musician +Olivia O'Brien,Q23978275,female,solo,dance-pop; pop music,United States,American singer +Alessia Cara,Q20090091,female,solo,contemporary R&B; indie pop; pop music; soul; alternative R&B,Italy,"Italian-Canadian singer, songwriter and actress" +Selena Gomez,Q83287,female,solo,pop music; dance-pop; electropop; pop rock; contemporary R&B; alternative R&B,United States,American singer and actress (born 1992) +Meghan Trainor,Q17403494,female,solo,pop music; contemporary R&B; dance-pop; soul; doo-wop,United States,American singer-songwriter (born 1993) +John Legend,Q44857,male,solo,soul; pop music; contemporary R&B,United States,American singer-songwriter +Charlie Puth,Q19810390,male,solo,pop music; contemporary R&B,United States,"American singer, songwriter, and record producer" +James Bay,Q17811637,male,solo,rock music; pop music; soul; Americana,United Kingdom,British singer-songwriter +Zara Larsson,Q4177928,female,solo,dance-pop; pop music; electropop,Sweden,Swedish singer and songwriter (born 1997) +MNEK,Q6716905,male,solo,pop music,United Kingdom,British singer +DJ Snake,Q16728588,male,solo,electronic dance music; EDM trap music,France,French record producer (born 1986) +Kent Jones,Q25190082,male,solo,pop rap; contemporary R&B; pop music,United States,American singer and rapper (born 1993) +Florida Georgia Line,Q3644642,,group,country music,United States,American country music duo +Kiiara,Q23042767,female,solo,pop music; electropop; alternative R&B; experimental pop,United States,American singer and songwriter +Fetty Wap,Q19361511,male,solo,hip-hop; trap music; pop rap,United States,American rapper and singer (born 1991) +Remy Boyz,Q63405888,,group,hip-hop,United States,musical group +Jeremih,Q472891,male,solo,contemporary R&B,United States,American R&B singer (born 1987) +Kevin Gates,Q6396353,male,solo,hip-hop; gangsta rap; hardcore hip-hop; dirty south; Southern hip-hop,United States,American rapper +Daft Punk,Q185828,,group,house music; electronic music; French house; synth-pop; electronic rock; electronica; nu-disco; dance music; leftfield house,France,French electronic music band +DJ Khaled,Q80805,male,solo,hip-hop; pop rap,United States,American DJ and rapper (born 1975) +Flume,Q4038837,male,solo,electronica; trip hop; future bass,Australia,Australian music producer +Kai,Q6347342,female,solo,pop music,Canada,Canadian singer and songwriter +Elle King,Q5364665,female,solo,country music; Americana; blues rock; alternative country,United States,"American singer, songwriter, and actress" +Thomas Rhett,Q7793491,male,solo,country music,United States,American country singer +Post Malone,Q21621919,male,solo,hip-hop; pop music; country music,United States,American musician (born 1995) +Nick Jonas,Q201656,male,solo,pop music; rock music; contemporary R&B,United States,"American singer, actor and musician" +Tove Lo,Q15948790,female,solo,pop music; indie pop; synth-pop; electropop,Sweden,Swedish singer +X Ambassadors,Q15963313,,group,alternative rock,United States,"American rock band from Ithaca, New York" +Lil Wayne,Q15615,male,solo,Southern hip-hop; bounce music; gangsta rap; trap music; pop rap; dirty south; hardcore hip-hop,United States,American rapper and singer +Wiz Khalifa,Q117139,male,solo,East Coast hip-hop; pop rap,United States,"American rapper, singer and actor (born 1987)" +Imagine Dragons,Q391348,,group,pop rock; electropop; alternative rock; new wave,United States,American pop rock band +Logic,Q6667429,male,solo,hip-hop; pop rap; conscious hip-hop; East Coast hip-hop,United States,American rapper from Maryland +Yo Gotti,Q586919,male,solo,Southern hip-hop; trap music; dirty south; hardcore hip-hop; gangsta rap; Memphis rap,United States,American rapper +Nicki Minaj,Q162202,female,solo,hip-hop; pop music; contemporary R&B; trap music; dance music,Trinidad and Tobago,Trinidadian rapper (born 1982) +Tory Lanez,Q18353558,male,solo,hip-hop; contemporary R&B,Canada,Canadian rapper and singer (born 1992) +Beyoncé,Q36153,female,solo,pop music; contemporary R&B; country music,United States,"American singer, songwriter, record producer, and dancer (born 1981)" +Coldplay,Q45188,,group,alternative rock; pop music,United Kingdom,British rock band +Travis Scott,Q13605596,male,solo,hip-hop; trap music; pop rap; psychedelic rap; rock music; Southern hip-hop; contemporary R&B; psychedelic music; pop rock; alternative rock; alternative hip-hop; alternative R&B; psychedelic rock; psychedelic hip-hop,United States,"American rapper, singer and record producer" +Ruth B,Q22056665,female,solo,pop music,Canada,Canadian singer-songwriter +Taylor Swift,Q26876,female,solo,pop music; country music,United States,American singer-songwriter (born 1989) +Bipolar Sunshine,Q16727521,male,solo,pop music,United Kingdom,British musician +Ellie Goulding,Q31164,female,solo,synth-pop; electropop; indie pop; folktronica; alternative rock; alternative music; dream pop; indietronica,United Kingdom,English singer and songwriter (born 1986) +Jay Z,Q62766,male,solo,East Coast hip-hop; mafioso rap; gangsta rap; hardcore hip-hop,United States,"American rapper, record producer, and entrepreneur" +Kanye West,Q15935,male,solo,hip-hop; pop music; electro; soul; gospel music; progressive rap; art pop; pop rap; hip-hop soul; chipmunk soul; contemporary R&B,United States,American rapper and record producer +Camila Cabello,Q18810940,female,solo,pop music; contemporary R&B; Latin pop; Gothic pop,Cuba,Cuban-Mexican-American singer +Fat Joe,Q309888,male,solo,East Coast hip-hop; gangsta rap; hardcore hip-hop; boom bap; mafioso rap,United States,American rapper +Remy Ma,Q275437,female,solo,East Coast hip-hop,United States,American rapper +French Montana,Q1455261,male,solo,East Coast hip-hop; gangsta rap; hardcore hip-hop,Morocco,Moroccan and American rapper +Infared,Q140814135,male,solo,hip-hop,,hip-hop songwriter and rapper featured on the 2016 single All the Way Up +Silentó,Q20354248,male,solo,hip-hop; pop rap; trap music; comedy hip-hop; snap music; pop music,United States,American rapper and singer (born 1998) +Chris Brown,Q155700,male,solo,contemporary R&B; pop music; hip-hop,United States,American singer and rapper (born 1989) +Usher,Q165911,male,solo,contemporary R&B; pop music; soul; hip-hop; dance music; dance-pop; crunk,United States,American R&B singer (born 1978) +Young Thug,Q15637814,male,solo,Southern hip-hop; trap music; mumble rap; progressive rap; SoundCloud rap,United States,American rapper and singer +O.T. Genasis,Q18623749,male,solo,hip-hop; trap music,United States,American rapper +Young Dolph,Q23541285,male,solo,Southern hip-hop; gangsta rap; trap music; Memphis rap; dirty south; hardcore hip-hop,United States,American rapper (1985–2021) +Hailee Steinfeld,Q231726,female,solo,pop music; dance-pop,United States,American actress and singer-songwriter (born 1996) +Grey,Q28451953,,group,electronic music; pop music; future bass,United States,American electronic music group consisting of brothers Kyle and Michael Trewartha +Zedd,Q21088,male,solo,electronic dance music; house music; electro house,Russia,German electronic music producer (born 1989) +Tim McGraw,Q356487,male,solo,country music,United States,American country singer and actor (born 1967) +One Direction,Q146027,,group,dance-pop; teen pop; pop rock,United Kingdom,English-Irish musical group; boy band +Ed Sheeran,Q47447,male,solo,pop music; folk music; hip-hop; dance music; soul; rock music,United Kingdom,English singer-songwriter and guitarist (born 1991) +Luis Fonsi,Q1387726,male,solo,pop music; Latin pop; ballad; reggaeton,United States,Puerto Rican singer +Daddy Yankee,Q272591,male,solo,reggaeton; Latin hip-hop,United States,Puerto Rican rapper and actor (born 1976) +Bruno Mars,Q1450,male,solo,pop music; contemporary R&B; hip-hop; new wave; reggae; funk,United States,American singer-songwriter and record producer (born 1985) +Kendrick Lamar,Q130798,male,solo,West Coast hip-hop; progressive rap; alternative hip-hop; jazz rap; conscious hip-hop; gangsta rap; hardcore hip-hop; jazz; pop music,United States,American rapper (born 1987) +Migos,Q15777045,,group,Southern hip-hop; trap music; mumble rap,United States,American hip hop group +Lil Uzi Vert,Q23771950,non-binary,solo,hip-hop; emo rap; trap music; SoundCloud rap; punk rap; psychedelic rap; rage; cloud rap; psychedelic hip-hop; East Coast hip-hop,United States,American rapper +Sam Hunt,Q16223171,male,solo,country music,United States,American singer and songwriter +Quavo,Q30072039,male,solo,hip-hop; trap music,United States,American rapper +James Arthur,Q717626,male,solo,pop music,United Kingdom,British singer (born 1988) +Chance the Rapper,Q12470060,male,solo,hip-hop; alternative hip-hop; contemporary R&B; conscious hip-hop; alternative R&B; gospel music,United States,"American rapper, singer and record producer (born 1993)" +Swae Lee,Q30103025,male,solo,hip-hop; trap music,United States,American rapper and singer-songwriter (born 1993) +Rae Sremmurd,Q18123226,,group,hip-hop; trap music; pop music; pop rap,United States,American hip hop duo +Gucci Mane,Q206032,male,solo,Southern hip-hop; trap music; gangsta rap; hardcore hip-hop; dirty south; Atlanta hip-hop; dirty rap,United States,American rapper and music executive (born 1980) +Khalid,Q28816483,male,solo,contemporary R&B,United States,American singer (born 1998) +Cardi B,Q29033668,female,solo,hip-hop; pop music; pop rap,United States,American rapper (born 1992) +Childish Gambino,Q1239933,male,solo,hip-hop; funk; contemporary R&B; soul; psychedelic music; indie pop; comedy hip-hop,United States,"American actor, comedian, musician, and filmmaker (born 1983)" +Kygo,Q16845512,male,solo,tropical house; deep house; downtempo; progressive house; house music; electronic dance music,Norway,Norwegian DJ and music producer +KYLE,Q28606665,male,solo,West Coast hip-hop; pop rap,United States,American rapper and singer +Julia Michaels,Q20051697,female,solo,pop music; electronic dance music,United States,American singer and songwriter (born 1993) +Niall Horan,Q775231,male,solo,pop music; pop rock; folk-pop,Ireland,Irish singer-songwriter +Big Sean,Q367634,male,solo,hip-hop; Midwest hip-hop; Michigan rap; pop rap,United States,American rapper (born 1988) +Liam Payne,Q2756349,male,solo,pop music; contemporary R&B,United Kingdom,English singer and songwriter (1993–2024) +Maroon 5,Q182223,,group,alternative rock; funk rock; pop rock; pop music,United States,American pop rock band +Machine Gun Kelly,Q1354843,male,solo,hip-hop; pop rap; rap rock; pop-punk; alternative rock; alternative hip-hop,United States,American musician (born 1990) +Clean Bandit,Q11838808,,group,electropop,United Kingdom,English electronic music group +Anne-Marie,Q25396976,female,solo,pop music; contemporary R&B,United Kingdom,British singer (born 1990) +Portugal. The Man,Q606879,,group,pop music; indie rock,United States,American rock band +Demi Lovato,Q41173,genderfluid,solo,pop rock; pop music; contemporary R&B; power pop; dance-pop; teen pop; electropop; soul; pop-punk; nu metal,United States,"American singer, songwriter, and actress (born 1992)" +21 Savage,Q25095399,male,solo,hip-hop; trap music,United Kingdom,British-born rapper (born 1992) +J Balvin,Q966845,male,solo,reggaeton; Latin pop; Latin trap,Colombia,Colombian singer +Willy William,Q17016749,male,solo,house music; reggaeton; zouk,France,"French DJ, record producer, and singer (born 1981)" +Kodak Black,Q22005867,male,solo,hip-hop; trap music; SoundCloud rap; gangsta rap; Southern hip-hop; mumble rap,United States,American rapper (born. 1997) +Brett Young,Q25189679,male,solo,country music,United States,American country musician +Aminé,Q27830860,male,solo,West Coast hip-hop; alternative hip-hop; trap music; contemporary R&B,United States,"American rapper, singer, and songwriter" +Ayo,Q55860029,male,solo,hip-hop,United States,American singer and dancer +Teo,Q55860035,male,solo,hip-hop,United States,American singer and dancer +Zay Hilfigerrr,Q55859289,male,solo,hip-hop,United States,American rapper +Zayion McCall,Q55859293,male,solo,hip-hop,United States, +Kesha,Q33605,female,solo,electropop; dance-pop; synth-pop; pop rock; pop rap; rock music; dance music; country music,United States,American singer and songwriter (born 1987) +Frank Ocean,Q357645,male,solo,alternative R&B; psychedelic soul; progressive soul; contemporary R&B; pop music; soul; hip-hop; avant-pop,United States,"American singer, songwriter, and rapper" +Kane Brown,Q21620939,male,solo,country music,United States,American country music singer +Lauren Alaina,Q271796,female,solo,country music; folk music,United States,American singer & songwriter (born 1994) +Katy Perry,Q42493,female,solo,pop music; rock music; pop rock; dance music; synth-pop; electropop; gospel music,United States,American singer +Skip Marley,Q21621948,male,solo,reggae; pop music; hip-hop; rock music,Jamaica,Jamaican singer and songwriter +Pharrell Williams,Q14313,male,solo,contemporary R&B; hip-hop; pop music; funk; rock music; pop rap,United States,American record producer and songwriter +Jon Bellion,Q18044284,male,solo,pop music; contemporary R&B; hip-hop; indie rock; pop rap; alternative hip-hop; indie pop; alternative rock; dance-pop; soul,United States,"American singer, rapper, songwriter, and record producer" +Luke Combs,Q23884740,male,solo,country music,United States,American country singer +Sam Smith,Q15123969,non-binary,solo,soul; contemporary R&B; pop music; rhythm and blues,United Kingdom,British singer-songwriter +Playboi Carti,Q27671080,male,solo,Southern hip-hop; trap music; mumble rap; cloud rap; rage; punk rap; alternative hip-hop; plugg music; lo-fi music; SoundCloud rap,United States,American rapper +SZA,Q16210722,female,solo,contemporary R&B; alternative R&B; neo soul; pop music; hip-hop,United States,American singer +A Boogie wit da Hoodie,Q27927826,male,solo,hip-hop; trap music,United States,American rapper (born 1995) +Jason Derulo,Q243639,male,solo,pop music; rhythm and blues; contemporary R&B; hip-hop,United States,American singer-songwriter +Harry Styles,Q3626966,male,solo,soft rock; pop music; rock music; Britpop; traditional folk music,United Kingdom,"English singer, songwriter and actor" +Miley Cyrus,Q4235,genderfluid,solo,pop music; country music; pop rock; dance-pop; electropop; rock music; hip-hop; teen pop; country pop,United States,"American singer, actress, and television personality (born 1992)" +Marian Hill,Q20858093,,group,electronic music,United States,American musical duo +Cheat Codes,Q23978754,,group,electronic music; electronic dance music; tropical house; dancehall,United States,electronic music DJ trio +Dustin Lynch,Q5317118,male,solo,country music,United States,American singer +YFN Lucci,Q28000403,male,solo,hip-hop,United States,American rapper (born 1991) +PnB Rock,Q28155029,male,solo,hip-hop; contemporary R&B; trap music; pop music; pop rap,United States,American rapper (1991–2022) +blackbear,Q22974007,male,solo,pop music; emo; contemporary R&B; hip-hop,United States,"American singer, songwriter, and record producer (born 1990)" +XXXTENTACION,Q28561969,male,solo,hip-hop; emo rap; SoundCloud rap; rap rock; lo-fi music; hardcore hip-hop; alternative rock; lofi hip-hop; indie rock; nu metal; cloud rap; trap music; hardcore punk; trap metal; punk rock; rock music; rap metal; alternative R&B; contemporary R&B; punk rap; heavy metal music,United States,American rapper (1998–2018) +Keith Urban,Q315547,male,solo,country music,Australia,Australian-American musician +Carrie Underwood,Q215546,female,solo,pop music; country music; country pop; country rock,United States,American singer and actor (born 1983) +Bad Bunny,Q44333953,male,solo,Latin trap; reggaeton,Puerto Rico,"Puerto Rican rapper, singer, and record producer (born 1994)" +Maren Morris,Q21622122,female,solo,country music; country pop; folk rock; folk music,United States,American country singer +Juice WRLD,Q52151598,male,solo,hip-hop; emo rap; alternative rock; trap music; pop rap; SoundCloud rap; contemporary R&B; alternative hip-hop; mumble rap,United States,American rapper and singer (1998–2019) +Ella Mai,Q29033458,female,solo,contemporary R&B,United Kingdom,British singer +Dua Lipa,Q21914464,female,solo,pop music; post-disco; house music; funktronica; indie pop; dream pop; synth-pop,United Kingdom,British singer-songwriter (born 1995) +Normani,Q28148022,female,solo,pop music; contemporary R&B,United States,American singer and dancer +Bazzi,Q48720231,male,solo,pop music,United States,American-born Lebanese singer-songwriter (born 1997) +BlocBoy JB,Q49481016,male,solo,hip-hop; trap music,United States,American rapper from Tennessee +Lil Baby,Q50527563,male,solo,hip-hop; trap music; Southern hip-hop,United States,American rapper (born 1994) +Marshmello,Q26334949,male,solo,electronic dance music,United States,American DJ and electronic music producer (born 1992) +Tyga,Q358087,male,solo,West Coast hip-hop; pop rap,United States,American rapper +Offset,Q30612480,male,solo,hip-hop; trap music; Southern hip-hop,United States,American rapper +NF,Q19571648,male,solo,hip-hop,United States,American rapper from Michigan +A$AP Rocky,Q129910,male,solo,East Coast hip-hop; psychedelic rap; experimental hip-hop; alternative hip-hop; cloud rap; trap music,United States,American rapper (born 1988) +6ix9ine,Q44479972,male,solo,hip-hop; East Coast hip-hop; gangsta rap; trap metal; hardcore hip-hop; SoundCloud rap; punk rap; drill; trap music; urbano music; reggaeton; Brooklyn drill,United States,American rapper from New York (born 1996) +Murda Beatz,Q27970613,male,solo,hip-hop,Canada,Canadian record producer (born 1994) +Dan Smyers,Q57630595,male,solo,country music,United States,songwriter and member of the duo Dan + Shay +Shay Mooney,Q116536996,male,solo,country music,United States,"American musician and songwriter, part of Dan + Shay" +Lauv,Q30122224,male,solo,electronic music; pop music,United States,American musician +5 Seconds of Summer,Q4641147,,group,pop rock; pop-punk,Australia,Australian pop rock band +Metro Boomin,Q16235273,male,solo,hip-hop; trap music,United States,American record producer (born 1993) +Lil Pump,Q38002101,male,solo,hip-hop; trap music; SoundCloud rap; punk rap; Southern hip-hop,United States,American rapper +Zacari,Q68593407,male,solo,contemporary R&B; hip-hop; alternative hip-hop; alternative R&B,United States,"American singer, songwriter, and multi-instrumentalist (born 1994)" +Lil Dicky,Q16876006,male,solo,hip-hop; pop rap; comedy hip-hop,United States,American rapper and comedian (born 1988) +Rich The Kid,Q18637908,male,solo,hip-hop; trap music; mumble rap; plugg music; Southern hip-hop,United States,American rapper +YG,Q3076050,male,solo,West Coast hip-hop; gangsta rap; dirty rap; hardcore hip-hop,United States,American rapper and actor +2 Chainz,Q129668,male,solo,Southern hip-hop; trap music; gangsta rap; dirty south; Atlanta hip-hop; hardcore hip-hop,United States,American rapper +MAX,Q1190160,male,solo,pop music,United States,American singer +Jason Aldean,Q195439,male,solo,country music,United States,American country singer +A$AP Ferg,Q15648254,male,solo,East Coast hip-hop; trap music,United States,American rapper (born 1988) +Miguel,Q73463,male,solo,contemporary R&B; alternative R&B; hip-hop; pop music; soul,United States,American R&B singer +benny blanco,Q818048,male,solo,pop music,United States,American pop music producer (born 1988) +Jay Rock,Q152632,male,solo,West Coast hip-hop,United States,American rapper (born 1985) +James Blake,Q350362,male,solo,pop music; electronica; post-dubstep; electronic dance music,United Kingdom,English singer-songwriter (born 1988) +Bastille,Q689,,group,synth-pop,United Kingdom,English indie pop band +Casper Mágico,Q60333702,male,solo,Latin trap; reggaeton,United States,Puerto Rican rapper +Nio García,Q107358694,male,solo,Latin trap; reggaeton,United States,Puerto Rican singer +Darell,Q56467651,male,solo,reggaeton; Latin trap,United States,Puerto Rican rapper and singer +Nicky Jam,Q1374560,male,solo,reggaeton; Latin trap,United States,American singer +Ozuna,Q28739901,male,solo,reggaeton; Latin trap; Latin pop; dancehall,United States,Puerto Rican singer +N*E*R*D,Q941293,,group,funk rock; bossa nova; orchestral pop; hip-hop; skate punk,United States,American rock and hip-hop band +Chris Stapleton,Q5108148,male,solo,country music,United States,American country musician +Kenny Chesney,Q467526,male,solo,country music,United States,American country musician +YoungBoy Never Broke Again,Q36490532,male,solo,hip-hop; Southern hip-hop; gangsta rap; trap music,United States,American rapper (born 1999) +Juicy J,Q602779,male,solo,Southern hip-hop; gangsta rap; trap music; dirty rap; Memphis rap; dirty south; horrorcore; hardcore hip-hop; crunk,United States,American record producer and rapper (born 1975) +Lil Nas X,Q62591281,male,solo,hip-hop; pop rap; country rap; trap music; pop music; rock music; pop rock; country music; Southern hip-hop,United States,"American rapper, singer and songwriter (born 1999)" +Billy Ray Cyrus,Q231259,male,solo,country music; country pop; country rock; country blues,United States,American country singer and actor +Billie Eilish,Q29564107,female,solo,electropop; indie pop; alternative pop,United States,American singer-songwriter (born 2001) +Jonas Brothers,Q134233,,group,pop music; pop rock,United States,American pop rock band +Panic! At The Disco,Q277551,,group,pop rock; synth-pop; pop-punk; baroque pop; emo pop; alternative rock,United States,American musical group; rock solo project band +Lizzo,Q16232225,female,solo,hip-hop; soul; contemporary R&B; gospel music,United States,American singer and rapper (born 1988) +Meek Mill,Q1897911,male,solo,East Coast hip-hop; hardcore hip-hop; gangsta rap; trap music; hip-hop,United States,American rapper +Lady Gaga,Q19848,female,solo,electropop; dance-pop; electronic dance music; art pop; pop music; synth-pop,United States,"American singer, songwriter, and actress (born 1986)" +Bradley Cooper,Q205707,male,solo,,United States,American actor (born 1975) +Ava Max,Q56755505,female,solo,dance-pop,United States,American singer and songwriter (born 1994) +DaBaby,Q62107457,male,solo,hip-hop; trap music; pop rap; Southern hip-hop,United States,American rapper (born 1991) +J. Cole,Q204018,male,solo,hip-hop; Southern hip-hop; pop rap; East Coast hip-hop; conscious hip-hop,United States,American rapper and record producer (born 1985) +Gunna,Q55613105,male,solo,hip-hop; trap music; Southern hip-hop; contemporary R&B,United States,American rapper (born 1993) +Lewis Capaldi,Q42315001,male,solo,blue-eyed soul; pop music; pop rock; alternative rock; soft rock,United Kingdom,Scottish singer-songwriter (born 1996) +Lil Tecca,Q65054660,male,solo,hip-hop; trap music,United States,American rapper and singer-songwriter +Rick Ross,Q297831,male,solo,Southern hip-hop; gangsta rap; mafioso rap; trap music; hardcore hip-hop; dirty south,United States,American rapper and record executive +Brendon Urie,Q1047474,male,solo,pop rock; alternative rock,United States,American singer and musician +Polo G,Q62953486,male,solo,hip-hop; drill; trap music; conscious hip-hop; Midwest hip-hop; Chicago hip-hop,United States,American rapper +Lil Tjay,Q61670761,male,solo,hip-hop,United States,American rapper +Blueface,Q60463134,male,solo,hip-hop; trap music; West Coast hip-hop; gangsta rap,United States,American rapper (born 1997) +Sheck Wes,Q56513324,male,solo,East Coast hip-hop; trap music,United States,American rapper (born 1998) +Morgan Wallen,Q29561472,male,solo,country music; country pop; bro-country; country rap,United States,American country singer +Blake Shelton,Q94831,male,solo,country music,United States,American country music singer and television personality +Dean Lewis,Q29905254,male,solo,pop music,Australia,Australian singer-songwriter (born 1987) +Mustard,Q15941115,male,solo,hip-hop; ratchet music; West Coast hip-hop; contemporary R&B,United States,American record producer +Blanco Brown,Q64736729,male,solo,country rap; trap music; hip-hop,United States,American rapper and singer +Diplo,Q533781,male,solo,electronic dance music; trap music; hip-hop; pop music; moombahton; electropop; house music; EDM trap music,United States,American record producer (born 1978) +Calboy,Q66085272,male,solo,hip-hop,United States,American rapper (born 1999) +Lauren Daigle,Q18685595,female,solo,contemporary Christian music,United States,American singer and songwriter (born 1991) +Snow,Q94708,male,solo,dancehall; reggae; hip-hop; reggae fusion,Canada,Canadian dancehall rapper (born 1969) +YNW Melly,Q56276347,male,solo,Southern hip-hop; trap music; contemporary R&B; gangsta rap; drill; hardcore hip-hop,United States,American rapper +City Girls,Q60658858,,group,hip-hop,United States,American musical duo +SHAED,Q60643444,,group,electropop; indie pop,United States,American musical group +Flipp Dinero,Q57584556,male,solo,hip-hop,United States,American rapper +Pinkfong,Q55735607,,,children's music,South Korea,Korean children's entertainment brand +Saweetie,Q50486592,female,solo,hip-hop,United States,American rapper (born 1993) +YK Osiris,Q66095815,male,solo,contemporary R&B; hip-hop,United States,"American rapper, singer, and songwriter" +Luke Bryan,Q166197,male,solo,country music,United States,American country music singer +Lee Brice,Q1811729,male,solo,country music,United States,American country singer +Megan Thee Stallion,Q59268408,female,solo,hip-hop; trap music; dirty rap; pop rap; pop music,United States,"American rapper, singer and songwriter" +NLE Choppa,Q64009351,male,solo,trap music; hip-hop; drill; Memphis rap; gangsta rap; Southern hip-hop,United States,American rapper from Tennessee (born 2002) +Chase Rice,Q5087189,male,solo,country music,United States,American singer-songwriter and football player +Social House,Q65117469,,group,pop music,United States,American pop duo +Roddy Ricch,Q59209423,male,solo,West Coast hip-hop; trap music; pop rap; gangsta rap; hardcore hip-hop; hip-hop,United States,American rapper (born 1998) +Doja Cat,Q51120673,female,solo,contemporary R&B; pop music; electronic music; indie pop; hip-hop,United States,American singer and rapper (born 1995) +Gabby Barrett,Q53912835,female,solo,country pop,United States,American country singer +Jack Harlow,Q59727812,male,solo,hip-hop; pop rap,United States,American rapper and singer (born 1998) +Tones And I,Q62887629,female,solo,indie pop; electronic dance music; pop music,Australia,"Australian singer, songwriter and record producer" +Arizona Zervas,Q75124792,male,solo,hip-hop; contemporary R&B; pop music,United States,American singer and rapper (born 1995) +SAINt JHN,Q52274581,male,solo,hip-hop; trap music; contemporary R&B,United States,Guyanese-American rapper and singer +Trevor Daniel,Q81738417,male,solo,contemporary R&B; pop music,United States,American singer-songwriter +Lil Mosey,Q55819454,male,solo,hip-hop; pop rap; trap music,United States,American rapper and singer-songwriter +Jawsh 685,Q96472235,male,solo,pop music,New Zealand,New Zealand music producer +BTS,Q13580495,,group,K-pop; dance-pop; Korean hip-hop,South Korea,South Korean musical group; boy band +Lil Durk,Q2899818,male,solo,Midwest hip-hop; trap music; drill; gangsta rap; hardcore hip-hop; Chicago hip-hop; Chicago drill,United States,American rapper +Powfu,Q89069821,male,solo,hip-hop; lo-fi music; power pop; rock music,Canada,Canadian rapper and singer +beabadoobee,Q66817366,female,solo,indie rock; space rock; alternative rock; slacker rock; indie pop; folk-pop; bedroom pop,Philippines,Filipino singer +24kGoldn,Q83751918,male,solo,pop rap,United States,American rapper +iann dior,Q65631841,male,solo,emo rap; pop rap; rap rock; pop rock,United States,American rapper and singer +Pop Smoke,Q81698554,male,solo,East Coast hip-hop; Brooklyn drill; trap music; gangsta rap; drill; hip-hop,United States,American rapper (1999–2020) +Black Eyed Peas,Q134541,,group,alternative hip-hop; pop rap; hip-hop; pop music; Latin hip-hop,United States,American hip hop group from California +Rod Wave,Q78754352,male,solo,hip-hop; contemporary R&B; soul; trap music,United States,American rapper and singer-songwriter (born 1998) +Gwen Stefani,Q483379,female,solo,pop music; contemporary R&B; ska; electronica; new wave; rock music,United States,American singer +JP Saxe,Q86009464,male,solo,pop music,Canada,Canadian singer and musician +42 Dugg,Q95694810,male,solo,Michigan rap; Midwest hip-hop; trap music; gangsta rap,United States,American rapper +Old Dominion,Q20685018,,group,country pop,United States,American country pop music group (2007–) +Surfaces,Q76330429,,group,pop music,United States,American musical duo +Eminem,Q5608,male,solo,hip-hop; horrorcore; hardcore hip-hop; comedy hip-hop; Midwest hip-hop; rap rock,United States,"American rapper, record producer and actor" +StaySolidRocky,Q95918251,male,solo,hip-hop,United States,American rapper +Maddie,Q54607228,female,solo,country music,United States,"Singer, part of Maddie & Tae" +Tae,Q54607285,female,solo,country music,United States,"Singer, part of Maddie & Tae" +Mariah Carey,Q41076,female,solo,contemporary R&B; pop music; adult contemporary music; dance-pop; hip-hop,United States,American singer (born 1969) +Carly Pearce,Q29913337,female,solo,country music,United States,American country music singer-songwriter +50 Cent,Q6060,male,solo,East Coast hip-hop; gangsta rap; hardcore hip-hop,United States,American rapper (born 1975) +BENEE,Q61749159,female,solo,alternative pop; soul,New Zealand,New Zealand singer-songwriter +Gus Dapperton,Q51858412,male,solo,pop music,United States,American singer-songwriter +Kid Cudi,Q299138,male,solo,Midwest hip-hop; contemporary R&B; alternative hip-hop; pop rap; alternative rock; trip hop; neo-psychedelia; punk rock; grunge; emo rap; psychedelic rap; psychedelic hip-hop,United States,"American rapper, singer, record producer and actor (born 1984)" +ATR Son Son,Q140814072,male,solo,hip-hop,United States,American rapper +Miranda Lambert,Q231233,female,solo,country music; folk music,United States,American country singer +Eric Church,Q975911,male,solo,country music,United States,American country singer +Jhené Aiko,Q15929781,female,solo,hip-hop; pop music; rhythm and blues; soul; alternative R&B; neo soul; contemporary R&B,United States,American R&B singer (born 1988) +Surf Mesa,Q93771412,male,solo,tropical house,United States,American record producer and multi-instrumentalist +Emilee,Q98635221,female,solo,pop music,United States,"American singer, star of TikTok and influencer" +Moneybagg Yo,Q38459032,male,solo,hip-hop; trap music; gangsta rap; Memphis rap; Southern hip-hop,United States,American rapper from Tennessee +H.E.R.,Q30639849,female,solo,contemporary R&B,United States,American R&B singer +Olivia Rodrigo,Q63243883,female,solo,indie pop; pop music; rock music,United States,American singer and actress (born 2003) +Anderson .Paak,Q20810369,male,solo,contemporary R&B; hip-hop; soul; funk; pop music,United States,"American singer, rapper, drummer, and record producer (born 1986)" +Daniel Caesar,Q29589083,male,solo,contemporary R&B; soul; neo soul,Canada,Canadian R&B singer +Giveon,Q96418479,male,solo,contemporary R&B,United States,American R&B singer +The Kid LAROI,Q82890761,male,solo,Australian hip-hop; contemporary R&B; pop music; emo rap,Australia,Australian rapper and singer (born 2003) +Glass Animals,Q16839267,,group,indie rock; psychedelic pop; indietronica; alternative pop; electronic rock; psychedelic rock,United Kingdom,English indie rock band +Masked Wolf,Q105490126,male,solo,hip-hop,Australia,Australian rapper +Walker Hayes,Q7962267,male,solo,country music,United States,American musician +Jhay Cortez,Q75997191,male,solo,Latin trap; reggaeton; Latin R&B,United States,Puerto Rican singer +6LACK,Q27886640,male,solo,alternative R&B; contemporary R&B; trap music; cloud rap; industrial music; soul; hip-hop soul,United States,American singer-songwriter and rapper (born 1992) +Tate McRae,Q95129121,female,solo,pop music; alternative pop; dance-pop,Canada,"Canadian singer, songwriter, and dancer" +Pooh Shiesty,Q105105274,male,solo,trap music; Southern hip-hop; drill; hardcore hip-hop; gangsta rap; Memphis rap,United States,American rapper from Tennessee (born. 1999) +SpotemGottem,Q105105325,male,solo,mumble rap; trap music; drill; Southern hip-hop,United States,American rapper +Kali Uchis,Q19004484,female,solo,contemporary R&B,United States,American singer and songwriter (born 1994) +CJ,Q103868817,male,solo,drill; trap music; East Coast hip-hop; hip-hop,United States,American rapper +Internet Money,Q99691610,,,hip-hop,United States,American record label and record producer collective +Don Toliver,Q56513383,male,solo,hip-hop; trap music; contemporary R&B,United States,American singer and rapper (born 1994) +NAV,Q28831246,male,solo,hip-hop; cloud rap; trap music; contemporary R&B,Canada,"Canadian rapper, singer, and record producer (born 1989)" +AJR,Q16154058,,group,indie pop; alternative pop; electropop,United States,American indie pop band +Yung Bleu,Q105105248,male,solo,contemporary R&B; hip-hop,United States,American rapper and singer +Tems,Q72759173,female,solo,alternative R&B; Afrobeats,Nigeria,Nigerian singer-songwriter (born 1995) +Ryan Hurd,Q42224127,male,solo,country music,United States,American country singer +Måneskin,Q56884561,,group,pop rock; glam rock,Italy,Italian rock band +Chris Young,Q952176,male,solo,country music,United States,American country music artist +Nelly,Q218992,male,solo,Midwest hip-hop; pop rap; contemporary R&B; pop music; country rap; country music,United States,"American rapper, singer and actor (born 1974)" +Mooski,Q113554625,male,solo,hip-hop,United States,Musical artist +Coi Leray,Q106376206,female,solo,hip-hop; pop music,United States,American rapper and singer +Duncan Laurence,Q60758679,male,solo,pop music,Kingdom of the Netherlands,Dutch singer +Niko Moon,Q67399825,male,solo,country music,United States,musical artist +Project Pat,Q593741,male,solo,Memphis rap; gangsta rap; hardcore hip-hop; Southern hip-hop; crunk; dirty south; trap music; horrorcore,United States,American rapper +Parmalee,Q7139162,,group,country music,United States,American country music band +Brenda Lee,Q230925,female,solo,rock and roll; country music; pop music; rockabilly,United States,American singer and recording artist (born 1944) +Ritt Momney,Q105105308,male,solo,folk music; indie pop; synth-pop,United States,American singer-songwriter from Utah +Cole Swindell,Q15101510,male,solo,country music,United States,American singer-songwriter +Lainey Wilson,Q106378786,female,solo,country music; folk music,United States,American singer-songwriter +BRS Kash,Q105409004,male,solo,hip-hop,United States,American rapper +Rauw Alejandro,Q61586861,male,solo,contemporary R&B; urban contemporary; Latin R&B; reggaeton; Latin trap; Latin pop; salsa,United States,Puerto Rican singer and songwriter +Latto,Q65029930,female,solo,hip-hop; trap music; pop music; pop rap,United States,American rapper (born 1998) +Elton John,Q2808,male,solo,rock music; pop rock; glam rock; soft rock; rhythm and blues,United Kingdom,"British singer, songwriter and musician (born 1947)" +JID,Q29588830,male,solo,hip-hop,United States,"American rapper, singer, and songwriter from Georgia" +GAYLE,Q109768702,female,solo,pop-punk; pop rock; power pop; emo pop; alternative pop; alternative rock,United States,American singer-songwriter +Chencho Corleone,Q113625582,male,solo,reggaeton,United States,Puerto Rican singer (born 1979) +Kate Bush,Q636,female,solo,art pop; progressive pop; singer-songwriter music; baroque pop; experimental pop; art rock; pop rock,United Kingdom,"English singer, pianist and songwriter (born 1958)" +Carolina Gaitán,Q5753201,female,solo,,Colombia,Colombian actress and singer +Mauro Castillo,Q16605580,male,solo,salsa,Colombia,"Colombian singer, composer, and actor" +Adassa,Q576181,female,solo,reggaeton; urban contemporary; pop music,United States,American singer-songwriter +Rhenzy Feliz,Q34344805,male,solo,,United States,American actor +Diane Guerrero,Q17166318,female,solo,,United States,American actress +Stephanie Beatriz,Q16832110,female,solo,,United States,Argentine-American actress +Encanto Cast,Q140814124,,,,United States,ensemble cast credited on songs from the 2021 Disney animated film Encanto +Steve Lacy,Q56733980,male,solo,alternative R&B,United States,"American singer, songwriter, guitarist, and record producer (born 1998)" +Nicky Youre,Q112666811,male,solo,indie pop,United States,American singer-songwriter +dazy,Q140814120,male,solo,pop music,,Los Angeles-based record producer and musician +Em Beihold,Q111333139,female,solo,pop music,United States,American singer-songwriter +Cody Johnson,Q17916913,male,solo,country music; country rock,United States,American country singer and songwriter +OneRepublic,Q1438730,,group,pop rock; alternative rock; cello rock,United States,American pop rock band +Zach Bryan,Q106770023,male,solo,alternative country; red dirt; country music,United States,American country musician +Jordan Davis,Q44274417,male,solo,country pop; country music; pop music,United States,American country music singer and songwriter +Dove Cameron,Q13560423,female,solo,pop music; alternative pop; soul,United States,American actress and singer +Joji,Q23826617,male,solo,contemporary R&B; alternative R&B; trip hop; lo-fi music; surreal humour; satirical music; satire; comedy hip-hop; black comedy; alternative rock; comedy music,Japan,Japanese-Australian entertainer (born 1992) +Jessica Darrow,Q109718598,non-binary,solo,,United States,American actor and singer +Bailey Zimmerman,Q112581616,male,solo,country music,United States,American singer +CKay,Q98312451,male,solo,Afrobeats,Nigeria,Nigerian singer-songwriter +Muni Long,Q7245622,female,solo,contemporary R&B; pop music,United States,American singer +Becky G,Q1393997,female,solo,pop music; hip-hop; Latin music; Latin pop; reggaeton,United States,American singer and actress (born 1997) +Karol G,Q22001324,female,solo,reggaeton; Latin pop; Latin R&B,Colombia,Colombian singer and songwriter +Tyler Hubbard,Q20740966,male,solo,country music,United States,American singer +NEIKED,Q26703294,,group,,Sweden,Swedish artist collective +Mae Muller,Q83696883,female,solo,pop music; rhythm and blues,United Kingdom,British singer +WILLOW,Q298209,female,solo,indie pop; alternative R&B; experimental pop; neo soul; hip-hop; electronic dance music; pop music; dream pop; contemporary R&B; rock music; indie rock; progressive rock; pop-punk; alternative rock; alternative pop; jazz,United States,American singer and actress (born 2000) +Tyler Cole,Q140814245,male,solo,,United States,"American singer, songwriter and record producer" +Lauren Spencer-Smith,Q110630787,female,solo,pop music,Canada,Canadian singer and songwriter +Bobby Helms,Q888554,male,solo,country music,United States,American male singer (1933–1997) +Bomba Estéreo,Q887911,,group,cumbia,Colombia,Colombian band +Burl Ives,Q315723,male,solo,folk music,United States,American musician (1909–1995) +Russell Dickerson,Q24572267,male,solo,country music,United States,American country singer-songwriter +Jake Scott,Q140814152,male,solo,pop music,United States,American singer-songwriter +Scotty McCreery,Q255969,male,solo,country music,United States,American country music singer +Jon Pardi,Q6271354,male,solo,country music,United States,American country singer +ERNEST,Q110442717,male,solo,country music,United States,American hip hop and country musician +JNR CHOI,Q111507858,male,solo,hip-hop; trap music; world music,United Kingdom,Gambian-British rapper and singer +Sam Tompkins,Q112155955,male,solo,pop music,United Kingdom,British singer-songwriter +Kim Petras,Q60964,trans woman,solo,pop music,Germany,German singer +Rema,Q65071082,male,solo,Afrobeats,Nigeria,Nigerian singer-songwriter and rapper (2000–) +David Guetta,Q8298,male,solo,electronic dance music; house music; electroclash; electro house,France,French DJ and electronic music producer (born 1967) +PinkPantheress,Q107570416,female,solo,pop music; bedroom pop; dance music; alternative pop; drum and bass; 2-step garage; jungle; hyperpop,United Kingdom,British singer-songwriter +Ice Spice,Q114114977,female,solo,Bronx drill; hip-hop; East Coast hip-hop; Jersey club; pop rap,United States,American rapper (born 2000) +Toosii,Q106725126,male,solo,hip-hop,United States,American rapper from New York +Katelyn Brown,Q140814175,female,solo,pop music,United States,American singer +Eslabón Armado,Q112702265,,group,regional Mexican,United States,American hispanic radio music group +Peso Pluma,Q117242724,male,solo,regional Mexican; corridos tumbados; reggaeton; Latin trap; cumbia norteña mexicana,Mexico,Mexican singer (born 1999) +Jelly Roll,Q6176881,male,solo,Southern hip-hop; country rap; country music; hardcore hip-hop,United States,American rapper (born 1984) +Yng Lvcas,Q117834547,male,solo,reggaeton,Mexico,Mexican singer and rapper +JVKE,Q114865231,male,solo,pop music,United States,American singer and producer +Fifty Fifty,Q116731010,,group,K-pop; J-pop,South Korea,South Korean girl group +Aqua,Q213388,,group,pop music; bubblegum music; Eurodance,Denmark,Danish-Norwegian music group +RAYE,Q28071144,female,solo,contemporary R&B,United Kingdom,British singer-songwriter +070 Shake,Q54861040,female,solo,hip-hop; cloud rap; alternative hip-hop,United States,American rapper and singer +Grupo Frontera,Q118141487,,group,regional Mexican; norteño; cumbia norteña mexicana,United States,Mexican-American musical band +Stephen Sanchez,Q113023532,male,solo,pop music,United States,American singer-songwriter +HARDY,Q63341424,male,solo,country music,United States,American country singer +David Kushner,Q117745879,male,solo,,United States,American male singer-songwriter +Shakira,Q34424,female,solo,pop music; Latin pop; pop rock; Latin rock; dance music; folk music; world music; contemporary R&B; rock music,Spain,"Colombian singer, songwriter, and actress" +Megan Moroney,Q116786312,female,solo,country music; folk music,United States,American country music singer +GloRilla,Q114132368,female,solo,hip-hop; trap music; crunk; Memphis rap; Southern hip-hop,United States,American rapper +Fuerza Regida,Q118466501,,group,regional Mexican; banda music; corrido; corridos tumbados; sierreña; norteño,United States,regional Mexican group +Kacey Musgraves,Q2783855,female,solo,country music; folk music,United States,American singer (born 1988) +Doechii,Q106629186,female,solo,hip-hop; contemporary R&B; pop music; alternative hip-hop; pop rap,United States,American rapper and singer +Oliver Anthony Music,Q121391237,male,solo,country music; folk music,United States,American singer-songwriter +Noah Kahan,Q53832755,male,solo,folk-pop,United States,American singer-songwriter (born 1997) +Jung Kook,Q22338877,male,solo,K-pop,South Korea,South Korean singer +Wham!,Q744087,,group,pop music; new wave; blue-eyed soul,United Kingdom,British pop band +Parker McCollum,Q20684755,male,solo,Americana; country music,United States,American musician +Kaliii,Q115633164,female,solo,hip-hop; trap music; Southern hip-hop,United States,American rapper +Natanael Cano,Q104904549,male,solo,corridos tumbados; regional Mexican,Mexico,Mexican rapper and singer +Corey Kent,Q113452679,male,solo,country music,United States,American singer +Young Nudy,Q62036530,male,solo,hip-hop,United States,American rapper (born 1992) +Bizarrap,Q99937928,male,solo,Latin trap; reggaeton,Argentina,Argentine DJ and producer (born 1998) +Teddy Swims,Q105542333,male,solo,country music; pop music; rhythm and blues; soul,United States,American singer-songwriter +Shaboozey,Q125295750,male,solo,alternative country; country music; hip-hop; Americana,United States,American singer (born 1995) +Benson Boone,Q110008128,male,solo,pop music; pop rock,United States,American singer +Sabrina Carpenter,Q7396400,female,solo,pop music; teen pop; dance-pop; contemporary R&B; electropop; country pop,United States,American singer-songwriter and actress (born 1999) +Tommy Richman,Q125778611,male,solo,pop rap; contemporary R&B,United States,American singer +Hozier,Q16233232,male,solo,blues; blues rock; rhythm and blues; traditional folk music; soul; indie rock,Ireland,Irish musician and singer +Chappell Roan,Q110281146,female,solo,synth-pop; pop music; electronic music,United States,American singer (born 1998) +Tyla,Q118105408,female,solo,contemporary R&B; pop music; Amapiano; new jack swing,South Africa,South African singer-songwriter +Dasha,Q125157260,female,solo,country pop,United States,American country singer-songwriter +Jessie Murph,Q122157373,female,solo,country rock; country pop; pop music,United States,American singer +Sexyy Red,Q119842023,female,solo,hip-hop; trap music; dirty rap; Midwest hip-hop,United States,American rapper (born 1998) +Djo,Q27064031,male,solo,synth-pop,United States,American actor and musician (born 1992) +Flo Milli,Q89352069,female,solo,hip-hop,United States,American rapper +FloyyMenor,Q125819969,male,solo,reggaeton,Chile,Chilean singer-songwriter +Cris MJ,Q111425015,male,solo,reggaeton,Chile,Chilean singer-songwriter +Warren Zeiders,Q120986980,male,solo,country music,United States,American country musician +Artemas,Q125176874,male,solo,alternative pop,United Kingdom,British singer-songwriter and record producer +Xavi,Q124219723,male,solo,regional Mexican; corridos tumbados,United States,American singer +Myles Smith,Q126076580,male,solo,folk-pop; folk music; pop music; folk rock,United Kingdom,British singer +Paul Russell,Q122932071,male,solo,hip-hop; contemporary R&B; pop rap,United States,American singer +Summer Walker,Q61959478,female,solo,contemporary R&B,United States,American R&B singer +4batz,Q124313062,male,solo,contemporary R&B,United States,American singer (born 2003) +Koe Wetzel,Q107434048,male,solo,country music; rock music,United States,American musician +Yeat,Q108761978,male,solo,experimental hip-hop; rage; trap music,United States,"American rapper, singer-songwriter, and record producer (born 2000)" +Nate Smith,Q110707312,male,solo,country music; country rock,United States,"American country music singer, musician and songwriter" +Victoria Monét,Q19864989,female,solo,pop music; contemporary R&B; hip-hop,United States,American singer and songwriter (born 1989) +Tucker Wetmore,Q124754909,male,solo,country music,United States,American country music artist +Alex Warren,Q121842162,male,solo,pop music,United States,"American singer, songwriter, YouTuber, and influencer" +ROSÉ,Q27655344,female,solo,K-pop; dance-pop; hip-hop; electronic dance music,New Zealand,South Korean-New Zealand singer (born 1997) +Gracie Abrams,Q106301562,female,solo,bedroom pop; pop music,United States,American singer-songwriter (born 1999) +Lefty Gunplay,Q131374668,male,solo,hip-hop; West Coast hip-hop; gangsta rap,United States,American rapper +Leon Thomas,Q714375,male,solo,contemporary R&B; pop music; soul; hip-hop,United States,"American actor, singer, songwriter and producer" +EJAE,Q135108208,female,solo,K-pop; pop music; contemporary R&B,South Korea,South Korean-American singer and songwriter (born 1991) +Audrey Nuna,Q105538243,female,solo,K-pop; pop music; hip-hop; contemporary R&B; trap music,United States,South Korean-American singer and rapper +REI AMI,Q104842557,female,solo,K-pop; pop music; hip-hop; contemporary R&B,South Korea,South Korean singer and rapper +Lola Young,Q115869427,female,solo,pop music; soul; contemporary R&B,United Kingdom,British singer-songwriter (born 2001) +Ravyn Lenae,Q28962463,female,solo,contemporary R&B; alternative R&B; neo soul,United States,American singer +The Marías,Q85807852,,group,indie pop; psychedelic soul; dream pop; synth-pop,United States,American band +BigXthaPlug,Q124750337,male,solo,rapping; hip-hop,United States,American rapper +Riley Green,Q57418818,male,solo,country music,United States,American singer/songwriter +Gigi Perez,Q130235359,female,solo,indie folk,United States,American singer-songwriter +"Tyler, The Creator",Q167635,male,solo,alternative hip-hop; neo soul; West Coast hip-hop; jazz rap; hardcore hip-hop; horrorcore,United States,American rapper and record producer (born 1991) +sombr,Q133444448,male,solo,alternative pop; bedroom pop,United States,American singer-songwriter +Zach Top,Q126414316,male,solo,country music,United States,American singer/songwriter +Sam Barber,Q124428137,male,solo,country music,United States,American singer-songwriter +Avery Anna,Q131401753,female,solo,pop rock; country music,United States,American singer +AzChike,Q131417489,male,solo,hip-hop,United States,American rapper +Andrew Choi,Q15229870,male,solo,K-pop; pop music; contemporary R&B,South Korea,South Korean singer and songwriter +Neckwav,Q140814188,male,solo,contemporary R&B,South Korea,South Korean R&B singer +Danny Chung,Q140814119,male,solo,hip-hop; K-pop,United States,"Korean-American rapper, singer and songwriter" +Kevin Woo,Q6397789,male,solo,K-pop,United States,Korean-American singer +samUIL Lee,Q140814222,male,solo,J-pop,South Korea,"South Korean singer, songwriter and record producer" +Jimin,Q20947093,male,solo,K-pop,South Korea,South Korean singer and dancer (BTS) +Mariah the Scientist,Q84356665,female,solo,alternative R&B,United States,American singer (born 1997) +Ella Langley,Q127559972,female,solo,country music,United States,American country singer +Brandon Lake,Q100314823,male,solo,contemporary Christian music; Christian rock,United States,American Christian musician +PARTYNEXTDOOR,Q15929398,male,solo,Canadian hip-hop; alternative R&B,Canada,Canadian singer (born 1993) +Cash Cobain,Q125989634,male,solo,hip-hop; drill; Jersey drill; pop rap; East Coast hip-hop; trap music,United States,American rapper and record producer +Óscar Maydon,Q130241052,male,solo,corridos tumbados,Mexico,Mexican singer +Hudson Westbrook,Q134125831,male,solo,country music,United States,American singer +Kehlani,Q20657541,non-binary,solo,contemporary R&B; pop music; neo soul; hip-hop,United States,American singer-songwriter (born 1995) +Netón Vega,Q129208676,male,solo,regional Mexican; corridos tumbados,Mexico,Mexican singer-songwriter diff --git a/docs/decision_log.md b/docs/decision_log.md index 07d71c7..78f5561 100644 --- a/docs/decision_log.md +++ b/docs/decision_log.md @@ -155,6 +155,20 @@ Scientist、Surf Mesa）。官方拼法逐筆以 DSP 官方頁查證 （維基百科／Wikidata 的大小寫是其自家規範，不作準）； 查證亦確認 Tones And I 的 chart 印法即官方寫法，不改。 +- **`fetch-artists` 解析演算法定案**：改用 WDQS 的 SPARQL + 索引查詢（不依賴搜尋排名、不用 LCASE——實測會逾時）。 + 步驟：(1) 歌手名與 label／alias 精確比對（@en＋@mul 兩層） + 取候選，型態限人類∪音樂團體子樹∪original cast；(2) 唯一 + 即定案；(3) 歧義以上榜歌曲佐證——歌名精確比對取歌曲條目 + 的 P175 演出者（含其 P527 成員）與候選交集；(4) 再落空則 + 反向錨定：候選（含所屬團體）演出過的歌曲以 casefold 比對 + 歌名；(5) 均落空記查無並續行。每步須恰一命中才選定， + 實測零錯選。Last-resort 釘定表僅一筆（Pinkfong——唯一 + P31 為品牌的上榜演出者，型態閘門依設計排除）。暫時性 + 故障（429／5xx／讀取逾時）退避重試，重試盡記 error 列； + not found 專指程序查證後無所得。全量實跑 469 位全數 + 解析且與人工查證的 QID 逐筆相符；首次快照隨本次入 + 捕捉層。 - **先補完 Wikidata、再捕捉快照**：歌手資料查證做在上游—— 以 subagent（`artist-wikidata-updater` 定義檔）逐位查證 快照欄位（P21、P31、P136、P27／P495，新條目加 P106）並 diff --git a/docs/project_structure.md b/docs/project_structure.md index a4c9f7e..90c90d8 100644 --- a/docs/project_structure.md +++ b/docs/project_structure.md @@ -15,7 +15,7 @@ pop-fem-audit/ │ │ └── yearend_hot100_2016_2025.csv # 原始榜單 │ ├── captures/ # 外部捕捉：只由 fetch 命令與 │ │ │ # 私人匯入腳本寫入 -│ │ ├── artists_wikidata.csv # Wikidata 快照 +│ │ ├── artists-wikidata.csv # Wikidata 快照 │ │ ├── lyrics_provenance.csv # 歌詞出處 │ │ ├── lyrics_missing.csv # 歌詞缺漏報表 │ │ └── lyrics/ # 歌詞 .txt 快取 diff --git a/tools/src/pop_fem_audit_tools/fetch_artists.py b/tools/src/pop_fem_audit_tools/fetch_artists.py index d276f1b..fe50041 100644 --- a/tools/src/pop_fem_audit_tools/fetch_artists.py +++ b/tools/src/pop_fem_audit_tools/fetch_artists.py @@ -11,9 +11,10 @@ store is only read, never written; the ``build-db`` subcommand assembles the captured files into the store on the next rebuild. Every fetched row is meant for later human verification: the -description of the search hit is recorded in the note column so -that a bad match can be spotted. A search miss or an error on -one artist is noted on its row and does not fail the run. +description of the resolved item is recorded in the note column +so that a bad match can be spotted. An unresolved artist or an +error on one artist is noted on its row and does not fail the +run. """ import argparse import csv @@ -21,6 +22,7 @@ import enum import json import sys import time +import urllib.error import urllib.parse import urllib.request from collections.abc import Sequence @@ -31,26 +33,69 @@ from typing import Any, Literal import sqlalchemy as sa from sqlalchemy.orm import Session +from . import VERSION from .database import ds -from .models import Artist +from .models import Artist, Song, SongArtist API_URL: str = "https://www.wikidata.org/w/api.php" """The URL of the Wikidata API endpoint.""" -USER_AGENT: str = ("pop-fem-audit-tools" - " (https://github.com/imacat/pop-fem-audit)") +SPARQL_URL: str = "https://query.wikidata.org/sparql" +"""The URL of the Wikidata Query Service SPARQL endpoint.""" +USER_AGENT: str = ( + f"pop-fem-audit-tools/{VERSION}" + " (https://github.com/imacat/pop-fem-audit;" + " mailto:imacat@mail.imacat.idv.tw)") """The User-Agent header sent on every HTTP request.""" TIMEOUT: float = 30.0 -"""The timeout of an HTTP request, in seconds.""" +"""The timeout of an API HTTP request, in seconds.""" +SPARQL_TIMEOUT: float = 90.0 +"""The timeout of a SPARQL HTTP request, in seconds. + +Higher than the API timeout: the WDQS server aborts a slow +query at 60 seconds, and a lower client timeout would race +that server-side abort and misclassify a slow-but-answerable +query as a client-side timeout instead of letting the server's +own HTTP error response arrive and enter the retry path.""" SLEEP_SECONDS: float = 1.0 """The delay between consecutive HTTP requests, in seconds.""" +MAX_ATTEMPTS: int = 5 +"""The maximum number of attempts on a transient error.""" +RETRY_SECONDS: float = 15.0 +"""The back-off unit on a transient error, in seconds; +multiplied by the attempt number already made.""" +RETRY_STATUSES: frozenset[int] = frozenset({429, 500, 502, 503}) +"""The HTTP statuses that are retried with a back-off.""" +MAX_STAGE1_TITLES: int = 3 +"""The maximum number of charted titles used for the stage-1 song +corroboration.""" HUMAN_QID: str = "Q5" """The Wikidata item ID of "human".""" +ENSEMBLE_QID: str = "Q2088357" +"""The Wikidata item ID of "musical ensemble".""" +ORIGINAL_CAST_QID: str = "Q106497009" +"""The Wikidata item ID of "original cast".""" GROUP_KEYWORDS: Sequence[str] = ("band", "group", "duo", "trio") """The label keywords that suggest a musical ensemble, covering labels like "boy band" and "girl group".""" NOTE_NOT_FOUND: str = "not found" -"""The note sentinel of an artist without a Wikidata search hit, -written to the snapshot and read back for the classification.""" +"""The note sentinel of an artist without a resolved Wikidata +item, written to the snapshot and read back for the +classification.""" +PINNED_QIDS: dict[str, str] = { + "Pinkfong": "Q55735607", +} +"""The last-resort pinned item IDs, keyed by the artist name. + +Each entry is for an artist the algorithm documented on +``ArtistFetcher`` is structurally unable to resolve, with its +justification recorded here: + +- "Pinkfong": the only charting act whose item is typed as a + brand (P31 = Q431289), which the type gate (human / musical + ensemble / original cast) excludes by design. + +A pinned name skips the candidate retrieval and corroboration +steps; its item ID is used directly.""" class ArtistType(enum.StrEnum): @@ -84,7 +129,7 @@ class ArtistSnapshot: """The country label, or empty when unresolved.""" note: str = "" """The note for human verification: the description of the - search hit, ``not found``, or ``error: ``.""" + resolved item, ``not found``, or ``error: ``.""" def to_row(self) -> dict[str, str]: """Return this snapshot as a CSV row. @@ -101,7 +146,8 @@ SNAPSHOT_FIELDS: Sequence[str] = tuple( @dataclass class ArtistClaims: - """The item-ID claim targets of a Wikidata artist item.""" + """The item-ID claim targets and description of a Wikidata + artist item.""" gender_ids: list[str] = field(default_factory=list) """The item IDs of the gender targets.""" @@ -113,6 +159,17 @@ class ArtistClaims: """The item IDs of the country-of-citizenship targets.""" origin_country_ids: list[str] = field(default_factory=list) """The item IDs of the country-of-origin targets.""" + description: str = "" + """The English description of the item, or empty when + absent.""" + + +class RetryExhausted(Exception): + """The retries on a transient error are exhausted. + + A transient error is a retryable HTTP status (429, 500, + 502, or 503) or a read timeout. + """ def parse_args(argv: list[str] | None) -> argparse.Namespace: @@ -132,70 +189,228 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: class ArtistFetcher: - """A fetcher of artist metadata from Wikidata.""" + """A fetcher of artist metadata from Wikidata. + + An artist name is resolved to a Wikidata item ID through the + Wikidata Query Service (SPARQL), not the search API, so that + every step is a deterministic indexed lookup with no ranking + and no case folding of the label text itself: + + 1. Candidate retrieval: the items whose ``rdfs:label`` or + ``skos:altLabel`` exactly equals the artist name at + ``@en`` or ``@mul`` (the multilingual default layer), + restricted to a human (P31 = Q5), a musical ensemble + (P31/P279* = Q2088357), or an original cast + (P31 = Q106497009). + 2. A single candidate is selected outright. + 3. With multiple candidates, stage 1 corroborates with up to + the first 3 charted titles: the song items whose label or + alias exactly equals a title at ``@en`` or ``@mul`` are + looked up with their P175 performers and, optionally, + those performers' P527 members. The candidates are + intersected with the performers, and, only when that + intersection is empty, with the members; a stage succeeds + only when the intersection has exactly one item. + 4. Stage 2 is an anchored, case-insensitive fallback: every + song performed by a candidate, directly or via a parent + group, is compared against the charted titles with a + casefold match on the song label, without a language + restriction; a single matching candidate is selected. + 5. Zero candidates, or no stage narrowing to exactly one + item, leaves the artist unresolved. + + As a last resort, a name listed in ``PINNED_QIDS`` uses its + pinned item ID directly, skipping every step above. + """ def __init__(self) -> None: """Construct the fetcher.""" self.__sent: int = 0 """The number of the HTTP requests already sent.""" - def fetch(self, name: str) -> ArtistSnapshot: + def fetch(self, name: str, + titles: Sequence[str]) -> ArtistSnapshot: """Fetch the metadata of an artist. - The note carries the description of the search hit for - human verification. A search miss yields a snapshot with - the note ``not found``. An HTTP, network, or decoding - error yields a snapshot with what was resolved so far and - the note ``error: ``. + The note carries the description of the resolved item + for human verification. ``not found`` is reserved for + the algorithm genuinely finding nothing for the artist. + Any HTTP, network, or decoding error -- including the + retries on a transient error being exhausted -- yields a + snapshot with what was resolved so far and the note + ``error: ``. - :param name: The artist name to query with. + :param name: The artist name to resolve. + :param titles: The charted song titles credited to the + artist, used for the multi-candidate resolution. :return: The snapshot of the artist. """ snapshot: ArtistSnapshot = ArtistSnapshot(name=name) try: - hit: tuple[str, str] | None = self.__search(name) - if hit is None: + qid: str | None = self.__resolve_qid(name, titles) + if qid is None: snapshot.note = NOTE_NOT_FOUND return snapshot - snapshot.qid, snapshot.note = hit + snapshot.qid = qid self.__resolve(snapshot) - except (OSError, ValueError) as error: + except (RetryExhausted, OSError, ValueError) as error: snapshot.note = f"error: {error}" return snapshot - def __search(self, name: str) -> tuple[str, str] | None: - """Search Wikidata for an artist. + def __resolve_qid(self, name: str, + titles: Sequence[str]) -> str | None: + """Resolve an artist name to a Wikidata item ID. - :param name: The artist name to search for. - :return: The QID and the description of the first hit, or - None when there is no hit. - :raises OSError: On an HTTP or network error. + :param name: The artist name. + :param titles: The charted song titles credited to the + artist. + :return: The pinned item ID from ``PINNED_QIDS`` when the + name is listed there; otherwise the resolved item + ID, or None when the algorithm documented on this + class does not narrow to exactly one item. + :raises OSError: On a non-retryable HTTP or network + error. + :raises RetryExhausted: When the retries on a + transient error are exhausted. :raises ValueError: On a JSON decoding error. """ - data: Any = self.__get_json({ - "action": "wbsearchentities", "search": name, - "language": "en", "type": "item", "format": "json"}) - hits: Any = data.get("search") \ - if isinstance(data, dict) else None - if not isinstance(hits, list) or len(hits) == 0: + if name in PINNED_QIDS: + return PINNED_QIDS[name] + candidates: list[str] = self.__candidates(name) + if len(candidates) == 0: return None - hit: Any = hits[0] - if not isinstance(hit, dict) \ - or not isinstance(hit.get("id"), str): + if len(candidates) == 1: + return candidates[0] + selected: str | None = self.__stage1(candidates, titles) + if selected is not None: + return selected + return self.__stage2(candidates, titles) + + def __candidates(self, name: str) -> list[str]: + """Retrieve the Wikidata items matching an artist name. + + :param name: The artist name. + :return: The item IDs of the human, musical-ensemble, or + original-cast items whose label or alias exactly + equals the name. + :raises OSError: On a non-retryable HTTP or network + error. + :raises RetryExhausted: When the retries on a + transient error are exhausted. + :raises ValueError: On a JSON decoding error. + """ + query: str = f""" + SELECT DISTINCT ?item WHERE {{ + VALUES ?name {{ {self.__literals([name])} }} + {{ ?item rdfs:label ?name }} + UNION {{ ?item skos:altLabel ?name }} + {{ + ?item wdt:P31 wd:{HUMAN_QID} + }} UNION {{ + ?item wdt:P31/wdt:P279* wd:{ENSEMBLE_QID} + }} UNION {{ + ?item wdt:P31 wd:{ORIGINAL_CAST_QID} + }} + }} + """ + rows: list[dict[str, str]] = self.__sparql(query) + return [self.__qid(x["item"]) for x in rows + if "item" in x] + + def __stage1(self, candidates: Sequence[str], + titles: Sequence[str]) -> str | None: + """Corroborate the candidates with the charted titles. + + :param candidates: The candidate item IDs. + :param titles: The charted song titles credited to the + artist. + :return: The single candidate among the performers of a + matching song, or, only when no such single + candidate exists, among those performers' group + members; None when neither intersection has exactly + one item, or there are no titles to try. + :raises OSError: On a non-retryable HTTP or network + error. + :raises RetryExhausted: When the retries on a + transient error are exhausted. + :raises ValueError: On a JSON decoding error. + """ + subset: Sequence[str] = titles[:MAX_STAGE1_TITLES] + if len(subset) == 0: return None - description: Any = hit.get("description") - return hit["id"], \ - description if isinstance(description, str) else "" + query: str = f""" + SELECT DISTINCT ?performer ?member WHERE {{ + VALUES ?title {{ {self.__literals(subset)} }} + {{ ?song rdfs:label ?title }} + UNION {{ ?song skos:altLabel ?title }} + ?song wdt:P175 ?performer . + OPTIONAL {{ ?performer wdt:P527 ?member . }} + }} + """ + rows: list[dict[str, str]] = self.__sparql(query) + performers: set[str] = { + self.__qid(x["performer"]) for x in rows + if "performer" in x} + hit: set[str] = set(candidates) & performers + if len(hit) == 1: + return next(iter(hit)) + members: set[str] = { + self.__qid(x["member"]) for x in rows + if "member" in x} + hit = set(candidates) & members + if len(hit) == 1: + return next(iter(hit)) + return None + + def __stage2(self, candidates: Sequence[str], + titles: Sequence[str]) -> str | None: + """Rescue a single candidate by a case-insensitive match. + + :param candidates: The candidate item IDs. + :param titles: The charted song titles credited to the + artist. + :return: The single candidate with a charted song among + the songs it, or a parent group, performs, matched + case-insensitively against the song label; None when + no such single candidate exists. + :raises OSError: On a non-retryable HTTP or network + error. + :raises RetryExhausted: When the retries on a + transient error are exhausted. + :raises ValueError: On a JSON decoding error. + """ + values: str = " ".join(f"wd:{x}" for x in candidates) + query: str = f""" + SELECT DISTINCT ?cand ?label WHERE {{ + VALUES ?cand {{ {values} }} + {{ ?song wdt:P175 ?cand }} + UNION {{ ?g wdt:P527 ?cand . ?song wdt:P175 ?g }} + ?song rdfs:label ?label . + }} + """ + rows: list[dict[str, str]] = self.__sparql(query) + folded: set[str] = {x.casefold() for x in titles} + hits: set[str] = { + self.__qid(x["cand"]) for x in rows + if "cand" in x and "label" in x + and x["label"].casefold() in folded} + if len(hits) == 1: + return next(iter(hits)) + return None def __resolve(self, snapshot: ArtistSnapshot) -> None: """Resolve the claims of an artist into the snapshot. :param snapshot: The snapshot, with the QID set. :return: None. - :raises OSError: On an HTTP or network error. + :raises OSError: On a non-retryable HTTP or network + error. + :raises RetryExhausted: When the retries on a + transient error are exhausted. :raises ValueError: On a JSON decoding error. """ claims: ArtistClaims = self.__get_claims(snapshot.qid) + snapshot.note = claims.description country_ids: list[str] = claims.country_ids if len(country_ids) == 0: country_ids = claims.origin_country_ids @@ -212,30 +427,56 @@ class ArtistFetcher: snapshot.country = labels.get(country_ids[0], "") def __get_claims(self, qid: str) -> ArtistClaims: - """Fetch the item-ID claim targets of a Wikidata item. + """Fetch the claims and the description of a Wikidata + item. :param qid: The item ID. :return: The item-ID targets of the gender, instance-of, - genre, and country properties. - :raises OSError: On an HTTP or network error. + genre, and country properties, and the English + description. + :raises OSError: On a non-retryable HTTP or network + error. + :raises RetryExhausted: When the retries on a + transient error are exhausted. :raises ValueError: On a JSON decoding error. """ data: Any = self.__get_json({ "action": "wbgetentities", "ids": qid, - "props": "claims", "format": "json"}) - claims: Any = None + "props": "claims|descriptions", "languages": "en", + "format": "json"}) + entity: Any = None if isinstance(data, dict) \ - and isinstance(data.get("entities"), dict) \ - and isinstance(data["entities"].get(qid), dict): - claims = data["entities"][qid].get("claims") + and isinstance(data.get("entities"), dict): + entity = data["entities"].get(qid) + claims: Any = entity.get("claims") \ + if isinstance(entity, dict) else None if not isinstance(claims, dict): - return ArtistClaims() + claims = {} return ArtistClaims( gender_ids=self.__targets(claims.get("P21")), instance_of_ids=self.__targets(claims.get("P31")), genre_ids=self.__targets(claims.get("P136")), country_ids=self.__targets(claims.get("P27")), - origin_country_ids=self.__targets(claims.get("P495"))) + origin_country_ids=self.__targets(claims.get("P495")), + description=self.__description(entity)) + + @staticmethod + def __description(entity: Any) -> str: + """Extract the English description of a Wikidata entity. + + :param entity: The entity data, or None. + :return: The description, or the empty string when + absent. + """ + descriptions: Any = entity.get("descriptions") \ + if isinstance(entity, dict) else None + if not isinstance(descriptions, dict): + return "" + description: Any = descriptions.get("en") + if isinstance(description, dict) \ + and isinstance(description.get("value"), str): + return description["value"] + return "" @staticmethod def __targets(statements: Any) -> list[str]: @@ -270,7 +511,10 @@ class ArtistFetcher: :param qids: The item IDs, duplicates allowed. :return: The English labels, keyed by the item ID; the items without an English label are left out. - :raises OSError: On an HTTP or network error. + :raises OSError: On a non-retryable HTTP or network + error. + :raises RetryExhausted: When the retries on a + transient error are exhausted. :raises ValueError: On a JSON decoding error. """ unique: list[str] = list(dict.fromkeys(qids)) @@ -318,25 +562,146 @@ class ArtistFetcher: return ArtistType.GROUP return "" + def __sparql(self, query: str) -> list[dict[str, str]]: + """Run a SPARQL query against the Wikidata Query Service. + + :param query: The SPARQL query text. + :return: The result bindings, each variable name mapped + to its bound value. + :raises OSError: On a non-retryable HTTP or network + error. + :raises RetryExhausted: When the retries on a + transient error are exhausted. + :raises ValueError: On a JSON decoding error. + """ + url: str = (f"{SPARQL_URL}?" + f"{urllib.parse.urlencode({'query': query})}") + request: urllib.request.Request = urllib.request.Request( + url, headers={ + "User-Agent": USER_AGENT, + "Accept": "application/sparql-results+json"}) + body: bytes = self.__send( + request, timeout=SPARQL_TIMEOUT) + data: Any = json.loads(body) + bindings: Any = None + if isinstance(data, dict) \ + and isinstance(data.get("results"), dict): + bindings = data["results"].get("bindings") + if not isinstance(bindings, list): + return [] + rows: list[dict[str, str]] = [] + binding: Any + for binding in bindings: + if not isinstance(binding, dict): + continue + row: dict[str, str] = {} + key: str + cell: Any + for key, cell in binding.items(): + if isinstance(cell, dict) \ + and isinstance(cell.get("value"), str): + row[key] = cell["value"] + rows.append(row) + return rows + def __get_json(self, params: dict[str, str]) -> Any: """Send a GET request to the API and return the JSON body. - Consecutive requests are separated by a fixed delay. - :param params: The query parameters. :return: The parsed JSON body. - :raises OSError: On an HTTP or network error. + :raises OSError: On a non-retryable HTTP or network + error. + :raises RetryExhausted: When the retries on a + transient error are exhausted. :raises ValueError: On a JSON decoding error. """ + url: str = f"{API_URL}?{urllib.parse.urlencode(params)}" + request: urllib.request.Request = urllib.request.Request( + url, headers={"User-Agent": USER_AGENT}) + return json.loads(self.__send(request)) + + def __send(self, request: urllib.request.Request, + timeout: float = TIMEOUT) -> bytes: + """Send an HTTP request, retrying on a transient error. + + Consecutive requests are separated by a fixed delay. A + transient error -- a response with a retryable HTTP + status, or a read timeout -- is retried with an + increasing back-off, up to ``MAX_ATTEMPTS`` attempts in + total. + + :param request: The prepared HTTP request. + :param timeout: The read timeout, in seconds. + :return: The raw response body. + :raises OSError: On a non-retryable HTTP or network + error. + :raises RetryExhausted: When the retries on a + transient error are exhausted. + """ if self.__sent > 0: time.sleep(SLEEP_SECONDS) self.__sent += 1 - url: str = f"{API_URL}?{urllib.parse.urlencode(params)}" - request: urllib.request.Request = urllib.request.Request( - url, headers={"User-Agent": USER_AGENT}) - with urllib.request.urlopen( - request, timeout=TIMEOUT) as response: - return json.load(response) + attempt: int = 1 + reason: str + cause: BaseException + while True: + try: + with urllib.request.urlopen( + request, timeout=timeout) as response: + return response.read() + except urllib.error.HTTPError as error: + if error.code not in RETRY_STATUSES: + raise + reason = str(error) + cause = error + except TimeoutError as error: + reason = str(error) or "timed out" + cause = error + except urllib.error.URLError as error: + if not isinstance(error.reason, TimeoutError): + raise + reason = str(error.reason) or "timed out" + cause = error + if attempt >= MAX_ATTEMPTS: + raise RetryExhausted( + f"retries exhausted ({reason})") from cause + time.sleep(RETRY_SECONDS * attempt) + attempt += 1 + + @staticmethod + def __literals(texts: Sequence[str]) -> str: + """Build the SPARQL literals of texts at both languages. + + :param texts: The texts to embed as string literals. + :return: The literals, each text once tagged ``@en`` and + once tagged ``@mul``, space-separated. + """ + parts: list[str] = [] + text: str + for text in texts: + escaped: str = ArtistFetcher.__escape(text) + parts.append(f'"{escaped}"@en') + parts.append(f'"{escaped}"@mul') + return " ".join(parts) + + @staticmethod + def __escape(text: str) -> str: + """Escape a text for embedding as a SPARQL string literal. + + :param text: The text to embed. + :return: The text with the backslashes and double quotes + escaped. + """ + return text.replace("\\", "\\\\").replace('"', '\\"') + + @staticmethod + def __qid(uri: str) -> str: + """Extract the item ID from a Wikidata entity URI. + + :param uri: The entity URI. + :return: The item ID, the last path segment of the URI. + """ + return uri.rsplit("/", 1)[-1] def read_snapshot_names(path: Path) -> set[str]: @@ -355,6 +720,23 @@ def read_snapshot_names(path: Path) -> set[str]: return {x["name"] for x in reader} +def read_artist_titles(session: Session, + artist_id: int) -> list[str]: + """Read the charted song titles credited to an artist. + + :param session: The database session. + :param artist_id: The artist ID. + :return: The song titles credited to the artist, ordered by + the song ID, with the duplicate titles removed. + """ + titles: Sequence[str] = session.scalars( + sa.select(Song.title) + .join(SongArtist, SongArtist.song_id == Song.id) + .where(SongArtist.artist_id == artist_id) + .order_by(Song.id)).all() + return list(dict.fromkeys(titles)) + + def append_row(path: Path, snapshot: ArtistSnapshot) -> None: """Append a snapshot row to the snapshot CSV file. @@ -394,13 +776,16 @@ def main(argv: list[str] | None = None) -> int: session: Session = ds.get_db() try: done: set[str] = read_snapshot_names(args.wikidata_csv) - name: str - for name in session.scalars( - sa.select(Artist.name).order_by(Artist.id)): - if name in done: + artist: Artist + for artist in session.scalars( + sa.select(Artist).order_by(Artist.id)): + if artist.name in done: skipped += 1 continue - snapshot: ArtistSnapshot = fetcher.fetch(name) + titles: list[str] = read_artist_titles( + session, artist.id) + snapshot: ArtistSnapshot = fetcher.fetch( + artist.name, titles) append_row(args.wikidata_csv, snapshot) status: str = snapshot.qid if snapshot.note == NOTE_NOT_FOUND: @@ -411,7 +796,7 @@ def main(argv: list[str] | None = None) -> int: status = snapshot.note else: fetched += 1 - print(f"artist \"{name}\": {status}", + print(f"artist \"{artist.name}\": {status}", file=sys.stderr) except (OSError, sa.exc.SQLAlchemyError) as error: print(f"error: {error}", file=sys.stderr) diff --git a/tools/tests/test_fetch_artists.py b/tools/tests/test_fetch_artists.py index 52c8edc..e85f8c8 100644 --- a/tools/tests/test_fetch_artists.py +++ b/tools/tests/test_fetch_artists.py @@ -18,7 +18,7 @@ from sqlalchemy.orm import Session from pop_fem_audit_tools import config, fetch_artists from pop_fem_audit_tools.database import Base, DataSource -from pop_fem_audit_tools.models import Artist +from pop_fem_audit_tools.models import Artist, Role, Song, SongArtist class TestFetchArtists(unittest.TestCase): @@ -45,26 +45,49 @@ class TestFetchArtists(unittest.TestCase): patchers: list[Any] = [ mock.patch.object(fetch_artists, "ds", self.__ds), mock.patch.object(fetch_artists, "SLEEP_SECONDS", + 0.0), + mock.patch.object(fetch_artists, "RETRY_SECONDS", 0.0)] for patcher in patchers: patcher.start() self.addCleanup(patcher.stop) - def __seed(self, names: list[str]) -> None: + def __seed(self, names: list[str]) -> dict[str, int]: """Create the schema and the fixture artists. - The artist IDs are assigned in list order starting from - 1. - :param names: The artist names. - :return: None. + :return: The created artist IDs, keyed by the name. """ Base.metadata.create_all(self.__ds.engine) session: Session = self.__ds.get_db() + ids: dict[str, int] = {} try: name: str for name in names: - session.add(Artist(name=name)) + artist: Artist = Artist(name=name) + session.add(artist) + session.flush() + ids[name] = artist.id + session.commit() + finally: + session.close() + return ids + + def __seed_song(self, artist_id: int, title: str) -> None: + """Add a charted song credited to an artist. + + :param artist_id: The artist ID. + :param title: The song title. + :return: None. + """ + session: Session = self.__ds.get_db() + try: + song: Song = Song(title=title, artist_credit=title) + session.add(song) + session.flush() + session.add(SongArtist( + song_id=song.id, artist_id=artist_id, + role=Role.PRIMARY.value, position=0)) session.commit() finally: session.close() @@ -83,14 +106,14 @@ class TestFetchArtists(unittest.TestCase): return response @staticmethod - def __server_error() -> urllib.error.HTTPError: - """Build an HTTP 500 error. + def __http_error(status: int) -> urllib.error.HTTPError: + """Build an HTTP error. - :return: The HTTP 500 error. + :param status: The HTTP status code. + :return: The HTTP error. """ return urllib.error.HTTPError( - "https://example.com/", 500, - "Internal Server Error", None, None) + "https://example.com/", status, "Error", None, None) @staticmethod def __claim(qid: str) -> dict[str, Any]: @@ -113,6 +136,50 @@ class TestFetchArtists(unittest.TestCase): x: {"labels": {"en": {"value": y}}} for x, y in labels.items()}} + @staticmethod + def __claims(qid: str, claims: dict[str, Any], + description: str) -> dict[str, Any]: + """Build a claims-and-description response payload. + + :param qid: The item ID. + :param claims: The claim statements, keyed by the + property. + :param description: The English description. + :return: The response payload. + """ + return {"entities": {qid: { + "claims": claims, + "descriptions": {"en": {"value": description}}}}} + + @staticmethod + def __sparql(rows: list[dict[str, dict[str, str]]]) \ + -> dict[str, Any]: + """Build a SPARQL query response payload. + + :param rows: The result bindings. + :return: The response payload. + """ + return {"results": {"bindings": rows}} + + @staticmethod + def __uri(qid: str) -> dict[str, str]: + """Build a SPARQL URI binding cell for an item ID. + + :param qid: The item ID. + :return: The binding cell. + """ + return {"type": "uri", + "value": f"http://www.wikidata.org/entity/{qid}"} + + @staticmethod + def __literal(value: str) -> dict[str, str]: + """Build a SPARQL literal binding cell. + + :param value: The literal value. + :return: The binding cell. + """ + return {"type": "literal", "value": value} + def __run_fetch(self) -> tuple[int, str]: """Run the fetcher with the standard error captured. @@ -135,23 +202,25 @@ class TestFetchArtists(unittest.TestCase): with open(path, encoding="utf-8", newline="") as file: return list(csv.reader(file)) - def test_human_artist(self) -> None: - """Test a human artist resolving the full metadata.""" + def test_unique_candidate_selected(self) -> None: + """Test a single candidate resolving the full metadata.""" self.__seed(["Adele"]) - search: dict[str, Any] = {"search": [ - {"id": "Q1", "description": "English singer"}]} - claims: dict[str, Any] = {"entities": {"Q1": {"claims": { - "P21": [self.__claim("Q2")], - "P31": [self.__claim("Q5")], - "P136": [self.__claim("Q3"), self.__claim("Q4")], - "P27": [self.__claim("Q6")]}}}} + candidates: dict[str, Any] = self.__sparql( + [{"item": self.__uri("Q1")}]) + claims: dict[str, Any] = self.__claims( + "Q1", { + "P21": [self.__claim("Q2")], + "P31": [self.__claim("Q5")], + "P136": [self.__claim("Q3"), self.__claim("Q4")], + "P27": [self.__claim("Q6")]}, + "English singer") labels: dict[str, Any] = self.__labels({ "Q2": "female", "Q5": "human", "Q3": "pop", "Q4": "soul music", "Q6": "United Kingdom"}) urlopen: mock.Mock with mock.patch( "urllib.request.urlopen", - side_effect=[self.__response(search), + side_effect=[self.__response(candidates), self.__response(claims), self.__response(labels)]) as urlopen: status: int @@ -159,26 +228,14 @@ class TestFetchArtists(unittest.TestCase): status, stderr = self.__run_fetch() self.assertEqual(status, 0) self.assertEqual(urlopen.call_count, 3) - request: Any = urlopen.call_args_list[0][0][0] - self.assertEqual(request.get_header("User-agent"), + first: Any = urlopen.call_args_list[0][0][0] + self.assertEqual(first.get_header("User-agent"), fetch_artists.USER_AGENT) - urls: list[str] = [x[0][0].full_url - for x in urlopen.call_args_list] + self.assertTrue( + first.full_url.startswith(fetch_artists.SPARQL_URL)) self.assertEqual( - urls[0], - "https://www.wikidata.org/w/api.php" - "?action=wbsearchentities&search=Adele&language=en" - "&type=item&format=json") - self.assertEqual( - urls[1], - "https://www.wikidata.org/w/api.php" - "?action=wbgetentities&ids=Q1&props=claims" - "&format=json") - self.assertEqual( - urls[2], - "https://www.wikidata.org/w/api.php" - "?action=wbgetentities&ids=Q2%7CQ5%7CQ3%7CQ4%7CQ6" - "&props=labels&languages=en&format=json") + first.get_header("Accept"), + "application/sparql-results+json") rows: list[list[str]] = self.__read_rows( self.__snapshot) self.assertEqual(len(rows), 2) @@ -190,70 +247,223 @@ class TestFetchArtists(unittest.TestCase): "1 fetched, 0 not found, 0 errors, 0 skipped", stderr) - def test_band(self) -> None: - """Test a band resolving the group type and the origin.""" - self.__seed(["BTS"]) - search: dict[str, Any] = {"search": [ - {"id": "Q10", - "description": "South Korean boy band"}]} - claims: dict[str, Any] = {"entities": {"Q10": {"claims": { - "P31": [self.__claim("Q11")], - "P136": [self.__claim("Q12")], - "P495": [self.__claim("Q13")]}}}} - labels: dict[str, Any] = self.__labels({ - "Q11": "boy band", "Q12": "K-pop", - "Q13": "South Korea"}) - with mock.patch( - "urllib.request.urlopen", - side_effect=[self.__response(search), - self.__response(claims), - self.__response(labels)]): - status: int = self.__run_fetch()[0] - self.assertEqual(status, 0) - rows: list[list[str]] = self.__read_rows( - self.__snapshot) - self.assertEqual(len(rows), 2) - self.assertEqual(rows[1], [ - "BTS", "Q10", "", "group", "K-pop", "South Korea", - "South Korean boy band"]) - - def test_not_found(self) -> None: - """Test that a search miss writes a not-found row.""" - self.__seed(["Nobody"]) + def test_pinned_qid_skips_search(self) -> None: + """Test that a pinned name short-circuits the search.""" + self.__seed(["Pinkfong"]) + qid: str = fetch_artists.PINNED_QIDS["Pinkfong"] + claims: dict[str, Any] = self.__claims( + qid, {}, "South Korean children's brand") urlopen: mock.Mock with mock.patch( "urllib.request.urlopen", - side_effect=[self.__response({"search": []})] + side_effect=[self.__response(claims)]) as urlopen: + status: int = self.__run_fetch()[0] + self.assertEqual(status, 0) + self.assertEqual(urlopen.call_count, 1) + first: Any = urlopen.call_args_list[0][0][0] + self.assertTrue(first.full_url.startswith( + fetch_artists.API_URL)) + rows: list[list[str]] = self.__read_rows( + self.__snapshot) + self.assertEqual(rows[1], [ + "Pinkfong", qid, "", "", "", + "", "South Korean children's brand"]) + + def test_stage1_performer_intersection(self) -> None: + """Test the multi-candidate resolution via a performer.""" + ids: dict[str, int] = self.__seed(["Boyz"]) + self.__seed_song(ids["Boyz"], "Song A") + self.__seed_song(ids["Boyz"], "Song B") + candidates: dict[str, Any] = self.__sparql([ + {"item": self.__uri("Q1")}, + {"item": self.__uri("Q2")}]) + stage1: dict[str, Any] = self.__sparql( + [{"performer": self.__uri("Q2")}]) + claims: dict[str, Any] = self.__claims( + "Q2", {}, "A boy band") + urlopen: mock.Mock + with mock.patch( + "urllib.request.urlopen", + side_effect=[self.__response(candidates), + self.__response(stage1), + self.__response(claims)]) as urlopen: + status: int = self.__run_fetch()[0] + self.assertEqual(status, 0) + self.assertEqual(urlopen.call_count, 3) + rows: list[list[str]] = self.__read_rows( + self.__snapshot) + self.assertEqual(rows[1], [ + "Boyz", "Q2", "", "", "", "", "A boy band"]) + + def test_stage1_member_hop(self) -> None: + """Test the multi-candidate resolution via a member.""" + ids: dict[str, int] = self.__seed(["Trio"]) + self.__seed_song(ids["Trio"], "Track") + candidates: dict[str, Any] = self.__sparql([ + {"item": self.__uri("Q1")}, + {"item": self.__uri("Q2")}]) + stage1: dict[str, Any] = self.__sparql([{ + "performer": self.__uri("Q9"), + "member": self.__uri("Q2")}]) + claims: dict[str, Any] = self.__claims( + "Q2", {}, "A member of a group") + urlopen: mock.Mock + with mock.patch( + "urllib.request.urlopen", + side_effect=[self.__response(candidates), + self.__response(stage1), + self.__response(claims)]) as urlopen: + status: int = self.__run_fetch()[0] + self.assertEqual(status, 0) + self.assertEqual(urlopen.call_count, 3) + rows: list[list[str]] = self.__read_rows( + self.__snapshot) + self.assertEqual(rows[1], [ + "Trio", "Q2", "", "", "", "", "A member of a group"]) + + def test_stage2_casefold_rescue(self) -> None: + """Test the case-insensitive stage-2 rescue.""" + ids: dict[str, int] = self.__seed(["Case"]) + self.__seed_song(ids["Case"], "Song One") + candidates: dict[str, Any] = self.__sparql([ + {"item": self.__uri("Q1")}, + {"item": self.__uri("Q2")}]) + stage1: dict[str, Any] = self.__sparql([]) + stage2: dict[str, Any] = self.__sparql([{ + "cand": self.__uri("Q2"), + "label": self.__literal("song one")}]) + claims: dict[str, Any] = self.__claims( + "Q2", {}, "Rescued by casefold") + urlopen: mock.Mock + with mock.patch( + "urllib.request.urlopen", + side_effect=[self.__response(candidates), + self.__response(stage1), + self.__response(stage2), + self.__response(claims)]) as urlopen: + status: int = self.__run_fetch()[0] + self.assertEqual(status, 0) + self.assertEqual(urlopen.call_count, 4) + rows: list[list[str]] = self.__read_rows( + self.__snapshot) + self.assertEqual(rows[1], [ + "Case", "Q2", "", "", "", "", "Rescued by casefold"]) + + def test_unresolved_continues_to_next_artist(self) -> None: + """Test that an unresolved artist does not stop the run.""" + self.__seed(["Ambiguous", "Nobody"]) + candidates_ambiguous: dict[str, Any] = self.__sparql([ + {"item": self.__uri("Q1")}, + {"item": self.__uri("Q2")}]) + stage2_empty: dict[str, Any] = self.__sparql([]) + candidates_nobody: dict[str, Any] = self.__sparql([]) + urlopen: mock.Mock + with mock.patch( + "urllib.request.urlopen", + side_effect=[self.__response(candidates_ambiguous), + self.__response(stage2_empty), + self.__response(candidates_nobody)] ) as urlopen: status: int stderr: str status, stderr = self.__run_fetch() self.assertEqual(status, 0) - self.assertEqual(urlopen.call_count, 1) + self.assertEqual(urlopen.call_count, 3) rows: list[list[str]] = self.__read_rows( self.__snapshot) - self.assertEqual(len(rows), 2) - self.assertEqual(rows[0], self.HEADER) + self.assertEqual(len(rows), 3) self.assertEqual(rows[1], [ + "Ambiguous", "", "", "", "", "", "not found"]) + self.assertEqual(rows[2], [ "Nobody", "", "", "", "", "", "not found"]) self.assertIn( - "0 fetched, 1 not found, 0 errors, 0 skipped", + "0 fetched, 2 not found, 0 errors, 0 skipped", stderr) - def test_http_error_continues(self) -> None: - """Test that an HTTP error is noted and the run goes on.""" - self.__seed(["Broken", "Nobody"]) + def test_retry_429_then_success(self) -> None: + """Test a 429 retry followed by a successful request.""" + self.__seed(["Retry"]) + candidates: dict[str, Any] = self.__sparql([]) + urlopen: mock.Mock with mock.patch( "urllib.request.urlopen", - side_effect=[self.__server_error(), - self.__response({"search": []})]): + side_effect=[self.__http_error(429), + self.__response(candidates)] + ) as urlopen: status: int stderr: str status, stderr = self.__run_fetch() self.assertEqual(status, 0) + self.assertEqual(urlopen.call_count, 2) + rows: list[list[str]] = self.__read_rows( + self.__snapshot) + self.assertEqual(rows[1], [ + "Retry", "", "", "", "", "", "not found"]) + self.assertIn( + "0 fetched, 1 not found, 0 errors, 0 skipped", + stderr) + + def test_timeout_then_success(self) -> None: + """Test a read timeout retried into a successful request.""" + self.__seed(["SlowQuery"]) + candidates: dict[str, Any] = self.__sparql([]) + urlopen: mock.Mock + with mock.patch( + "urllib.request.urlopen", + side_effect=[TimeoutError("timed out"), + self.__response(candidates)] + ) as urlopen: + status: int + stderr: str + status, stderr = self.__run_fetch() + self.assertEqual(status, 0) + self.assertEqual(urlopen.call_count, 2) + rows: list[list[str]] = self.__read_rows( + self.__snapshot) + self.assertEqual(rows[1], [ + "SlowQuery", "", "", "", "", "", "not found"]) + self.assertIn( + "0 fetched, 1 not found, 0 errors, 0 skipped", + stderr) + + def test_retry_exhausted_is_error(self) -> None: + """Test that exhausted retries yield an error row.""" + self.__seed(["Exhausted"]) + urlopen: mock.Mock + with mock.patch( + "urllib.request.urlopen", + side_effect=[self.__http_error(503)] * 5) as urlopen: + status: int + stderr: str + status, stderr = self.__run_fetch() + self.assertEqual(status, 0) + self.assertEqual(urlopen.call_count, 5) + rows: list[list[str]] = self.__read_rows( + self.__snapshot) + self.assertEqual(rows[1][:2], ["Exhausted", ""]) + self.assertTrue(rows[1][6].startswith( + "error: retries exhausted")) + self.assertIn( + "0 fetched, 0 not found, 1 errors, 0 skipped", + stderr) + + def test_non_retryable_error_continues(self) -> None: + """Test that a non-retryable HTTP error is noted as an + error and does not stop the run.""" + self.__seed(["Broken", "Nobody"]) + urlopen: mock.Mock + with mock.patch( + "urllib.request.urlopen", + side_effect=[self.__http_error(404), + self.__response(self.__sparql([]))] + ) as urlopen: + status: int + stderr: str + status, stderr = self.__run_fetch() + self.assertEqual(status, 0) + self.assertEqual(urlopen.call_count, 2) rows: list[list[str]] = self.__read_rows( self.__snapshot) - self.assertEqual(len(rows), 3) self.assertEqual(rows[1][:2], ["Broken", ""]) self.assertTrue(rows[1][6].startswith("error: ")) self.assertEqual(rows[2], [ @@ -277,7 +487,7 @@ class TestFetchArtists(unittest.TestCase): urlopen: mock.Mock with mock.patch( "urllib.request.urlopen", - side_effect=[self.__response({"search": []})] + side_effect=[self.__response(self.__sparql([]))] ) as urlopen: status: int stderr: str