imacat/pop-fem-audit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Replace the quote newline escape with the lyric line-break convention

Browse Source 
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
imacat
2026-08-17 22:38:40 +08:00
co-authored by Claude Fable 5
parent c3d6c06910
commit 7917d23d6e
8 changed files with 1944 additions and 1946 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/src/pop_fem_audit_tools/commands/build_db.py
+6 -8
View File
@@ -690,9 +690,6 @@ class CodingImporter:
COLUMNS: tuple[str, ...] = (
"Song", "Artist Credit", "Keyword", "Quote")
"""The required columns of the coding CSV file."""
NEWLINE_ESCAPE: str = "\\n"
"""The two characters standing for a newline in the quote
column, so that the CSV file is one row per line."""
def __init__(self, session: Session) -> None:
"""Initialize the importer.
@@ -707,10 +704,11 @@ class CodingImporter:
A None input leaves the coding unloaded. Otherwise every
row of the CSV file yields one coding of the song named by
its title and artist credit, with the quote column stored
verbatim except that each two-character ``\\n`` escape
becomes a newline; an empty quote column stores an empty
string. When the method returns, the imported codings are
queryable in the session.
verbatim; a quote carries the lyric line-break convention
``" / "`` where the lyric has a line break, and an empty
quote column stores an empty string. When the method
returns, the imported codings are queryable in the
session.
:param path: The settled coding table CSV file to import,
or None to skip the coding.
@@ -766,7 +764,7 @@ class CodingImporter:
seen.add(coding_key)
self.__session.add(Coding(
song=song, keyword=row["Keyword"],
quotes=row["Quote"].replace(self.NEWLINE_ESCAPE, "\n")))
quotes=row["Quote"]))
@classmethod
def __check_columns(cls, path: Path,
tools/src/pop_fem_audit_tools/commands/tally_codings.py
+34 -55
View File
@@ -15,10 +15,13 @@ three runs assign it, so three votes never tie, and it carries
the lyric quotes of every run that assigned it, pooled,
deduplicated, sorted by Unicode code point, and joined with a
single ``|``: the three runs are peers, so the quote order
follows the text alone. The written quote carries the two
characters ``\n`` where the lyric has a newline, the mirror of
the unescaping the correction table loader does, so the coding
table holds one row per line. The three
follows the text alone. A quote carries the lyric line-break
convention ``" / "`` where the lyric has a newline, applied once
when the run records load, so the corrections, the coding table,
and the database all share the one representation and nothing is
ever converted back. No lyric of the 883-song corpus contains
``" / "`` -- a corpus fact checked exhaustively, not a structural
guarantee -- so the convention is unambiguous here. The three
archives must cover exactly the same set of song IDs, every
record must be a successful result, and every record's "text"
must parse to a JSON object; otherwise the tally fails and
@@ -161,11 +164,10 @@ class CorrectionsLoader:
of the runs the command was given, and a known type. The
file is read with the CSV reader, so a quoted field may
hold a comma or a double quote. No field holds a line
break: the two text fields carry the two characters
``\n`` where the text has a newline, so the file holds one
row per line, and each of them is unescaped to a single LF
before the repair is matched or applied, as that is what
the archived records carry. Nothing is written.
break: the two text fields carry the lyric line-break
convention ``" / "`` where the text has a newline -- the
same representation the loaded run records carry -- and
are matched and applied verbatim. Nothing is written.
:return: The repairs, in file order.
:raises TallyError: When the file cannot be read, the
@@ -249,21 +251,7 @@ class CorrectionsLoader:
raise ValueError(f"{label}: unknown type \"{row[2]}\"")
return Correction(
line=line, song_id=song_id, run=row[1], type=row[2],
to_be_replaced=cls.__unescape(row[3]),
correct_term=cls.__unescape(row[4]))
@staticmethod
def __unescape(text: str) -> str:
r"""Unescape the newlines of a text field.
The mirror of the escaping :meth:`CodingTable.write` does
to the ``Quote`` field it writes out.
:param text: The field as the CSV reader gave it.
:return: The field with every two-character ``\n``
sequence turned into a single LF.
"""
return text.replace("\\n", "\n")
to_be_replaced=row[3], correct_term=row[4])
@staticmethod
def __parse_song_id(item_id: str, label: str) -> int:
@@ -361,6 +349,11 @@ class CodingTallier:
__QUOTE_SEPARATOR: str = "|"
"""The separator between the distinct lyric quotes of one
settled code."""
__LINE_BREAK: str = " / "
"""The lyric line-break convention replacing every LF inside a
quote. Unambiguous for this corpus only: none of the 883
songs' lyrics contains the three characters, checked
exhaustively; a new corpus must be re-checked."""
def __init__(self, run_dir_1: Path, run_dir_2: Path,
run_dir_3: Path,
@@ -645,8 +638,8 @@ class CodingTallier:
keywords, f"{path}: id {item_id}")
return records
@staticmethod
def __quote_lists(keywords: dict[str, Any], label: str) \
@classmethod
def __quote_lists(cls, keywords: dict[str, Any], label: str) \
-> dict[str, list[str]]:
"""Validate the lyric quote list of every keyword.
@@ -654,7 +647,8 @@ class CodingTallier:
:param label: The location of the record, for the error
message.
:return: The lyric quotes of every keyword, in the given
order.
order, every LF inside a quote turned into the lyric
line-break convention ``" / "``.
:raises ValueError: When a keyword's value is not a list
of strings.
"""
@@ -667,7 +661,8 @@ class CodingTallier:
raise ValueError(
f"{label}: keyword \"{keyword}\": the quotes"
" are not a list of strings")
quotes[keyword] = value
quotes[keyword] = [
x.replace("\n", cls.__LINE_BREAK) for x in value]
return quotes
@classmethod
@@ -832,18 +827,17 @@ class CodingTable:
keyword, by Unicode code point."""
def write(self, output_csv: Path) -> None:
r"""Write the coding table CSV file.
"""Write the coding table CSV file.
Writes an RFC 4180 CSV file, UTF-8, with CRLF line
endings, carrying the header row
``Song,Artist Credit,Keyword,Quote`` and one row per
settled keyword, in the row order. The ``Quote`` field
carries the two characters ``\n`` where the quotes have a
newline, so no field holds a line break and the file
holds one row per line; the correction table loader
unescapes its own two text fields the same way, and
restoring a single LF gives the lyric text back. The
parent directory is created when it does not exist.
settled keyword, in the row order. Every field is
written verbatim; a quote carries the lyric line-break
convention ``" / "`` where the lyric has a line break, so
no field holds a line break and the file holds one row
per line. The parent directory is created when it does
not exist.
:param output_csv: The output CSV file.
:return: None.
@@ -854,22 +848,7 @@ class CodingTable:
newline="") as file:
writer: Any = csv.writer(file)
writer.writerow(self.__HEADER)
writer.writerows(
(*x[:3], self.__escape(x[3])) for x in self.rows)
@staticmethod
def __escape(text: str) -> str:
r"""Escape the newlines of the quote field.
The mirror of the unescaping the correction table loader
does to its own two text fields.
:param text: The joined lyric quotes of one settled
keyword.
:return: The quotes with every LF turned into the two
characters ``\n``.
"""
return text.replace("\n", "\\n")
writer.writerows(self.rows)
class CodingTableBuilder:
@@ -1017,9 +996,9 @@ def main(argv: list[str] | None = None) -> int:
song named by its title and its stored artist credit from the
SQLite working store, and the keyword carrying the pooled,
deduplicated, and sorted lyric quotes of the runs that
assigned it, joined with a single ``|`` and written with the
two characters ``\n`` where the quotes have a newline, so the
table holds one row per line. The records are
assigned it, joined with a single ``|`` and carrying the
lyric line-break convention ``" / "`` where the lyric has a
newline, so the table holds one row per line. The records are
repaired from the ``--corrections`` table and then checked
against the ``--valid-keywords`` list, when either is given.
Nothing is written when the three archives do not cover the
tools/tests/test_build_db.py
+7 -7
View File
@@ -984,12 +984,13 @@ class TestBuildDB(unittest.TestCase):
CODINGS_CSV: str = (
"Song,Artist Credit,Keyword,Quote\n"
"Hello,Adele,longing,\"Hello from the other side\\nI must"
"Hello,Adele,longing,\"Hello from the other side / I must"
" have called a thousand times\"\n"
"Hello,Adele,regret,I'm sorry|It's no secret\n"
"One Dance,Drake featuring Wizkid,desire,\n")
"""The coding CSV fixture: a two-line quote, two quotes joined
by "|", and an empty quote."""
"""The coding CSV fixture: a quote carrying the " / "
line-break convention, two quotes joined by "|", and an empty
quote."""
def __stored_codings(self) -> dict[tuple[str, str], str]:
"""Read the stored codings keyed by the song and keyword.
@@ -1003,8 +1004,7 @@ class TestBuildDB(unittest.TestCase):
def test_codings_imported(self) -> None:
"""Test that the coding CSV imports one row per song and
keyword, restoring the newline and keeping the "|"-joined
quotes verbatim."""
keyword, storing the quote column verbatim."""
self.__write_codings(self.CODINGS_CSV)
status: int
stderr: str
@@ -1015,8 +1015,8 @@ class TestBuildDB(unittest.TestCase):
self.assertEqual(
self.__stored_codings(),
{("Hello", "longing"):
"Hello from the other side\n"
"I must have called a thousand times",
"Hello from the other side / I must have called"
" a thousand times",
("Hello", "regret"): "I'm sorry|It's no secret",
("One Dance", "desire"): ""})
tools/tests/test_tally_codings.py
+20 -21
View File
@@ -251,11 +251,10 @@ class TestTallyCodings(unittest.TestCase):
b"Alpha,A Singer,kw,"
b"\"she said \"\"no\"\", twice\"\r\n")
def test_newline_in_quote_written_as_two_characters(
self) -> None:
r"""Test that a quote spanning two lyric lines is written
with the two characters ``\n`` where the newline is,
needing no RFC 4180 quoting."""
def test_newline_in_quote_written_as_line_break(self) -> None:
"""Test that a quote spanning two lyric lines is written
with the lyric line-break convention " / " where the
newline is, needing no RFC 4180 quoting."""
self.__seed([("Alpha", "A Singer")])
codings: dict[int, dict[str, list[str]]] = {
1: {"kw": ["You needed me\nTo feel a little more"]}}
@@ -267,13 +266,12 @@ class TestTallyCodings(unittest.TestCase):
self.__output_csv.read_bytes(),
b"Song,Artist Credit,Keyword,Quote\r\n"
b"Alpha,A Singer,kw,"
b"You needed me\\nTo feel a little more\r\n")
b"You needed me / To feel a little more\r\n")
def test_multi_line_quotes_keep_one_row_per_line(self) -> None:
r"""Test that the file holds exactly one line per row, no
field carrying a line break, and that turning the two
characters ``\n`` back into a single LF gives the quotes
as the runs wrote them."""
"""Test that the file holds exactly one line per row, no
field carrying a line break, every LF inside a quote
carried as the line-break convention " / " verbatim."""
self.__seed([("Alpha", "A Singer")])
codings: dict[int, dict[str, list[str]]] = {
1: {"kw1": ["one line"],
@@ -289,8 +287,9 @@ class TestTallyCodings(unittest.TestCase):
rows: list[list[str]] = self.__read_rows()
self.assertEqual(len(rows), 4)
self.assertEqual(
[x[3].replace("\\n", "\n") for x in rows[1:]],
["one line", "first line\nsecond line", "a\nb\nc|d\ne"])
[x[3] for x in rows[1:]],
["one line", "first line / second line",
"a / b / c|d / e"])
def test_empty_quote_lists_yield_an_empty_cell(self) -> None:
"""Test that a settled keyword whose runs all gave an empty
@@ -711,11 +710,11 @@ class TestTallyCodings(unittest.TestCase):
self.assertEqual(self.__read_rows()[1:], [
["Alpha", "A Singer", "kw", ""]])
def test_correction_with_an_escaped_newline(self) -> None:
r"""Test that a correction whose two text fields carry the
two characters ``\n`` matches and replaces a quote that
genuinely spans two lines, the file itself holding one row
per line."""
def test_correction_with_a_line_break_quote(self) -> None:
"""Test that a correction whose two text fields carry the
line-break convention " / " matches and replaces a quote
that genuinely spans two lines, the file itself holding
one row per line."""
self.__seed([("Alpha", "A Singer")])
quote: str = "You needed me\nTo feel a little more"
codings: dict[int, dict[str, list[str]]] \
@@ -725,16 +724,16 @@ class TestTallyCodings(unittest.TestCase):
path.write_bytes(
b"Song ID,Run,Type,To Be Replaced,Correct Term\r\n"
b"song-1,run1,evidence,"
b"You needed me\\nTo feel a little more,"
b"you needed me\\nTo feel a little more\r\n")
b"You needed me / To feel a little more,"
b"you needed me / To feel a little more\r\n")
self.assertEqual(len(path.read_bytes().split(b"\r\n")), 3)
status: int
status, _ = self.__run_tally("--corrections", str(path))
self.assertEqual(status, 0)
self.assertEqual(self.__read_rows()[1:], [
["Alpha", "A Singer", "kw",
"You needed me\\nTo feel a little more"
"|you needed me\\nTo feel a little more"]])
"You needed me / To feel a little more"
"|you needed me / To feel a little more"]])
def test_stale_correction_row_rejected(self) -> None:
"""Test that a correction matching nothing fails the run,