Add the export-llm-input subcommand for the lyrics-only LLM input

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-17 22:38:26 +08:00
co-authored by Claude Fable 5
parent 55926b8dc4
commit 396848e898
7 changed files with 279 additions and 0 deletions
@@ -17,6 +17,7 @@ from types import ModuleType
from pop_fem_audit_tools import (
build_db,
export_llm_input,
fetch_artists,
fetch_lyrics,
run_llm,
@@ -27,6 +28,7 @@ MODULE_PROG: str = "python -m pop_fem_audit_tools"
SUBCOMMANDS: dict[str, Callable[[list[str] | None], int]] = {
"build-db": build_db.main,
"export-llm-input": export_llm_input.main,
"fetch-artists": fetch_artists.main,
"fetch-lyrics": fetch_lyrics.main,
"run-llm": run_llm.main,
@@ -0,0 +1,84 @@
# Tools for A Feminist Audit of Pop Music.
# Copyright 2026 imacat. All rights reserved.
# Authors:
# imacat@mail.imacat.idv.tw (imacat), 2026/8/4
"""The exporter of the LLM input JSONL file.
Exports the songs from the SQLite working store into the JSONL
input file for the ``run-llm`` subcommand, given as the positional
command-line argument. This is the enforcement point of the
project's lyrics-only firewall: the output carries only the
lyrics text of each song, identified by an opaque song key; no
title, artist, or chart data crosses into the LLM input.
"""
import argparse
import json
import sys
from pathlib import Path
import sqlalchemy as sa
from sqlalchemy.orm import Session
from .database import ds
from .models import Song
def parse_args(argv: list[str] | None) -> argparse.Namespace:
"""Parse the command-line arguments.
:param argv: The command-line arguments, or None for
``sys.argv``.
:return: The parsed arguments.
"""
parser: argparse.ArgumentParser = argparse.ArgumentParser(
description="Export the LLM input JSONL file (lyrics"
" only) from the SQLite working store.")
parser.add_argument(
"output_jsonl", type=Path,
help="the JSONL output file")
return parser.parse_args(argv)
def build_lines(session: Session) -> list[str]:
"""Build the JSONL lines of every song's lyrics.
:param session: The database session.
:return: The JSON lines, one per song, ordered by song ID.
:raises ValueError: When a song has no lyrics.
"""
lines: list[str] = []
song: Song
for song in session.scalars(sa.select(Song).order_by(Song.id)):
if song.lyrics is None:
raise ValueError(
f"song {song.id} \"{song.title}\": no lyrics")
record: dict[str, str] = {
"id": f"song-{song.id}", "content": song.lyrics}
lines.append(json.dumps(record, ensure_ascii=False))
return lines
def main(argv: list[str] | None = None) -> int:
"""Export the LLM input JSONL file from the working store.
:param argv: The command-line arguments, or None for
``sys.argv``.
:return: The exit status: 0 on success, non-zero on failure.
"""
args: argparse.Namespace = parse_args(argv)
session: Session = ds.get_db()
lines: list[str]
try:
lines = build_lines(session)
except (OSError, sa.exc.SQLAlchemyError, ValueError) as error:
print(f"error: {error}", file=sys.stderr)
return 1
finally:
session.close()
args.output_jsonl.parent.mkdir(parents=True, exist_ok=True)
with open(args.output_jsonl, "w", encoding="utf-8") as file:
line: str
for line in lines:
file.write(line + "\n")
print(f"done: {len(lines)} songs exported", file=sys.stderr)
return 0