"""Build the self-contained report from committed evaluation evidence.""" import html import json from collections import Counter from pathlib import Path ROOT = Path(__file__).resolve().parent SITE = ROOT.parents[1] DATA = {name: json.loads((ROOT / f"{name}.json").read_text()) for name in ("haiku", "gemini")} JUDGMENTS = json.loads((ROOT / "judgments.json").read_text()) RUBRIC = json.loads((ROOT / "rubric.json").read_text()) LABELS = {"haiku": "Claude Haiku 4.5", "gemini": "Gemini 3 Flash Preview"} ARMS = {"default": "Default", "override": "Readable override"} COLORS = {"pass": "#337d67", "partial": "#b07b29", "fail": "#b34c40"} ESC = lambda value: html.escape(str(value), quote=True) def distribution(rows, key, arm): return Counter(row[key] for row in rows if row["arm"] == arm and row[key] != "not_applicable") def bar(counts): total = sum(counts.values()) segments = "".join(f'' for status in COLORS if counts[status]) caption = ", ".join(f"{counts[s]} {s}" for s in COLORS if counts[s]) return f'{ESC(caption)} · n={total}' def matrix(model=None): titles = [r for r in JUDGMENTS["titles"] if model is None or r["model_file"] == model] groups = [r for r in JUDGMENTS["groups"] if model is None or r["model_file"] == model] rows = [] for rule in RUBRIC["rules"]: key = rule["id"] source = groups if key in ("consistent", "variation") else titles cells = "".join(f'{bar(distribution(source, key, arm))}' for arm in ARMS) unit = "family groups" if source is groups else "titles" rows.append(f'{ESC(rule["name"])}{unit}{cells}') rows.append('Fidelity gateAccurate requested work' + "".join(f'{bar(distribution(titles, "fidelity", arm))}' for arm in ARMS) + '') return '
' + "".join(rows) + '
CriterionDefaultReadable override
' def pair(case_id, model, repeat=1, featured=False): case = next(c for c in DATA[model]["cases"] if c["id"] == case_id) records = {r["arm"]: r for r in DATA[model]["records"] if r["case_id"] == case_id and r["repeat"] == repeat} cards = [] for arm, label in ARMS.items(): record = records[arm] judgment = next(r for r in JUDGMENTS["titles"] if r["model_file"] == model and r["case_id"] == case_id and r["repeat"] == repeat and r["arm"] == arm) badges = " ".join(f'{key}: {judgment[key]}' for key in ("parse", "frontload", "fidelity")) cards.append(f'
{label}

{ESC(record["title"])}

{badges}

{ESC(judgment["reason"])}

{record["characters"]} characters · {record["latency_seconds"]:.2f}s
') request = f'
Full synthetic request · {len(case["input"]):,} characters
{ESC(case["input"])}
' return f'
{ESC(LABELS[model])} · run {repeat} · {ESC(case_id)}
{"".join(cards)}
{request}
' all_pairs = "".join(pair(c["id"], model, repeat) for model in DATA for repeat in (1, 2) for c in DATA[model]["cases"]) # Deliberately show improvements and regressions together, not just winning examples. featured = "".join(pair(c, model) for c, model in [("feedback-mixed", "gemini"), ("feedback-praise", "gemini"), ("auth-login", "haiku"), ("long-tail", "gemini")]) groups = [] for group in JUDGMENTS["groups"]: titles = [r["title"] for r in DATA[group["model_file"]]["records"] if r["repeat"] == group["repeat"] and r["arm"] == group["arm"] and next(c for c in DATA[group["model_file"]]["cases"] if c["id"] == r["case_id"])["family"] == group["family"]] groups.append(f'
{ESC(LABELS[group["model_file"]])} · {ARMS[group["arm"]]} · run {group["repeat"]} · {group["family"]} · consistency {group["consistent"]}, variation {group["variation"]}

{ESC(group["reason"])}

') protocol = "".join(f'
{ESC(r["name"])} · {ESC(r["scope"])}' + "".join(f'

{status.title()}: {ESC(r[status])}

' for status in ("pass", "partial", "fail") if status in r) + '
' for r in RUBRIC["rules"]) + '

The initial parse rule did not define partial explicitly. Its post-scoring clarification appears in the adjudication record above; the original rubric remains unchanged.

' embedded = json.dumps({"models": DATA, "judgments": JUDGMENTS, "rubric": RUBRIC, "original_judgments": json.loads((ROOT / "judgments-original.json").read_text()), "adjudication": json.loads((ROOT / "adjudication.json").read_text())}, ensure_ascii=False).replace(" Can a writing skill improve conversation titles? — Engel’s Notebook
An experiment in readable AI text

Better titles.
Some new friction.

A writing-skill override corrected wrong-task titles in this small sample. It did not improve every writing rule. Action-first phrasing sometimes pushed the topic back or ran into the existing 50-character output limit.

128real SDK generations
2 × 16 × 2models · cases · repeats / arm
5 rulesblinded AI rubric

Keep testing before changing the default. The override reduced fidelity failures from 4/64 to 2/64. Strict front-loading passes fell from 57/64 to 55/64. Both are observations from these prompts and models, not general performance guarantees.

Default vs a title-specific wrapper around the full readable-text skill. Both arms used the same uncapped SDK helper. Results are the displayed titles after SDK output processing. Audit every judgment ↗

Five rules do not add up to one score.

Green means a full rubric pass. Amber means partial. Red means a failure. Consistency and meaningful variation are judged across sibling title groups, so their denominators differ. Inapplicable cases are excluded, never counted as passes.

PassPartialFail
@@MATRIX@@

After the blind review: a cross-session audit found inconsistent grading of identical outputs and compatible abbreviations. I adjudicated those cases and two praise-attribution errors before publication. The chart uses the corrected grades. Original blind grades · Every correction and post-scoring clarification.

Combined descriptive counts; the model-specific results below show why pooling alone is insufficient. Fidelity is a separate accuracy check, not a sixth writing principle.

Claude Haiku 4.5 · 64 titles@@HAIKU_MATRIX@@
Gemini 3 Flash Preview · 64 titles@@GEMINI_MATRIX@@
Read the scoring rules@@PROTOCOL@@

Model difference: four default-arm fidelity failures came from Gemini. Haiku’s override added two fidelity failures by changing “summarize customer praise” into “praise the lessons.” The override’s front-loading count stayed level for Haiku and fell by two for Gemini.

The useful corrections have a cost.

Four contrasts, deliberately including regressions. These are exact first-repeat outputs, not rewritten examples. The complete two-repeat dataset follows below.

@@FEATURED@@

The output cap remains. Titles ended in an ellipsis in 2/64 default outputs and 5/64 override outputs. These are observed ellipsis endings, not instrumented raw-response truncation counts. The blind scorer judged the text a user would see.

The praise example moves “offline” later. The long-input example retains the correct topic in both arms, but the default changes “investigate” to “fix.” Removing the input cap and improving instruction fidelity solve different problems.

Inspect sibling collections · all 24 family-level judgments@@GROUPS@@

Full inputs are a separate, verified change.

The model comparison isolates prompt wording: both arms receive full input. It cannot measure the effect of removing the old input caps. Five separate deterministic regressions cover that SDK change.

Input caps before and after; output cap unchangedBefore: instruction overrides stop at 2000 characters and first-message input at 1000. After: both inputs remain complete. Both paths retain a 50-character title output cap.BeforeAfterOverride: reject above 2,000 charactersUser message: keep only first 1,000Override: complete instructionsUser message: complete requestTitle LLM → output cap: 50 charactersFallback behavior unchangedTitle LLM → output cap: 50 charactersProvider context limits still apply
Input preservation changes. Output constraints do not. generate_title_with_llm:83–185 · ConversationConfig.prompt
  • Four real SDK HTTP-transport cases: default prompt, placeholder substitution, appended message, and a long override.
  • One real configuration/schema round-trip case with an override longer than 2,000 characters.
  • All five fail against the old source and pass with the patch. The title and server-span suites total 27 passing tests.

Regression tests:509–553 · SDK PR #6337 · Paired docs #1053

Every title. No winning-example filter.

All 64 matched pairs are below: 16 synthetic requests, two models, two repeats. Each pair shows the same request under both prompts. Expand any request to inspect what the title should preserve.

64 pairs
@@ALL_PAIRS@@

A small, inspectable experiment.

Generation ran on 30 October 2026 through the actual Python SDK generate_title_with_llm helper, not a reconstructed prompt-only mock. The 2,855-character override contains the skill body and a title-specific wrapper. Arms alternate order by case and repeat. No fallback title is counted as a successful generation.

  • Models: litellm_proxy/anthropic/claude-haiku-4-5 and litellm_proxy/gemini-3-flash-preview. These are requested aliases, not independently verified provider model versions.
  • Settings: temperature configured as 0, reasoning effort none, 256 output tokens, 50 title characters, 45-second timeout, one configured retry. Provider/model handling may omit unsupported sampling parameters. Anthropic legacy thinking was disabled for Haiku; Gemini’s unused Anthropic-specific budget retained its default.
  • Judge: Codex / gpt-6-astra, two separate sessions. Each received shuffled titles, original requests, and the prewritten rubric. Arm labels, model identities, and generation prompts were hidden. The sessions used the same scorer model, not independent human raters.
  • Scope: 16 hand-chosen synthetic requests. Three auth siblings, three upgrade siblings, two filler-heavy requests, three feedback sentiments, comparison, negation, uncertainty, completed-work status, and one long-tail request. No private conversations.

Limits: two low-temperature repeats are not independent user studies. This convenience sample does not measure reading speed, preference, or statistical significance. The wrapper adds title-specific constraints as well as the skill, so the comparison evaluates that combined override. Emoji constraints differ too: the baseline offers a fixed category list; the override asks for a relevant emoji. The fixed corpus was not used to tune and rerun the successful prompt.

Setup failures are retained: nine failed calls preceded the successful comparison: six Haiku requests hit a too-small manual-thinking budget; three requests used an unavailable Gemini 2.5 alias. Those calls produced no titles and are excluded from the 128-title quality denominators. One earlier configuration-only run failed before an API call. Setup-failure records ↗

Exact override · 2,855 characters
@@OVERRIDE@@

The original metadata names the base plus uncommitted patch at generation time. That source was later committed as 01bb167; the datasets retain both the original label and the verified commit. Both arms share the SDK system message; the override replaces only its user-message template. Exact default and system prompts ↗

Evidence and reproduction

Recommendation: preserve the full input, then refine the override’s topic-first wording and brevity in a new, separately labeled experiment. These results do not justify silently replacing the SDK default.

''' replacements = {"MATRIX": matrix(), "HAIKU_MATRIX": matrix("haiku"), "GEMINI_MATRIX": matrix("gemini"), "PROTOCOL": protocol, "FEATURED": featured, "GROUPS": "".join(groups), "ALL_PAIRS": all_pairs, "OVERRIDE": ESC(DATA["haiku"]["override"]), "EVIDENCE": embedded} for key, value in replacements.items(): page = page.replace(f"@@{key}@@", value) assert "@@" not in page output = SITE / "arch/readable-title-evaluation.html" output.parent.mkdir(exist_ok=True) output.write_text(page) print(output, len(page))