"""Build the separately labeled follow-up report, without changing study one.""" import html import json from collections import Counter from pathlib import Path ROOT = Path(__file__).resolve().parent SITE = ROOT.parents[1] P = json.loads((ROOT / "protocol.json").read_text()) D = {n: json.loads((ROOT / f"{n}.json").read_text()) for n in P["models"]} J = json.loads((ROOT / "judgments.json").read_text()) A = {"default": "Default", "previous": "Previous override", "topic_first": "Topic-first override"} N = {"haiku": "Haiku 4.5", "gemini": "Gemini 3 Flash Preview", "luna": "GPT-6 Luna · low"} E = lambda x: html.escape(str(x), quote=True) CASES = {c["id"]: c for c in P["cases"]} LOOKUP = {(r["case_id"], r["title"]): r for r in J["titles"]} def rows(split="all", model="all"): result = [] for n, data in D.items(): if model != "all" and n != model: continue for r in data["records"]: if split != "all" and r["split"] != split: continue if not r["title"]: continue result.append({**LOOKUP[(r["case_id"], r["title"])], "arm": r["arm"], "model": n}) return result def meter(counts): total = sum(counts.values()) if not total: return "—" parts = "".join(f'' for k, v in counts.items() if k != "not_applicable") text = " · ".join(f"{counts[k]} {k}" for k in ("pass", "partial", "fail") if counts[k]) return f'{E(text)} / {total}' def table(split="all", model="all"): title_rows = rows(split, model) criteria = [(r["id"], r["name"]) for r in P["rubric"]["rules"] if r["id"] not in ("consistent", "variation")] + [("fidelity", "Requested-work fidelity")] output = [] for key, label in criteria: cells = [] for arm in A: counts = Counter(r[key] for r in title_rows if r["arm"] == arm and r[key] != "not_applicable") cells.append("" + meter(counts) + "") output.append(f"{E(label)}{''.join(cells)}") return '
' + "".join(f"" for a in A.values()) + '' + "".join(output) + '
Per-title criterion{a}
' def group_table(): output = [] for key, label in (("consistent", "Consistent structure"), ("variation", "Meaningful variation")): cells = [] for arm in A: counts = Counter(g[key] for g in J["groups"] if g["arm"] == arm and g[key] != "not_applicable") cells.append('' + meter(counts) + '') output.append(f'{label}{"".join(cells)}') return '
' + "".join(f'' for a in A.values()) + '' + "".join(output) + '
Family-level criterion{a}
' triplets = [] for model, data in D.items(): for repeat in (1, 2): for case in P["cases"]: cells = [] for arm in A: r = next(r for r in data["records"] if r["case_id"] == case["id"] and r["repeat"] == repeat and r["arm"] == arm) if r["title"]: j = LOOKUP[(r["case_id"], r["title"])] reason = j["reason"] badges = " ".join(f'{k}: {j[k]}' for k in ("frontload", "fidelity")) else: reason = "No usable displayed title." + (" Error: " + ", ".join(r["errors"]) if r["errors"] else " The helper returned None without an exception callback.") badges = 'Failed generation' cells.append(f'
{A[arm]}

{E(r["title"] or "No title")}

{badges}

{E(reason)}

{r["characters"]} characters · {r["latency_seconds"]:.2f}s
') triplets.append(f'
{N[model]} · run {repeat} · {case["id"]} · {case["split"]}
{"".join(cells)}
Original synthetic request
{E(case["input"])}
') all_rows = rows() summary = {} for arm in A: rr = [r for r in all_rows if r["arm"] == arm] summary[arm] = {"fidelity_fail": sum(r["fidelity"] == "fail" for r in rr), "frontload_pass": sum(r["frontload"] == "pass" for r in rr), "n": len(rr), "attempts": sum(r["arm"] == arm for d in D.values() for r in d["records"]), "ellipsis": sum(r["title"].endswith("...") for d in D.values() for r in d["records"] if r["arm"] == arm and r["title"])} (ROOT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") summary_cards = "".join(f'
{A[arm]}
{s["frontload_pass"]}/{s["n"]}front-loading passes

{s["fidelity_fail"]} fidelity failures · {s["ellipsis"]} ellipsis endings · {s["attempts"]-s["n"]} missing titles

' for arm, s in summary.items()) model_tables = "".join(f'
{name} · all cases{table(model=n)}
' for n, name in N.items()) prompts = "".join(f'
{A[arm]} · {len(p) if p else 0} custom-prompt characters
{E(p or "No override: the unchanged SDK default user prompt applies.")}
' for arm, p in P["prompts"].items()) groups = "".join(f'
{N[g["model"]]} · {A[g["arm"]]} · run {g["repeat"]} · {g["family"]}

Consistency: {g["consistent"]} · variation: {g["variation"]}

{E(g["reason"])}

' for g in J["groups"]) evidence = json.dumps({"protocol": P, "models": D, "judgments": J, "original_judgments": json.loads((ROOT / "judgments-original.json").read_text()), "adjudication": json.loads((ROOT / "adjudication.json").read_text()), "scoring_titles": (ROOT / "scoring-titles.txt").read_text(), "scoring_groups": (ROOT / "scoring-groups.txt").read_text(), "reproduction_notes": (ROOT / "reproduce.txt").read_text()}, ensure_ascii=False).replace(" Topic-first titles: the follow-up — Engel’s Notebook
Follow-up experiment · 30 October 2026

Put the topic first.
Then check the meaning.

The shorter topic-first prompt improved front-loading in this sample, with fewer observed ellipsis endings. It still produced one missing title and did not make every collection consistent. Three models, three prompts, 24 requests. This follow-up tests a shorter topic-first override against both the SDK default and the previous action-first override. GPT-6 Luna joins through a ChatGPT subscription with low reasoning requested.

Read the unchanged first experiment ↗

Front-loading improves. Collection structure still varies.

432 generation attempts: 24 cases × three prompts × two repeats × three models. 431 returned a displayed title. Gemini returned no usable title once for the topic-first long-input case; that missing title remains visible below. The quality charts exclude it and show their actual successful-title denominators. No combined “quality score” hides the differences.

@@CARDS@@

Green: pass. Amber: partial. Red: fail. Connotation applies only to feedback and the neutral database comparison. Fidelity checks whether the title still describes the requested work.

@@TABLE@@

Scoring audit: deduplicating identical text did not prevent inconsistent treatment of equivalent wording across AI batches. Before publication, nine unique grades were adjudicated under the fixed rubric: six inspect/test fidelity grades and three clipped-detail parse grades. The tables use those corrected grades. Original blind judgments · Every correction.

Pooled descriptive counts only. The models use different transports and sampling controls; inspect each model before generalizing.

@@MODEL_TABLES@@

Separate familiar examples from fresh ones.

The 16 reused requests informed the prompt revision. Eight fresh requests have no outputs in the first study. They were still designed by the same researcher after seeing that study, so they are not an independent external benchmark.

Reused cases

@@REUSED@@

Fresh cases

@@FRESH@@

Collections have their own denominators.

Important tradeoff: full consistency passes fell from 10/12 default groups to 4/12 topic-first groups; the previous override passed 9/12. Topic-first subjects did not produce a shared grammatical pattern reliably. Consistency is scored across auth and upgrade siblings. Meaningful variation also includes feedback and backup-operation groups. Each group belongs to one model, prompt, and repeat. Do not add these counts to the per-title scores.

@@GROUP_TABLE@@
Every family judgment@@GROUPS@@

All 144 matched comparisons.

Each row contains three actual displayed SDK outputs for the same synthetic request. Exact duplicate request/title pairs share one blind judgment, even when different models produced them.

144 comparisons
@@TRIPLETS@@

Fixed prompts. Visible limits.

The protocol was committed as 159f1b9 before the main run. One successful subscription smoke call preceded it and is excluded from the 432 attempts. The real SDK generate_title_with_llm helper receives the same full input and retains the same 50-character output cap in every arm. This compares bundled prompt changes, not the isolated causal effect of a skill.

@@PROMPTS@@

Subscription detail: Luna uses openai/gpt-6-luna, confirmed by Engel instead of the initially requested “6.1 Luna.” Low reasoning is passed explicitly in the request’s extra body and checked in normalized SDK options. The subscription endpoint does not accept the temperature or output-token cap used by the other two models. Its server-side interpretation is not independently verified.

Scoring: one AI judge model, blinded to model names, prompt labels, and generation instructions. Exact request/title duplicates are deduplicated before grading, then mapped back to every occurrence. Family groups are judged separately. This avoids the first experiment’s inconsistent duplicate grades; it does not turn AI judgments into human reading-speed measurements.

Limits: a small synthetic convenience sample, two repeats, prompt tuning informed by study one, provider aliases rather than independently attested model builds. There is no human usability study, significance claim, or general guarantee. Ellipsis endings are observed text, not instrumented proof of SDK truncation.

Audit and reproduction

''' for key, value in {"CARDS": summary_cards, "TABLE": table(), "MODEL_TABLES": model_tables, "REUSED": table("reused"), "FRESH": table("fresh"), "GROUP_TABLE": group_table(), "GROUPS": groups, "TRIPLETS": "".join(triplets), "PROMPTS": prompts, "EVIDENCE": evidence}.items(): page = page.replace("@@" + key + "@@", value) assert "@@" not in page output = SITE / "arch/topic-first-title-evaluation.html" output.write_text(page) print(output, len(page))