"""Build dual-judge title evidence without changing either prior study.""" import html import json import re from collections import Counter from pathlib import Path ROOT = Path(__file__).resolve().parent SITE = ROOT.parents[1] P = json.loads((ROOT / "protocol.json").read_text()) D = {n: json.loads((ROOT / f"{n}.json").read_text()) for n in P["models"]} ITEMS = json.loads((ROOT / "blinded-items.json").read_text()) M = json.loads((ROOT / "mapping.json").read_text()) CODEX = json.loads((ROOT / "codex.json").read_text()) JEV = json.loads((ROOT / "jev.json").read_text()) C = {r["id"]: r["answers"] for r in CODEX["items"]} V = {r["id"]: r for r in JEV["records"]} T = {(r["case_id"], r["title"]): r["id"] for r in ITEMS["titles"]} APPLICABLE = {r["id"]: r["criteria"] for r in ITEMS["titles"] + ITEMS["groups"]} E = lambda x: html.escape(str(x), quote=True) ARMS = {"default": "SDK default", "fixed": "Topic — action"} MODELS = {"haiku": "Haiku 4.5", "gemini": "Gemini 3 Flash Preview", "luna": "GPT-6 Luna · low"} RULES = {"clarity": "Clear task", "frontload": "Topic first", "connotation": "Connotation", "fidelity": "Requested-work fidelity", "collection": "Consistent, meaningful collection"} def label(judge, id, criterion): if judge == "codex": return C.get(id, {}).get(criterion) record = V.get(id, {}).get("response") return record.get("answers", {}).get(criterion, {}).get("choice") if record else None def counts(judge, criterion, arm, model="all"): if criterion == "collection": ids = [id for id, m in M.items() if id.startswith("g") and m["arm"] == arm and (model == "all" or m["model"] == model)] else: ids = [T[(r["case_id"], r["title"])] for n, d in D.items() if model in ("all", n) for r in d["records"] if r["arm"] == arm and r["title"] and criterion in APPLICABLE[T[(r["case_id"], r["title"])]]] result = Counter(label(judge, id, criterion) or "missing" for id in ids) return result def meter(c): total = sum(c.values()) if not total: return "Not applicable" pieces = "".join(f'' for k in ("pass", "partial", "fail", "missing") if c[k]) caption = " · ".join(f"{c[k]} {k}" for k in ("pass", "partial", "fail", "missing") if c[k]) return f'{E(caption)} · n={total}' def scorecard(model="all"): rows = [] for key, name in RULES.items(): cells = "".join('' + meter(counts(judge, key, arm, model)) + '' for judge in ("codex", "jev") for arm in ARMS) unit = "collection groups" if key == "collection" else "title occurrences, applicable cases only" rows.append(f'{name}{unit}{cells}') return '
' + ''.join(f'' for _ in range(2) for a in ARMS.values()) + '' + ''.join(rows) + '
CriterionCodex / gpt-6-astraJev / jev-1.14.0
{a}
' agreement = [] disagreements = [] for row in ITEMS["titles"] + ITEMS["groups"]: for key in row["criteria"]: c, v = label("codex", row["id"], key), label("jev", row["id"], key) agreement.append({"id": row["id"], "criterion": key, "codex": c, "jev": v, "agreement": c == v if c and v else None}) if c and v and c != v: probabilities = V[row["id"]]["response"]["answers"][key] content = f'
{E(json.dumps(row.get("items", {k: row[k] for k in ("request", "title", "first_two_words") if k in row}), ensure_ascii=False, indent=2))}
' disagreements.append(f'
{row["id"]} · {RULES[key]} · Codex {c} / Jev {v} · confidence {probabilities.get("confidence", 0):.3f}{content}
{E(json.dumps(probabilities, ensure_ascii=False, indent=2))}
') summary = {"generation_attempts": sum(len(d["records"]) for d in D.values()), "displayed_titles": sum(bool(r["title"]) for d in D.values() for r in d["records"]), "unique_titles": len(ITEMS["titles"]), "groups": len(ITEMS["groups"]), "scores": {j: {k: {a: dict(counts(j, k, a)) for a in ARMS} for k in RULES} for j in ("codex", "jev")}, "agreement": {k: {"compared": sum(r["criterion"] == k and r["agreement"] is not None for r in agreement), "agreed": sum(r["criterion"] == k and r["agreement"] is True for r in agreement)} for k in RULES}, "disagreements": len(disagreements)} (ROOT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") (ROOT / "agreement.json").write_text(json.dumps(agreement, indent=2) + "\n") agreement_rows = ''.join(f'{RULES[k]}{s["agreed"]}/{s["compared"]}{s["compared"]-s["agreed"]}' for k, s in summary["agreement"].items()) comparisons = [] for model, data in D.items(): for repeat in (1, 2): for case in P["cases"]: cards = [] for arm in ARMS: r = next(r for r in data["records"] if r["case_id"] == case["id"] and r["arm"] == arm and r["repeat"] == repeat) if r["title"]: id = T[(r["case_id"], r["title"])] badges = ''.join(f'
{name} ' + ' '.join(f'{k}: {label(j, id, k) or "missing"}' for k in ("clarity", "frontload", "fidelity")) + '
' for j, name in (("codex", "Codex"), ("jev", "Jev"))) else: badges = 'Generation failure' format_ok = bool(r["title"] and re.search(r"\S\s+—\s+\S", r["title"])) cards.append(f'
{ARMS[arm]}

{E(r["title"] or "No title")}

{badges}{r["characters"]} Python characters · em-dash layout {"yes" if format_ok else "no"}
') comparisons.append(f'
{MODELS[model]} · run {repeat} · {case["id"]}
{"".join(cards)}
Original synthetic request
{E(case["input"])}
') models = ''.join(f'
{name} · all five evaluation dimensions{scorecard(n)}
' for n, name in MODELS.items()) criteria = ''.join(f'
{RULES[k]}

{E(r["instructions"])}

{E(json.dumps(r["criteria"], ensure_ascii=False, indent=2))}
' for k, r in P["rubric"].items()) format_counts = {a: sum(bool(r["title"] and re.search(r"\S\s+—\s+\S", r["title"])) for d in D.values() for r in d["records"] if r["arm"] == a) for a in ARMS} embedded = json.dumps({"protocol": P, "models": D, "items": ITEMS, "mapping": M, "codex": CODEX, "jev": JEV, "summary": summary, "agreement": agreement, "codex_instructions": json.loads((ROOT / "codex-instructions.json").read_text()), "reproduction_notes": (ROOT / "reproduce.txt").read_text()}, ensure_ascii=False).replace(' Two judges, four title rules — Engel’s Notebook
Title experiment three · 30 October 2026

One title format.
Two independent judges.

A fixed “Topic — action” prompt combines consistent structure with meaningful differences. Codex and Jev independently classify the same displayed titles and sibling collections. Their disagreements remain visible; neither is treated as ground truth.

@@TOTAL@@ generations · @@UNIQUE@@ unique title texts for judging · 48 collections

First experiment · Topic-first follow-up — both unchanged.

Four instructions, plus a fidelity gate.

  1. Make the task clear.
  2. Put distinguishing topic words first.
  3. Keep parallel structure and meaningful distinctions together.
  4. Preserve connotation when sentiment matters.

“Make variation meaningful” is merged with collection consistency. It remains part of the evaluation. Requested-work fidelity is a separate requirement: a polished title still fails if it changes the task.

Exact fixed-format prompt
@@PROMPT@@
Exact shared scoring rubric@@CRITERIA@@

Format is measured in code, not by either judge: the em-dash layout occurred in @@FIXED_FORMAT@@/144 fixed-prompt titles and @@DEFAULT_FORMAT@@/144 default titles. This check does not prove that the topic or action is correct.

Both judges favor topic-first. Collections divide them.

Shared finding: the fixed format improved front-loading: Codex 107/144 → 144/144; Jev 102/144 → 141/144. Both gave all 144 fixed-format titles a fidelity pass. These are classifier judgments on the tuned set, not proof of correctness.

Unsettled: collection passes moved in opposite directions. Codex: 15/24 → 13/24. Jev: 17/24 → 19/24. The judges agreed on only 30/48 collection grades. I would inspect those disagreements before changing the SDK default.

Green means pass, amber partial, red fail, gray missing judge result. Per-title scores count every generated occurrence, even when identical text is judged once. Collection scores use their own denominators. No manual grade adjustments are applied.

@@SCORECARD@@

The four writing principles appear as clarity, topic-first, collection, and applicable connotation. Fidelity is separate. Model-specific results:

@@MODELS@@

Agreement is a measurement of judges, not truth.

The table compares one label for each unique request/title/criterion, plus the separate collection judgments. It does not overweight repeated identical outputs. No human gold labels were supplied.

@@AGREEMENT@@
CriterionAgreed / comparedDisagreed

Jev’s confidence is calculated from its answer-probability distribution. High confidence is not proof of correctness. All raw probabilities remain available. Codex returns labels, not comparable probabilities. There is no combined “best judge” or averaged quality score.

All @@DISAGREEMENT_COUNT@@ disagreements

@@DISAGREEMENTS@@

All 144 matched title pairs.

Default and fixed-format outputs were freshly generated, not copied from earlier studies. Expand a request before deciding which judge you agree with.

144 pairs
@@PAIRS@@

A diagnostic set, not a new benchmark.

All 24 synthetic requests appeared in experiment two. They informed the prompt and rubric, so none is held-out validation here. Three generators × two prompts × two repeats produce 288 planned attempts. The protocol was committed before generation.

  • Generation: real SDK helper at 01bb167; full input and the same 50-character display cap in both arms. Haiku and Gemini use the eval-proxy route with temperature 0 requested, 256 output tokens, reasoning none. Luna uses ChatGPT subscription, low reasoning requested, with no temperature/output-token control.
  • Codex judge: gpt-6-astra, medium reasoning, three blinded title batches and one collection batch. Labels only; no explanations were requested.
  • Jev judge: pinned jev-1.14.0 through TypeSafe’s API. One minimal request/title state with applicable Choice questions, or one sibling group with the collection question. Raw selected choices, probabilities, confidence, usage and errors are retained.
  • Blinding: neither judge receives generator names, prompt arm labels, generation instructions or the other judge’s answers. Both receive the same semantic questions, criteria, original request and displayed title. Jev option order is deterministically shuffled; Codex reads the rubric in its saved order. This interface difference is a limitation.
  • Deterministic checks: code computes title length, em-dash compliance and the first two Unicode lexical words. Both judges see those first two words. This parser is an operational definition, not a universal linguistic boundary.
  • Calibration: none established on human-labeled title data. This study compares judge outputs; agreement alone cannot validate either judge.

The generation cap retains ordinary Python string-length semantics. An observed “...” ending is not an independently recorded raw-model truncation event. No private conversations, memory, auth files or credentials were sent as classification state.

Evidence and reproduction

''' for key, value in {"TOTAL": summary["generation_attempts"], "UNIQUE": summary["unique_titles"], "PROMPT": E(P["prompts"]["fixed"]), "CRITERIA": criteria, "FIXED_FORMAT": format_counts["fixed"], "DEFAULT_FORMAT": format_counts["default"], "SCORECARD": scorecard(), "MODELS": models, "AGREEMENT": agreement_rows, "DISAGREEMENT_COUNT": len(disagreements), "DISAGREEMENTS": ''.join(disagreements), "PAIRS": ''.join(comparisons), "EVIDENCE": embedded}.items(): page = page.replace('@@' + key + '@@', str(value)) assert '@@' not in page out = SITE / 'arch/dual-judge-title-evaluation.html' out.write_text(page) print(out, len(page))