{
  "experiment": "Fixed-format titles, Codex and Jev",
  "frozen_at": "2026-10-30T09:37:59.169246+00:00",
  "source_sdk": "01bb167b04c00afc4d781f198fd8941a543eb266",
  "previous_report": "../../arch/readable-title-evaluation.html",
  "repeats": 2,
  "planned_generation_calls": 288,
  "title_limit": 50,
  "cases": [
    {
      "id": "auth-login",
      "family": "auth",
      "input": "Investigate why login fails after a password reset. No cause is confirmed yet.",
      "focus": "login",
      "risks": [
        "invented resolution",
        "lost uncertainty"
      ],
      "split": "reused"
    },
    {
      "id": "auth-session",
      "family": "auth",
      "input": "Investigate why sessions expire after five minutes. No cause is confirmed yet.",
      "focus": "session expiry",
      "risks": [
        "invented resolution",
        "lost uncertainty"
      ],
      "split": "reused"
    },
    {
      "id": "auth-oauth",
      "family": "auth",
      "input": "Investigate why OAuth callbacks return HTTP 400. No cause is confirmed yet.",
      "focus": "OAuth callbacks",
      "risks": [
        "invented resolution",
        "lost uncertainty"
      ],
      "split": "reused"
    },
    {
      "id": "upgrade-urllib3",
      "family": "upgrade",
      "input": "Upgrade urllib3 to the latest compatible patch release and run its regression tests.",
      "focus": "urllib3",
      "risks": [
        "invented completion"
      ],
      "split": "reused"
    },
    {
      "id": "upgrade-pydantic",
      "family": "upgrade",
      "input": "Upgrade Pydantic to the latest compatible patch release and run its regression tests.",
      "focus": "Pydantic",
      "risks": [
        "invented completion"
      ],
      "split": "reused"
    },
    {
      "id": "upgrade-pillow",
      "family": "upgrade",
      "input": "Upgrade Pillow to the latest compatible patch release and run its regression tests.",
      "focus": "Pillow",
      "risks": [
        "invented completion"
      ],
      "split": "reused"
    },
    {
      "id": "filler-cache",
      "family": "frontloading",
      "input": "I was hoping you could help me with a task today. What I need is an explanation of Redis cache eviction, especially the allkeys-lru policy.",
      "focus": "Redis cache eviction",
      "risks": [
        "generic opening"
      ],
      "split": "reused"
    },
    {
      "id": "filler-build",
      "family": "frontloading",
      "input": "I was hoping you could help me with a task today. What I need is an explanation of Rust incremental compilation and why cache misses slow builds.",
      "focus": "Rust incremental compilation",
      "risks": [
        "generic opening"
      ],
      "split": "reused"
    },
    {
      "id": "feedback-complaint",
      "family": "feedback",
      "input": "Summarize customer complaints: users say the lesson energy limit interrupts practice and forces them to wait. Do not turn this into praise.",
      "focus": "energy-limit complaints",
      "risks": [
        "wrong sentiment"
      ],
      "split": "reused"
    },
    {
      "id": "feedback-praise",
      "family": "feedback",
      "input": "Summarize customer praise: users love the offline lessons because they can practice on the train. Do not invent a complaint.",
      "focus": "offline-lesson praise",
      "risks": [
        "wrong sentiment"
      ],
      "split": "reused"
    },
    {
      "id": "feedback-mixed",
      "family": "feedback",
      "input": "Summarize mixed feedback about chess lessons: users enjoy the puzzles but complain that the checkmate rule sometimes rejects valid moves.",
      "focus": "mixed chess-lesson feedback",
      "risks": [
        "flattened mixed sentiment"
      ],
      "split": "reused"
    },
    {
      "id": "neutral-compare",
      "family": "fidelity",
      "input": "Compare SQLite and PostgreSQL for a small offline-first app. I have not selected either database.",
      "focus": "SQLite and PostgreSQL comparison",
      "risks": [
        "invented decision",
        "invented sentiment"
      ],
      "split": "reused"
    },
    {
      "id": "negation",
      "family": "fidelity",
      "input": "Explain how to inspect Docker disk usage. Do not delete containers, images, or volumes.",
      "focus": "Docker disk inspection",
      "risks": [
        "opposite action",
        "invented cleanup"
      ],
      "split": "reused"
    },
    {
      "id": "uncertainty",
      "family": "fidelity",
      "input": "Check whether TLS certificate rotation might explain the intermittent webhook failures. This is a hypothesis, not a confirmed cause.",
      "focus": "possible TLS-related webhook failures",
      "risks": [
        "hypothesis stated as fact",
        "invented resolution"
      ],
      "split": "reused"
    },
    {
      "id": "completed",
      "family": "status",
      "input": "The login fix has already merged and all regression tests passed. Write a short release note about the completed fix.",
      "focus": "release note for completed login fix",
      "risks": [
        "confusing requested writing with completed engineering"
      ],
      "split": "reused"
    },
    {
      "id": "long-tail",
      "family": "input-cap",
      "input": "Reference background only; the actual request follows at the end.\nThe project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. \nACTUAL REQUEST: Investigate Zephyr's Unicode CSV parser crash. Do not work on the CLI examples.",
      "focus": "Zephyr Unicode CSV parser crash",
      "risks": [
        "missing decisive text beyond character 1000"
      ],
      "split": "reused"
    },
    {
      "id": "backup-status",
      "family": "operations",
      "input": "The nightly backup failed with a permission error. Investigate it; do not claim that a backup completed.",
      "focus": "backup failure investigation",
      "split": "fresh",
      "risks": [
        "wrong requested action",
        "lost qualifiers or attribution"
      ]
    },
    {
      "id": "backup-restore",
      "family": "operations",
      "input": "Explain how to test a backup restore in a disposable directory. Do not overwrite the production database.",
      "focus": "safe backup restore test explanation",
      "split": "fresh",
      "risks": [
        "wrong requested action",
        "lost qualifiers or attribution"
      ]
    },
    {
      "id": "feedback-refunds",
      "family": "feedback",
      "input": "Summarize customer complaints that refund processing takes weeks. Report their frustration without endorsing or dismissing it.",
      "focus": "refund complaints summary",
      "split": "fresh",
      "risks": [
        "wrong requested action",
        "lost qualifiers or attribution"
      ]
    },
    {
      "id": "feedback-accessibility",
      "family": "feedback",
      "input": "Summarize customer praise for the new screen-reader labels. The praise is from customers, not a request for your endorsement.",
      "focus": "screen-reader praise summary",
      "split": "fresh",
      "risks": [
        "wrong requested action",
        "lost qualifiers or attribution"
      ]
    },
    {
      "id": "feedback-notifications",
      "family": "feedback",
      "input": "Summarize mixed customer feedback: users like notification scheduling but dislike the default notification sound.",
      "focus": "mixed notification feedback summary",
      "split": "fresh",
      "risks": [
        "wrong requested action",
        "lost qualifiers or attribution"
      ]
    },
    {
      "id": "proposal-not-approved",
      "family": "fidelity",
      "input": "Draft a proposal to replace Jenkins with Buildkite. No migration has been approved or started.",
      "focus": "draft migration proposal",
      "split": "fresh",
      "risks": [
        "wrong requested action",
        "lost qualifiers or attribution"
      ]
    },
    {
      "id": "suspected-leak",
      "family": "fidelity",
      "input": "Check whether retained image buffers might cause the memory growth. This is only a hypothesis; do not report a confirmed leak.",
      "focus": "possible image-buffer retention investigation",
      "split": "fresh",
      "risks": [
        "wrong requested action",
        "lost qualifiers or attribution"
      ]
    },
    {
      "id": "docs-after-revert",
      "family": "status",
      "input": "The deployment was reverted yesterday. Update the incident timeline; do not redeploy or reopen the rollout.",
      "focus": "incident timeline update after revert",
      "split": "fresh",
      "risks": [
        "wrong requested action",
        "lost qualifiers or attribution"
      ]
    }
  ],
  "prompts": {
    "default": null,
    "fixed": "Write one conversation title of at most {max_length} characters, including one relevant emoji. Use this exact layout: EMOJI Topic — action. Keep it near 42 characters by choosing shorter words, never by cutting a word. Rules:\n1. Make the requested work clear and faithful. Keep status, uncertainty, negation, and who expressed a view. Investigation is not repair; a proposal is not approval; summarizing customer praise is not praising the subject yourself. Never invent completion.\n2. Put the distinguishing topic in the first two meaningful words after the emoji.\n3. Use a parallel topic/action structure and stable terms for equivalent requests. Let different topics and genuinely different actions create the differences, not random wording.\n4. When relevant, preserve complaint, praise, mixed, or neutral intent.\nUse short action labels such as investigate, upgrade/test, explain, summarize feedback, draft proposal, or write note, only when accurate. Return only the title.\n\nUSER REQUEST:\n{conversation_content}"
  },
  "models": {
    "haiku": {
      "model": "litellm_proxy/anthropic/claude-haiku-4-5",
      "transport": "eval proxy",
      "reasoning_effort": "none",
      "max_output_tokens": 256,
      "temperature_requested": 0,
      "legacy_thinking_budget": null
    },
    "gemini": {
      "model": "litellm_proxy/gemini-3-flash-preview",
      "transport": "eval proxy",
      "reasoning_effort": "none",
      "max_output_tokens": 256,
      "temperature_requested": 0,
      "legacy_thinking_budget": null
    },
    "luna": {
      "model": "openai/gpt-6-luna",
      "transport": "ChatGPT subscription",
      "reasoning_effort": "low",
      "reasoning_sent_via": "litellm_extra_body.reasoning.effort; normalized SDK options verified",
      "max_output_tokens": null,
      "temperature_requested": null
    }
  },
  "rubric": {
    "clarity": {
      "instructions": "Does this displayed title clearly convey an understandable task or topic? Imperative, noun phrases and topic—action labels are all allowed. Ignore harmless clipping of a modifier when the core task remains clear.",
      "criteria": {
        "fail": "The core task is unclear or incoherent.",
        "partial": "The core task is partly understandable but its main clause is incomplete or grammatically ambiguous.",
        "pass": "The core task is clear and understandable."
      }
    },
    "frontload": {
      "instructions": "Do the first two meaningful lexical words after the emoji identify the distinguishing topic of this request? Ignore punctuation, not ordinary action words; the state includes the exact first two words computed in code.",
      "criteria": {
        "fail": "The displayed title does not identify a useful distinguishing topic.",
        "partial": "The topic is clear but appears after the first two meaningful words.",
        "pass": "The distinguishing topic appears in the first two meaningful words after the emoji."
      }
    },
    "fidelity": {
      "instructions": "Does the title preserve the requested work without an actual wrong action, invented completion/decision, reversed negation, false certainty, or changed attribution? Omission of minor details is allowed. A title naming an inspection or test is compatible with explaining that activity unless it explicitly claims execution or completion. Reporting a completed fix can be factually true but failing to mention writing the requested release note is loss of requested work.",
      "criteria": {
        "fail": "The title misstates requested work or makes a misleading claim, including praising rather than summarizing customers’ praise.",
        "pass": "The title describes the requested work or a compatible concise topic without a misleading claim."
      }
    },
    "connotation": {
      "instructions": "For these customer-feedback or neutral-comparison requests, does the title preserve the input’s sentiment without inventing endorsement or a preference? Explicit mixed feedback counts as both-sided. Generic feedback loses polarity but does not reverse it.",
      "criteria": {
        "fail": "The title reverses or invents sentiment or preference.",
        "partial": "The title becomes generic/neutral and loses a useful positive, negative or mixed distinction.",
        "pass": "The title preserves the relevant complaint, praise, mixed feedback or neutral comparison intent."
      }
    },
    "collection": {
      "instructions": "Judge the sibling titles together. Equivalent requests should use parallel topic/action grammar and stable action terms; meaningful differences in topics or requested actions must remain distinguishable. Ignore emoji, punctuation and compatible abbreviations such as Tests/Regression Tests. A fixed-format label is not required for default titles.",
      "criteria": {
        "fail": "Equivalent tasks use substantially inconsistent structures, or a meaningful distinction is lost or replaced with invented work.",
        "partial": "Subjects remain distinguishable but there is some arbitrary structural, vocabulary or specificity variation.",
        "pass": "The collection uses parallel structure/stable terms for equivalent tasks and preserves justified topic/action differences."
      }
    }
  },
  "method": [
    "Generate both arms again on all24cases, two repeats, three models; all input/output limits as prior study.",
    "All24requests have been observed in study two; this is a tuned diagnostic set, not held-out validation. Retain original split labels only as historical provenance.",
    "Blind both judges to model/arm labels and each other’s grades. Same questions/criteria and minimal original request/title state for both.",
    "Deduplicate exact request/title outputs once per judge; aggregate occurrences separately. Keep probabilities/confidence from Jev, not proof of truth.",
    "Precompute first two lexical words in code and supply to both judges; compute length/format compliance in code, not by Jev.",
    "Connotation applies only to feedback-family and neutral-compare. Collection rule applies auth/upgrade/feedback/operations separately permodel/arm/repeat.",
    "No adjudication or grade harmonization. Report disagreements and original scores separately. A report review may fix arithmetic or code defects, not rewrite judgments to agreement.",
    "No human gold labels are supplied; judge agreement does not establish accuracy. A small set of prior trouble cases is highlighted rather than presented as ground truth."
  ],
  "scoring_groups": {
    "collection": [
      "auth",
      "upgrade",
      "feedback",
      "operations"
    ]
  },
  "limits": "Tuned synthetic convenience sample; different generator transports and controls; AI judges may share biases. No human usability study, statistical significance or truth claim.",
  "rules": [
    "Clear and faithful task",
    "Distinguishing topic first",
    "Consistent structure with meaningful distinctions",
    "Connotation when relevant"
  ],
  "judges": {
    "codex": {
      "model": "gpt-6-astra",
      "reasoning": "medium"
    },
    "jev": {
      "model": "jev-1.14.0",
      "question_type": "choice",
      "label_rule": "returned choice; preserve probabilities and confidence; do not threshold away disagreements"
    }
  }
}