{ "experiment": "Topic-first titles, follow-up", "frozen_at": "2026-10-30T08:33:13.029169+00:00", "source_sdk": "01bb167b04c00afc4d781f198fd8941a543eb266", "previous_report": "../../arch/readable-title-evaluation.html", "repeats": 2, "planned_generation_calls": 432, "title_limit": 50, "cases": [ { "id": "auth-login", "family": "auth", "input": "Investigate why login fails after a password reset. No cause is confirmed yet.", "focus": "login", "risks": [ "invented resolution", "lost uncertainty" ], "split": "reused" }, { "id": "auth-session", "family": "auth", "input": "Investigate why sessions expire after five minutes. No cause is confirmed yet.", "focus": "session expiry", "risks": [ "invented resolution", "lost uncertainty" ], "split": "reused" }, { "id": "auth-oauth", "family": "auth", "input": "Investigate why OAuth callbacks return HTTP 400. No cause is confirmed yet.", "focus": "OAuth callbacks", "risks": [ "invented resolution", "lost uncertainty" ], "split": "reused" }, { "id": "upgrade-urllib3", "family": "upgrade", "input": "Upgrade urllib3 to the latest compatible patch release and run its regression tests.", "focus": "urllib3", "risks": [ "invented completion" ], "split": "reused" }, { "id": "upgrade-pydantic", "family": "upgrade", "input": "Upgrade Pydantic to the latest compatible patch release and run its regression tests.", "focus": "Pydantic", "risks": [ "invented completion" ], "split": "reused" }, { "id": "upgrade-pillow", "family": "upgrade", "input": "Upgrade Pillow to the latest compatible patch release and run its regression tests.", "focus": "Pillow", "risks": [ "invented completion" ], "split": "reused" }, { "id": "filler-cache", "family": "frontloading", "input": "I was hoping you could help me with a task today. What I need is an explanation of Redis cache eviction, especially the allkeys-lru policy.", "focus": "Redis cache eviction", "risks": [ "generic opening" ], "split": "reused" }, { "id": "filler-build", "family": "frontloading", "input": "I was hoping you could help me with a task today. What I need is an explanation of Rust incremental compilation and why cache misses slow builds.", "focus": "Rust incremental compilation", "risks": [ "generic opening" ], "split": "reused" }, { "id": "feedback-complaint", "family": "feedback", "input": "Summarize customer complaints: users say the lesson energy limit interrupts practice and forces them to wait. Do not turn this into praise.", "focus": "energy-limit complaints", "risks": [ "wrong sentiment" ], "split": "reused" }, { "id": "feedback-praise", "family": "feedback", "input": "Summarize customer praise: users love the offline lessons because they can practice on the train. Do not invent a complaint.", "focus": "offline-lesson praise", "risks": [ "wrong sentiment" ], "split": "reused" }, { "id": "feedback-mixed", "family": "feedback", "input": "Summarize mixed feedback about chess lessons: users enjoy the puzzles but complain that the checkmate rule sometimes rejects valid moves.", "focus": "mixed chess-lesson feedback", "risks": [ "flattened mixed sentiment" ], "split": "reused" }, { "id": "neutral-compare", "family": "fidelity", "input": "Compare SQLite and PostgreSQL for a small offline-first app. I have not selected either database.", "focus": "SQLite and PostgreSQL comparison", "risks": [ "invented decision", "invented sentiment" ], "split": "reused" }, { "id": "negation", "family": "fidelity", "input": "Explain how to inspect Docker disk usage. Do not delete containers, images, or volumes.", "focus": "Docker disk inspection", "risks": [ "opposite action", "invented cleanup" ], "split": "reused" }, { "id": "uncertainty", "family": "fidelity", "input": "Check whether TLS certificate rotation might explain the intermittent webhook failures. This is a hypothesis, not a confirmed cause.", "focus": "possible TLS-related webhook failures", "risks": [ "hypothesis stated as fact", "invented resolution" ], "split": "reused" }, { "id": "completed", "family": "status", "input": "The login fix has already merged and all regression tests passed. Write a short release note about the completed fix.", "focus": "release note for completed login fix", "risks": [ "confusing requested writing with completed engineering" ], "split": "reused" }, { "id": "long-tail", "family": "input-cap", "input": "Reference background only; the actual request follows at the end.\nThe project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. The project uses a small command-line interface and stores its examples in a dedicated folder. \nACTUAL REQUEST: Investigate Zephyr's Unicode CSV parser crash. Do not work on the CLI examples.", "focus": "Zephyr Unicode CSV parser crash", "risks": [ "missing decisive text beyond character 1000" ], "split": "reused" }, { "id": "backup-status", "family": "operations", "input": "The nightly backup failed with a permission error. Investigate it; do not claim that a backup completed.", "focus": "backup failure investigation", "split": "fresh", "risks": [ "wrong requested action", "lost qualifiers or attribution" ] }, { "id": "backup-restore", "family": "operations", "input": "Explain how to test a backup restore in a disposable directory. Do not overwrite the production database.", "focus": "safe backup restore test explanation", "split": "fresh", "risks": [ "wrong requested action", "lost qualifiers or attribution" ] }, { "id": "feedback-refunds", "family": "feedback", "input": "Summarize customer complaints that refund processing takes weeks. Report their frustration without endorsing or dismissing it.", "focus": "refund complaints summary", "split": "fresh", "risks": [ "wrong requested action", "lost qualifiers or attribution" ] }, { "id": "feedback-accessibility", "family": "feedback", "input": "Summarize customer praise for the new screen-reader labels. The praise is from customers, not a request for your endorsement.", "focus": "screen-reader praise summary", "split": "fresh", "risks": [ "wrong requested action", "lost qualifiers or attribution" ] }, { "id": "feedback-notifications", "family": "feedback", "input": "Summarize mixed customer feedback: users like notification scheduling but dislike the default notification sound.", "focus": "mixed notification feedback summary", "split": "fresh", "risks": [ "wrong requested action", "lost qualifiers or attribution" ] }, { "id": "proposal-not-approved", "family": "fidelity", "input": "Draft a proposal to replace Jenkins with Buildkite. No migration has been approved or started.", "focus": "draft migration proposal", "split": "fresh", "risks": [ "wrong requested action", "lost qualifiers or attribution" ] }, { "id": "suspected-leak", "family": "fidelity", "input": "Check whether retained image buffers might cause the memory growth. This is only a hypothesis; do not report a confirmed leak.", "focus": "possible image-buffer retention investigation", "split": "fresh", "risks": [ "wrong requested action", "lost qualifiers or attribution" ] }, { "id": "docs-after-revert", "family": "status", "input": "The deployment was reverted yesterday. Update the incident timeline; do not redeploy or reopen the rollout.", "focus": "incident timeline update after revert", "split": "fresh", "risks": [ "wrong requested action", "lost qualifiers or attribution" ] } ], "prompts": { "default": null, "previous": "Generate one conversation title for the user request below. Apply the following readable-text skill to a sidebar of task titles. Use one relevant emoji followed by an action in the continuous form and the specific task subject. Keep the distinguishing topic near the start. Describe the user's requested work, not an assumed result. Use the same vocabulary for the same concepts across titles. Preserve negation, uncertainty, and meaningful sentiment where relevant. Return only a title of at most {max_length} characters.\n\n\n# Readable Text\n\nApply these principles to text that readers scan as a collection. Follow the product's length and output-format constraints. Preserve facts, uncertainty, and distinctions that affect meaning.\n\n1. **Make the action easy to parse.** Prefer actor, action, then object. Omit an actor shared by every item and start with the action. Match tense to the reader's context: ongoing work can use \"Investigating login failures\"; a completed result can use \"Fixed login failures\". Never imply completion without evidence.\n2. **Front-load distinguishing words.** Put the words that identify this item near the start, ideally within the first two words. Remove repeated openings such as \"This conversation is about\". Keep the action clear while bringing its specific subject forward.\n3. **Reuse structure and vocabulary.** Choose one sentence pattern for comparable items and one term per concept. Avoid switching between \"issue\", \"page\", and \"ticket\" for the same thing. When generating one item at a time, follow a fixed template rather than inventing variety.\n4. **Make variation meaningful.** Use comparable detail and length for comparable items. Change structure or add a qualifier only when the underlying difference matters. Do not make one item appear more important through arbitrary verbosity. Keep necessary qualifiers even when they break the pattern.\n5. **Match verbs to purpose and evidence.** In feedback summaries, use \"Users praise\" for praise and \"Users complain\" for complaints, rather than hiding both behind \"Users discuss\". Retain mixed sentiment when present; never invent emotion or certainty. Keep neutral material neutral.\n\nBefore returning, scan only the beginnings: can readers tell the items apart? Then read the collection for consistent terms, parallel structure, and unsupported implications. Return the requested text, without explaining these rules.\n\n## References\n\n- Adapted from Shreya Shankar's [Five Principles for Generating Readable AI Text in Bulk](https://fsdatalab.github.io/blog/agent-text-at-scale/) (2026).\n- Nielsen Norman Group: [First 2 Words: A Signal for the Scanning Eye](https://www.nngroup.com/articles/first-2-words-a-signal-for-scanning/) and [Consistency and standards](https://www.nngroup.com/articles/ten-usability-heuristics/).\n\n\nUSER REQUEST:\n{conversation_content}", "topic_first": "Write one sidebar title, at most {max_length} characters including one relevant emoji. Put the distinguishing subject in the first two words after the emoji, then a short action or task label. Use parallel wording for equivalent tasks. Keep meaningful differences; omit filler. Prefer 42 characters when possible; shorten wording instead of cutting a word. Preserve the requested action, status, negation, and uncertainty. Summarizing someone’s praise or complaints is reporting it, not endorsing it. Keep mixed feedback mixed. Do not turn investigation into repair or a proposal into a decision. Do not invent completion. Return only the title.\n\nUSER REQUEST:\n{conversation_content}" }, "models": { "haiku": { "model": "litellm_proxy/anthropic/claude-haiku-4-5", "transport": "eval proxy", "reasoning_effort": "none", "max_output_tokens": 256, "temperature_requested": 0, "legacy_thinking_budget": null }, "gemini": { "model": "litellm_proxy/gemini-3-flash-preview", "transport": "eval proxy", "reasoning_effort": "none", "max_output_tokens": 256, "temperature_requested": 0, "legacy_thinking_budget": null }, "luna": { "model": "openai/gpt-6-luna", "transport": "ChatGPT subscription", "reasoning_effort": "low", "reasoning_sent_via": "litellm_extra_body.reasoning.effort; normalized SDK options verified", "max_output_tokens": null, "temperature_requested": null } }, "rubric": { "version": 1, "status": "Frozen before follow-up generation; revised after first-study scoring lessons, not a registered human trial.", "unit": "One displayed SDK title, except consistency and meaningful variation, which are scored across matched families within each model and repeat", "blinding": "Shuffle arms and hide their names and prompt text from the independent scorer. Supply original user inputs and this rubric. Reveal arms only after saving judgments.", "outcomes": [ "pass", "partial", "fail", "not_applicable" ], "rules": [ { "id": "parse", "name": "Match how people parse", "scope": "all successful outputs", "pass": "A direct, understandable task action and subject; omit redundant human or agent subjects. Do not force a verb form that changes requested status.", "fail": "Generic conversational filler, unclear actor/action/subject, or misleading tense.", "partial": "The core task is understandable but its main clause is incomplete or grammatically ambiguous. A clipped modifier alone does not make the core task unclear." }, { "id": "frontload", "name": "Put distinguishing words first", "scope": "all successful outputs; compare auth, upgrade, and frontloading families", "pass": "The first two meaningful words after the emoji expose the task's distinguishing topic. Necessary task verbs can count; generic wording cannot hide the distinguishing topic.", "partial": "The topic is clear but appears later than the first two meaningful words.", "fail": "The visible beginning gives no useful distinction from neighboring tasks." }, { "id": "consistent", "name": "Keep structure and vocabulary consistent", "scope": "auth and upgrade families, separately within each model, arm, and repeat", "pass": "Sibling titles use parallel grammatical structure and stable vocabulary for their shared action, while retaining their different objects. Compatible abbreviations such as Tests and Regression Tests do not change this grade.", "partial": "Minor unexplained wording or tense changes.", "fail": "Unexplained structural or lexical changes make equivalent tasks look different." }, { "id": "variation", "name": "Make variation meaningful", "scope": "auth, upgrade, and feedback families, separately within each model, arm, and repeat", "pass": "Equivalent tasks retain a shared pattern and meaningful differences remain visible; no incidental filler or arbitrary detail makes an item seem more important.", "partial": "Some arbitrary detail or inconsistent specificity without losing the key distinction.", "fail": "Meaningful differences disappear or unsupported detail creates false distinctions. Equal title length is not required." }, { "id": "connotation", "name": "Match connotation to purpose", "scope": "All feedback-family cases plus neutral-compare; all other titles not_applicable.", "pass": "Complaints remain negative, praise remains positive, mixed feedback retains both positive and negative aspects, and the neutral comparison acquires no invented preference.", "partial": "Broad but compatible sentiment that loses a useful distinction; a neutral description of mixed feedback that omits its two sides.", "fail": "A title reverses, invents, or falsely intensifies sentiment or preference." } ], "fidelity_gate": "Independently flag unsupported completion, decisions, deleted negation, hypotheses asserted as facts, wrong requested work, and loss of the decisive long-input request. A readable but inaccurate title is not an overall success. Distinguish summarizing reported praise from the agent praising the subject. Failure requires an actual wrong requested action or misleading claim, not omission of every input detail.", "reporting": "Keep all generated outputs and transport failures. Report denominators separately for per-title and family-level criteria. Do not average these into one success rate. Mark subjective judgments as such and show their reasons. Never treat failed generations as not_applicable successes.", "limits": "Small synthetic convenience sample, two low-temperature repeats, model- and prompt-specific findings. An AI scorer measures a rubric, not human reading speed, and provides no statistical proof of general improvement. Compare the override on identical uncapped inputs; test removed input caps separately." }, "method": [ "Run every arm again; do not recycle successful outputs from the first study.", "Rotate arm order by case/repeat; reverse on second repeat.", "Same complete input and 50-character SDK display cap in all arms.", "The shorter override is a different bundled prompt; this is not skill-only causal attribution.", "16 reused cases informed prompt revision; 8 fresh cases have no prior model outputs but were designed by the same researcher after seeing study one. Report splits separately.", "Blind scorer gets requests and exact displayed outputs, not arm/model labels. Deduplicate identical request/title pairs before scoring for consistent labels.", "Score family groups separately per model/arm/repeat. Compare structures and meaningful variations, not equal lengths or emoji identities.", "Preserve original judgments and label any subsequent adjudication; failed generations remain failed, not rubric passes.", "Low reasoning is requested for subscription Luna; provider-side behavior is not independently attested. No max-output-token or temperature control is available on that transport." ], "scoring_groups": { "consistent": [ "auth", "upgrade" ], "variation": [ "auth", "upgrade", "feedback", "operations" ] }, "limits": "Small synthetic convenience sample, two repeats, one AI evaluator, no human usability or statistical generalization. Fresh cases are not an independent external benchmark." }