{ "checkout_sha": "acca181af1f353975965bcf6486e1d0a58f444c0", "server_version": "1.54.0", "date": "2026-10-13", "mapped_features": 935, "mapped_families": 34, "valid_primary_family_reports": 32, "families_with_recorded_active_checks": 34, "primary_counts": { "pass": 655, "fail": 36, "xfail": 123, "blocked": 3, "retired": 1, "not-run": 117 }, "primary_recipe_row_counts": { "pass": 656, "fail": 36, "xfail": 125, "blocked": 3, "retired": 1, "not-run": 117 }, "latest_recorded_counts": { "pass": 778, "blocked": 3, "xfail": 149, "fail": 3, "xpass": 1, "retired": 1 }, "latest_recorded_rows": [ { "feature_id": "F01.alive", "family": "F01", "result": "pass", "label": "map run F01-server-status.md: Liveness (`F01.alive`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-01-liveness-f01-alive.json", "at": "2026-10-13T02:17:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F01.debug-tracebacks", "family": "F01", "result": "pass", "label": "map run F01-server-status.md: DEBUG tracebacks (`F01.debug-tracebacks`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-10-debug-tracebacks-f01-debug-tracebacks.json", "at": "2026-10-13T02:22:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F01.health", "family": "F01", "result": "pass", "label": "map run F01-server-status.md: Health (`F01.health`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-02-health-f01-health.json", "at": "2026-10-13T02:17:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F01.idle-time", "family": "F01", "result": "pass", "label": "map run F01-server-status.md: Idle time (`F01.idle-time`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-07-idle-time-f01-idle-time.json", "at": "2026-10-13T02:17:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F01.openapi", "family": "F01", "result": "pass", "label": "map run F01-server-status.md: OpenAPI document (`F01.openapi`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-08-openapi-document-f01-openapi.json", "at": "2026-10-13T02:17:43Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F01.preload-tools", "family": "F01", "result": "pass", "label": "map run F01-server-status.md: Tool preload (`F01.preload-tools`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-11-tool-preload-f01-preload-tools.json", "at": "2026-10-13T02:26:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F01.ready", "family": "F01", "result": "pass", "label": "map run F01-server-status.md: Readiness (`F01.ready`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-03-readiness-f01-ready.json", "at": "2026-10-13T02:17:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F01.root-info", "family": "F01", "result": "pass", "label": "map run F01-server-status.md: Root alias (`F01.root-info`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-05-root-alias-f01-root-info.json", "at": "2026-10-13T02:17:09Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F01.server-info", "family": "F01", "result": "pass", "label": "map run F01-server-status.md: Server info (`F01.server-info`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-04-server-info-f01-server-info.json", "at": "2026-10-13T02:17:09Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F01.static-files", "family": "F01", "result": "pass", "label": "map run F01-server-status.md: Static files (`F01.static-files`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-09-static-files-f01-static-files.json", "at": "2026-10-13T02:19:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F01.uptime-restart", "family": "F01", "result": "pass", "label": "map run F01-server-status.md: Uptime across a restart (`F01.uptime-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-06-uptime-across-a-restart-f01-uptime-resta.json", "at": "2026-10-13T02:17:38Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F02.cors-configured-origins", "family": "F02", "result": "pass", "label": "map run F02-auth-and-sessions.md: Configured CORS origins (`F02.cors-configured-origins`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-15-configured-cors-origins-f02-cors-configu.json", "at": "2026-10-13T02:44:27Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F02.cors-default-origins", "family": "F02", "result": "pass", "label": "map run F02-auth-and-sessions.md: Default CORS origins (`F02.cors-default-origins`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-14-default-cors-origins-f02-cors-default-or.json", "at": "2026-10-13T02:43:44Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F02.cors-workspace-routes", "family": "F02", "result": "pass", "label": "map run F02-auth-and-sessions.md: Workspace routes accept any origin (`F02.cors-workspace-routes`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-16-workspace-routes-accept-any-origin-f02-c.json", "at": "2026-10-13T02:44:32Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F02.default-bind", "family": "F02", "result": "blocked", "label": "CLI default bind address inside unshare -rn", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/F02.default-bind/no-key.json", "at": "2026-10-13T02:48:16Z", "expected": "Private network namespace available; verify default bind with and without a key", "actual": "unshare cannot open /proc/self/uid_map: Read-only file system; no server started", "note": "Environment prerequisite missing. The primary raw family result retains fail; this judgment marks the feature blocked, not a product bug." }, { "feature_id": "F02.http-header-only", "family": "F02", "result": "pass", "label": "map run F02-auth-and-sessions.md: Header only (`F02.http-header-only`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-02-header-only-f02-http-header-only.json", "at": "2026-10-13T02:41:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F02.http-key-required", "family": "F02", "result": "pass", "label": "map run F02-auth-and-sessions.md: Key required on HTTP (`F02.http-key-required`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-01-key-required-on-http-f02-http-key-requir.json", "at": "2026-10-13T02:41:35Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F02.keys-hidden-from-commands", "family": "F02", "result": "pass", "label": "map run F02-auth-and-sessions.md: Keys stay out of commands (`F02.keys-hidden-from-commands`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-13-keys-stay-out-of-commands-f02-keys-hidde.json", "at": "2026-10-13T02:43:40Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F02.legacy-env-key", "family": "F02", "result": "pass", "label": "map run F02-auth-and-sessions.md: Legacy `SESSION_API_KEY` (`F02.legacy-env-key`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-18-legacy-session-api-key-f02-legacy-env-ke.json", "at": "2026-10-13T02:45:44Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F02.multiple-keys", "family": "F02", "result": "pass", "label": "map run F02-auth-and-sessions.md: A second key (`F02.multiple-keys`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-11-a-second-key-f02-multiple-keys.json", "at": "2026-10-13T02:42:51Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F02.no-auth-mode", "family": "F02", "result": "pass", "label": "map run F02-auth-and-sessions.md: Unauthenticated mode (`F02.no-auth-mode`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-17-unauthenticated-mode-f02-no-auth-mode.json", "at": "2026-10-13T02:45:07Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F02.rotate-key", "family": "F02", "result": "pass", "label": "map run F02-auth-and-sessions.md: Rotate the key (`F02.rotate-key`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-12-rotate-the-key-f02-rotate-key.json", "at": "2026-10-13T02:43:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F02.workspace-cookie-access", "family": "F02", "result": "pass", "label": "map run F02-auth-and-sessions.md: The cookie alone opens workspace files (`F02.workspace-cookie-access`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-09-the-cookie-alone-opens-workspace-files-f.json", "at": "2026-10-13T02:42:23Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F02.workspace-cookie-root-path", "family": "F02", "result": "xfail", "label": "map run F02-auth-and-sessions.md: Cookie path ignores a prefix (`F02.workspace-cookie-root-path`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-20-cookie-path-ignores-a-prefix-f02-workspa.json", "at": "2026-10-13T02:46:45Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 9: control-agent-server exec --run \"$PREFIXED\" --save F02.workspace-cookie-root-path/file-with-cookie -- bash \"$F02/jar.sh\" GET \"/qa-f02-prefix/$PCID/workspace/qa-f02.html\" nokey 200 'has:qa-f02-workspace-file'" }, { "feature_id": "F02.workspace-session-delete", "family": "F02", "result": "pass", "label": "map run F02-auth-and-sessions.md: Delete the workspace cookie (`F02.workspace-session-delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-10-delete-the-workspace-cookie-f02-workspac.json", "at": "2026-10-13T02:42:25Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F02.workspace-session-ipv6-loopback", "family": "F02", "result": "xfail", "label": "map run F02-auth-and-sessions.md: IPv6 loopback is not a secure context (`F02.workspace-session-ipv6-loopback`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-21-ipv6-loopback-is-not-a-secure-context-f0.json", "at": "2026-10-13T02:46:47Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 5: control-agent-server api POST /api/auth/workspace-session --header 'Host: [::1]:8000' --expect 204 --save F02.workspace-session-ipv6-loopback/ipv6-host --check-header set-cookie contains SameSite=none --check-header set-cookie contains Secure --check-header set-cookie contains Partitioned" }, { "feature_id": "F02.workspace-session-mint", "family": "F02", "result": "pass", "label": "map run F02-auth-and-sessions.md: Mint the workspace cookie (`F02.workspace-session-mint`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-08-mint-the-workspace-cookie-f02-workspace-.json", "at": "2026-10-13T02:41:53Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F02.ws-auth-timeout", "family": "F02", "result": "pass", "label": "map run F02-auth-and-sessions.md: Auth timeout (`F02.ws-auth-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-05-auth-timeout-f02-ws-auth-timeout.json", "at": "2026-10-13T02:41:49Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F02.ws-first-frame", "family": "F02", "result": "pass", "label": "map run F02-auth-and-sessions.md: First-frame auth on every socket (`F02.ws-first-frame`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-03-first-frame-auth-on-every-socket-f02-ws-.json", "at": "2026-10-13T02:41:37Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F02.ws-first-frame-reject", "family": "F02", "result": "pass", "label": "map run F02-auth-and-sessions.md: Rejected first frames (`F02.ws-first-frame-reject`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-04-rejected-first-frames-f02-ws-first-frame.json", "at": "2026-10-13T02:41:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F02.ws-legacy-key", "family": "F02", "result": "pass", "label": "map run F02-auth-and-sessions.md: Deprecated query and header keys (`F02.ws-legacy-key`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-06-deprecated-query-and-header-keys-f02-ws-.json", "at": "2026-10-13T02:41:51Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F02.ws-legacy-reject", "family": "F02", "result": "pass", "label": "map run F02-auth-and-sessions.md: Wrong legacy key fails the handshake (`F02.ws-legacy-reject`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-07-wrong-legacy-key-fails-the-handshake-f02.json", "at": "2026-10-13T02:41:52Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.auth-before-gate", "family": "F03", "result": "retired", "label": "retired by #5700. It asserted that a dormant pod with boot session keys answered 401 to a missing key before the 503 gate; readiness is now checked first, see `F03.gate-before-auth`.", "run": null, "transcript": null, "variants": [ { "feature_id": "F03.auth-before-gate", "family": "F03", "result": "retired", "label": "retired by #5700. It asserted that a dormant pod with boot session keys answered 401 to a missing key before the 503 gate; readiness is now checked first, see `F03.gate-before-auth`.", "run": null, "transcript": null } ] }, { "feature_id": "F03.dormant-boot", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Dormant boot (`F03.dormant-boot`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-01-dormant-boot-f03-dormant-boot.json", "at": "2026-10-13T02:44:31Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.dormant-gates-api", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Dormant gate (`F03.dormant-gates-api`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-02-dormant-gate-f03-dormant-gates-api.json", "at": "2026-10-13T02:44:38Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.dormant-gates-sockets", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Dormant sockets (`F03.dormant-gates-sockets`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-03-dormant-sockets-f03-dormant-gates-socket.json", "at": "2026-10-13T02:44:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.dormant-gates-v1", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Dormant /v1 and app backends (`F03.dormant-gates-v1`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-04-dormant-v1-and-app-backends-f03-dormant-.json", "at": "2026-10-13T02:44:46Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.gate-before-auth", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Readiness precedes auth (`F03.gate-before-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-22-readiness-precedes-auth-f03-gate-before-.json", "at": "2026-10-13T02:47:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.init-applies-env", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Init env reaches bash (`F03.init-applies-env`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-12-init-env-reaches-bash-f03-init-applies-e.json", "at": "2026-10-13T02:44:59Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.init-delivered-cors-origins", "family": "F03", "result": "xfail", "label": "map run F03-deferred-init.md: Delivered CORS origins (`F03.init-delivered-cors-origins`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-18-delivered-cors-origins-f03-init-delivere.json", "at": "2026-10-13T02:45:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 5: control-agent-server api OPTIONS /api/settings --auth none --header 'Origin: https://qa-f03.example' --header 'Access-Control-Request-Method: GET' --expect 200 --check-header access-control-allow-origin eq https://qa-f03.example --save F03.init-delivered-cors-origins/preflight" }, { "feature_id": "F03.init-delivered-max-runs", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Delivered run capacity (`F03.init-delivered-max-runs`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-17-delivered-run-capacity-f03-init-delivere.json", "at": "2026-10-13T02:45:11Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.init-delivered-webhooks", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Delivered webhooks (`F03.init-delivered-webhooks`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-10-delivered-webhooks-f03-init-delivered-we.json", "at": "2026-10-13T02:44:58Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.init-delivered-worktree-root", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Delivered worktree root (`F03.init-delivered-worktree-root`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-16-delivered-worktree-root-f03-init-deliver.json", "at": "2026-10-13T02:45:04Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.init-enforces-session-keys", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Delivered keys are enforced (`F03.init-enforces-session-keys`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-11-delivered-keys-are-enforced-f03-init-enf.json", "at": "2026-10-13T02:44:59Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.init-failure-rollback", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Failure rollback (`F03.init-failure-rollback`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-08-failure-rollback-f03-init-failure-rollba.json", "at": "2026-10-13T02:44:54Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.init-key-auth", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Init key auth (`F03.init-key-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-05-init-key-auth-f03-init-key-auth.json", "at": "2026-10-13T02:44:48Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.init-keys-gate-bash-socket", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Delivered keys gate the bash socket (`F03.init-keys-gate-bash-socket`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-13-delivered-keys-gate-the-bash-socket-f03-.json", "at": "2026-10-13T02:45:00Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.init-open-without-secret", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Open init without a secret (`F03.init-open-without-secret`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-20-open-init-without-a-secret-f03-init-open.json", "at": "2026-10-13T02:46:06Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.init-opens-events-socket-and-v1", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Events socket and /v1 open after init (`F03.init-opens-events-socket-and-v1`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-14-events-socket-and-v1-open-after-init-f03.json", "at": "2026-10-13T02:45:01Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.init-redacts-secret-input", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Input redaction (`F03.init-redacts-secret-input`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-07-input-redaction-f03-init-redacts-secret-.json", "at": "2026-10-13T02:44:52Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.init-second-call-400", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Second init is 400 (`F03.init-second-call-400`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-19-second-init-is-400-f03-init-second-call-.json", "at": "2026-10-13T02:45:15Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.init-state-not-persisted", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Init state is not persisted (`F03.init-state-not-persisted`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-21-init-state-is-not-persisted-f03-init-sta.json", "at": "2026-10-13T02:46:40Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.init-success", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Successful init (`F03.init-success`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-09-successful-init-f03-init-success.json", "at": "2026-10-13T02:44:56Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.init-validation", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Init validation (`F03.init-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-06-init-validation-f03-init-validation.json", "at": "2026-10-13T02:44:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.init-web-url-root-path", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Delivered web URL sets the root path (`F03.init-web-url-root-path`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-15-delivered-web-url-sets-the-root-path-f03.json", "at": "2026-10-13T02:45:03Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F03.not-deferred-404", "family": "F03", "result": "pass", "label": "map run F03-deferred-init.md: Not deferred is 404 (`F03.not-deferred-404`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-23-not-deferred-is-404-f03-not-deferred-404.json", "at": "2026-10-13T02:47:46Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.batch-get", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Batch get (`F04.batch-get`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-19-batch-get-f04-batch-get.json", "at": "2026-10-13T02:49:26Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.batch-get-limit", "family": "F04", "result": "xfail", "label": "map run F04-conversation-lifecycle.md: 100 ids in one batch (`F04.batch-get-limit`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-20-100-ids-in-one-batch-f04-batch-get-limit.json", "at": "2026-10-13T02:49:29Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 2: control-agent-server exec --expect-output QA_BATCH_OK --save F04.batch-get-limit/hundred -- python3 \"$F/qa-f04-batch.py\" \"$IDS100\" 400,422" }, { "feature_id": "F04.count", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Count (`F04.count`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-23-count-f04-count.json", "at": "2026-10-13T02:49:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.create-agent-profile", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Stored Agent Profile (`F04.create-agent-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-03-stored-agent-profile-f04-create-agent-pr.json", "at": "2026-10-13T02:47:37Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.create-idempotent", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Idempotent create with a client id (`F04.create-idempotent`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-06-idempotent-create-with-a-client-id-f04-c.json", "at": "2026-10-13T02:48:07Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.create-initial-message-run-false", "family": "F04", "result": "xfail", "label": "map run F04-conversation-lifecycle.md: `initial_message.run: false` (`F04.create-initial-message-run-false`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-07-initial-message-run-false-f04-create-ini.json", "at": "2026-10-13T02:48:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 9: false" }, { "feature_id": "F04.create-inline-agent", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Inline agent (`F04.create-inline-agent`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-02-inline-agent-f04-create-inline-agent.json", "at": "2026-10-13T02:47:29Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.create-observability", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Observability headers (`F04.create-observability`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-10-observability-headers-f04-create-observa.json", "at": "2026-10-13T02:48:41Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.create-observability-precedence", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Body versus headers (`F04.create-observability-precedence`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-11-body-versus-headers-f04-create-observabi.json", "at": "2026-10-13T02:48:51Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.create-parent", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Parent and child (`F04.create-parent`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-08-parent-and-child-f04-create-parent.json", "at": "2026-10-13T02:48:30Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.create-secrets", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Start-time secrets (`F04.create-secrets`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-12-start-time-secrets-f04-create-secrets.json", "at": "2026-10-13T02:48:53Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.create-secrets-get", "family": "F04", "result": "xfail", "label": "map run F04-conversation-lifecycle.md: Start-time secret visible to GET (`F04.create-secrets-get`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-13-start-time-secret-visible-to-get-f04-cre.json", "at": "2026-10-13T02:48:54Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 1: control-agent-server api GET \"/$SECID\" --expect 200 --check secret_registry.secret_sources.QA_F04_TOKEN.kind eq StaticSecret --save F04.create-secrets-get/get" }, { "feature_id": "F04.create-secrets-plain", "family": "F04", "result": "xfail", "label": "map run F04-conversation-lifecycle.md: Plain-string secret value (`F04.create-secrets-plain`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-14-plain-string-secret-value-f04-create-sec.json", "at": "2026-10-13T02:48:55Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 1: control-agent-server api POST --json \"{\\\"workspace\\\": {\\\"working_dir\\\": \\\"$W\\\"}, \\\"agent\\\": $OFFLINE_AGENT, \\\"autotitle\\\": false, \\\"secrets\\\": {\\\"QA_F04_PLAIN\\\": \\\"qa-f04-plain-value\\\"}}\" --expect 201,422 --save F04.create-secrets-plain/start" }, { "feature_id": "F04.create-settings", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Create from the saved settings (`F04.create-settings`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-01-create-from-the-saved-settings-f04-creat.json", "at": "2026-10-13T02:47:28Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.create-validation", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Create validation (`F04.create-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-04-create-validation-f04-create-validation.json", "at": "2026-10-13T02:47:52Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.create-validation-redaction", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: 422 bodies keep the LLM key out (`F04.create-validation-redaction`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-05-422-bodies-keep-the-llm-key-out-f04-crea.json", "at": "2026-10-13T02:47:55Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.create-worktree", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Git worktree (`F04.create-worktree`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-09-git-worktree-f04-create-worktree.json", "at": "2026-10-13T02:48:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.delete", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Delete (`F04.delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-29-delete-f04-delete.json", "at": "2026-10-13T02:50:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.delete-unknown-id", "family": "F04", "result": "xfail", "label": "map run F04-conversation-lifecycle.md: DELETE on an unknown id (`F04.delete-unknown-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-30-delete-on-an-unknown-id-f04-delete-unkno.json", "at": "2026-10-13T02:50:44Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 4: control-agent-server api DELETE \"/$NOPE\" --expect 404 --save F04.delete-unknown-id/unknown" }, { "feature_id": "F04.final-response", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Final response (`F04.final-response`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-16-final-response-f04-final-response.json", "at": "2026-10-13T02:49:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.get", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Read one conversation (`F04.get`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-18-read-one-conversation-f04-get.json", "at": "2026-10-13T02:49:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.patch-errors", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: PATCH errors (`F04.patch-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-26-patch-errors-f04-patch-errors.json", "at": "2026-10-13T02:50:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.patch-tags", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Retag (`F04.patch-tags`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-25-retag-f04-patch-tags.json", "at": "2026-10-13T02:50:07Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.patch-title", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Rename (`F04.patch-title`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-24-rename-f04-patch-title.json", "at": "2026-10-13T02:49:54Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.patch-unknown-id", "family": "F04", "result": "xfail", "label": "map run F04-conversation-lifecycle.md: PATCH on an unknown id (`F04.patch-unknown-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-28-patch-on-an-unknown-id-f04-patch-unknown.json", "at": "2026-10-13T02:50:26Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 3: control-agent-server api PATCH \"/$NOPE\" --json '{\"title\": \"QA nobody\"}' --expect 404 --save F04.patch-unknown-id/unknown" }, { "feature_id": "F04.patch-whitespace-title", "family": "F04", "result": "xfail", "label": "map run F04-conversation-lifecycle.md: Whitespace-only title (`F04.patch-whitespace-title`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-27-whitespace-only-title-f04-patch-whitespa.json", "at": "2026-10-13T02:50:19Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 2: control-agent-server api PATCH \"/$CHILD\" --json '{\"title\": \" \"}' --expect 422 --save F04.patch-whitespace-title/patch" }, { "feature_id": "F04.read-no-hydrate", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Cold reads do not load a conversation (`F04.read-no-hydrate`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-32-cold-reads-do-not-load-a-conversation-f0.json", "at": "2026-10-13T02:51:33Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.restart-persist", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Restart (`F04.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-31-restart-f04-restart-persist.json", "at": "2026-10-13T02:51:26Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.run-real", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: A real run (`F04.run-real`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-15-a-real-run-f04-run-real.json", "at": "2026-10-13T02:49:03Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.search-filter", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Status filter (`F04.search-filter`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-22-status-filter-f04-search-filter.json", "at": "2026-10-13T02:49:48Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.search-page", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Search pages (`F04.search-page`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-21-search-pages-f04-search-page.json", "at": "2026-10-13T02:49:40Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F04.title-llm-profile", "family": "F04", "result": "pass", "label": "map run F04-conversation-lifecycle.md: Title from a saved LLM profile (`F04.title-llm-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-17-title-from-a-saved-llm-profile-f04-title.json", "at": "2026-10-13T02:49:14Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.control-errors", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Auth and malformed ids (`F05.control-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-13-auth-and-malformed-ids-f05-control-error.json", "at": "2026-10-13T02:55:17Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.interrupt-idle", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Interrupt with nothing running (`F05.interrupt-idle`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-07-interrupt-with-nothing-running-f05-inter.json", "at": "2026-10-13T02:54:17Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.interrupt-running", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Interrupt a running conversation (`F05.interrupt-running`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-18-interrupt-a-running-conversation-f05-int.json", "at": "2026-10-13T02:56:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.interrupt-tool-call", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Interrupt during a tool call (`F05.interrupt-tool-call`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-21-interrupt-during-a-tool-call-f05-interru.json", "at": "2026-10-13T02:56:52Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.max-iterations-resume", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Resume after the cap (`F05.max-iterations-resume`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-27-resume-after-the-cap-f05-max-iterations-.json", "at": "2026-10-13T02:58:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.max-iterations-stop", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Iteration cap (`F05.max-iterations-stop`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-26-iteration-cap-f05-max-iterations-stop.json", "at": "2026-10-13T02:57:56Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.pause-idle", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Pause an idle conversation (`F05.pause-idle`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-04-pause-an-idle-conversation-f05-pause-idl.json", "at": "2026-10-13T02:53:54Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.pause-repeat", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Pause twice (`F05.pause-repeat`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-05-pause-twice-f05-pause-repeat.json", "at": "2026-10-13T02:54:00Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.pause-running", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Pause during a long model call (`F05.pause-running`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-15-pause-during-a-long-model-call-f05-pause.json", "at": "2026-10-13T02:55:32Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.pause-running-crash", "family": "F05", "result": "xfail", "label": "map run F05-run-pause-interrupt.md: Crash after a mid-call pause (`F05.pause-running-crash`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-032700-5284-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-032700-5284-map-f05/evidence/map-run/F05-run-pause-interrupt-03-crash-after-a-mid-call-pause-f05-pause-r.json", "at": "2026-10-13T03:30:10Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 12: control-agent-server api GET \"/$K\" --check execution_status eq paused --quiet --save F05.pause-running-crash/get" }, { "feature_id": "F05.pause-running-get", "family": "F05", "result": "xfail", "label": "map run F05-run-pause-interrupt.md: GET after a mid-call pause (`F05.pause-running-get`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-16-get-after-a-mid-call-pause-f05-pause-run.json", "at": "2026-10-13T02:55:34Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 3: control-agent-server api GET \"/$P\" --check execution_status eq paused --quiet --save F05.pause-running-get/get" }, { "feature_id": "F05.pause-running-release", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Release a paused step with interrupt (`F05.pause-running-release`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-17-release-a-paused-step-with-interrupt-f05.json", "at": "2026-10-13T02:55:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.pause-terminal", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Pause and interrupt a terminal conversation (`F05.pause-terminal`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-09-pause-and-interrupt-a-terminal-conversat.json", "at": "2026-10-13T02:54:44Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.pause-tool-step", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Pause during a tool call (`F05.pause-tool-step`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-20-pause-during-a-tool-call-f05-pause-tool-.json", "at": "2026-10-13T02:56:44Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.restart-hard-running", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Crash while running (`F05.restart-hard-running`).", "run": "/workspace/verification-artifacts/runs/20261013-032700-5284-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-032700-5284-map-f05/evidence/map-run/F05-run-pause-interrupt-02-crash-while-running-f05-restart-hard-run.json", "at": "2026-10-13T03:29:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.restart-keeps-status", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Statuses survive the restart (`F05.restart-keeps-status`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-29-statuses-survive-the-restart-f05-restart.json", "at": "2026-10-13T02:59:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.restart-running-graceful", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Graceful restart while running (`F05.restart-running-graceful`).", "run": "/workspace/verification-artifacts/runs/20261013-032700-5284-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-032700-5284-map-f05/evidence/map-run/F05-run-pause-interrupt-01-graceful-restart-while-running-f05-resta.json", "at": "2026-10-13T03:27:54Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.run-after-interrupt", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Resume after an interrupt (`F05.run-after-interrupt`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-19-resume-after-an-interrupt-f05-run-after-.json", "at": "2026-10-13T02:56:25Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.run-capacity", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Run capacity (`F05.run-capacity`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-22-run-capacity-f05-run-capacity.json", "at": "2026-10-13T02:57:02Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.run-conflict", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Run while running (`F05.run-conflict`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-14-run-while-running-f05-run-conflict.json", "at": "2026-10-13T02:55:24Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.run-error", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: A run that fails (`F05.run-error`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-08-a-run-that-fails-f05-run-error.json", "at": "2026-10-13T02:54:35Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.run-error-responsive", "family": "F05", "result": "xfail", "label": "map run F05-run-pause-interrupt.md: Responsive while a run fails (`F05.run-error-responsive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-10-responsive-while-a-run-fails-f05-run-err.json", "at": "2026-10-13T02:55:01Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 20: test \"$SLOWEST\" -le 1500" }, { "feature_id": "F05.run-finished-noop", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Run with nothing to do (`F05.run-finished-noop`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-02-run-with-nothing-to-do-f05-run-finished-.json", "at": "2026-10-13T02:53:26Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.run-new-input", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: New input after finishing (`F05.run-new-input`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-03-new-input-after-finishing-f05-run-new-in.json", "at": "2026-10-13T02:53:35Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.run-queued", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Run queued input on a real model (`F05.run-queued`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-01-run-queued-input-on-a-real-model-f05-run.json", "at": "2026-10-13T02:53:14Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.run-resume-paused", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Resume a paused conversation (`F05.run-resume-paused`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-06-resume-a-paused-conversation-f05-run-res.json", "at": "2026-10-13T02:54:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.stuck-detected", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: A run that loops ends `stuck` (`F05.stuck-detected`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-23-a-run-that-loops-ends-stuck-f05-stuck-de.json", "at": "2026-10-13T02:57:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.stuck-new-input", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: New input un-sticks it (`F05.stuck-new-input`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-25-new-input-un-sticks-it-f05-stuck-new-inp.json", "at": "2026-10-13T02:57:40Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.stuck-rerun", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Run a stuck conversation again (`F05.stuck-rerun`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-24-run-a-stuck-conversation-again-f05-stuck.json", "at": "2026-10-13T02:57:32Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F05.unknown-id-pause", "family": "F05", "result": "xfail", "label": "map run F05-run-pause-interrupt.md: Pause or interrupt an unknown id (`F05.unknown-id-pause`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-12-pause-or-interrupt-an-unknown-id-f05-unk.json", "at": "2026-10-13T02:55:04Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 5: control-agent-server api POST \"/$NOPE/pause\" --expect 404 --save F05.unknown-id-pause/pause" }, { "feature_id": "F05.unknown-id-run", "family": "F05", "result": "pass", "label": "map run F05-run-pause-interrupt.md: Run an unknown id (`F05.unknown-id-run`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-11-run-an-unknown-id-f05-unknown-id-run.json", "at": "2026-10-13T02:55:01Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.batch-get", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Batch get (`F06.batch-get`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-10-batch-get-f06-batch-get.json", "at": "2026-10-13T03:01:37Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.count", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Count (`F06.count`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-07-count-f06-count.json", "at": "2026-10-13T03:01:28Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.get-one", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Get one event (`F06.get-one`).", "run": "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/map-run/F06-conversation-events-02-get-one-event-f06-get-one.json", "at": "2026-10-13T03:30:56Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.not-found-auth", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Unknown conversation, bad id, no key (`F06.not-found-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-11-unknown-conversation-bad-id-no-key-f06-n.json", "at": "2026-10-13T03:01:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.restart-persist", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: History survives a restart (`F06.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-25-history-survives-a-restart-f06-restart-p.json", "at": "2026-10-13T03:04:46Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.search-filters", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Filters (`F06.search-filters`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-04-filters-f06-search-filters.json", "at": "2026-10-13T03:01:17Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.search-kind-short", "family": "F06", "result": "xfail", "label": "map run F06-conversation-events.md: Short kind names (`F06.search-kind-short`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-05-short-kind-names-f06-search-kind-short-k.json", "at": "2026-10-13T03:01:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 4: control-agent-server api GET \"/$CID/events/search\" --query kind=MessageEvent --expect 200 --check items len-eq 2 --save F06.search-kind-short/short-name" }, { "feature_id": "F06.search-page", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Paging and sort order (`F06.search-page`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-02-paging-and-sort-order-f06-search-page.json", "at": "2026-10-13T03:01:07Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.search-time-window", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Time window (`F06.search-time-window`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-06-time-window-f06-search-time-window.json", "at": "2026-10-13T03:01:23Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.search-timestamp-order", "family": "F06", "result": "xfail", "label": "map run F06-conversation-events.md: Log order is timestamp order (`F06.search-timestamp-order`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-23-log-order-is-timestamp-order-f06-search-.json", "at": "2026-10-13T03:04:04Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 7: control-agent-server api GET \"/$LIVE/events/count\" --query timestamp__gte=\"$OBS_T\" --check . ge \"$FROM_OBS\" --save F06.search-timestamp-order/count-since-observation" }, { "feature_id": "F06.search-validation", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Bad search parameters (`F06.search-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-03-bad-search-parameters-f06-search-validat.json", "at": "2026-10-13T03:01:11Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.send-image", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Image content (`F06.send-image`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-13-image-content-f06-send-image.json", "at": "2026-10-13T03:02:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.send-no-run", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Append a message without running (`F06.send-no-run`).", "run": "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/map-run/F06-conversation-events-01-append-a-message-without-running-f06-sen.json", "at": "2026-10-13T03:30:55Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.send-non-user-role", "family": "F06", "result": "xfail", "label": "map run F06-conversation-events.md: Non-user role (`F06.send-non-user-role`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-14-non-user-role-f06-send-non-user-role-kno.json", "at": "2026-10-13T03:02:20Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 4: control-agent-server api POST \"/$CID/events\" --json '{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"QA_F06_ASSISTANT\"}]}' --expect 4xx --save F06.send-non-user-role/assistant" }, { "feature_id": "F06.send-resets-finished", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: A message reopens a finished conversation (`F06.send-resets-finished`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-24-a-message-reopens-a-finished-conversatio.json", "at": "2026-10-13T03:04:06Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.send-run", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Run a turn and watch it live (`F06.send-run`, `F06.ws-live`).", "run": "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/map-run/F06-conversation-events-03-run-a-turn-and-watch-it-live-f06-send-ru.json", "at": "2026-10-13T03:31:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.send-run-limit", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Run capacity full (`F06.send-run-limit`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-26-run-capacity-full-f06-send-run-limit.json", "at": "2026-10-13T03:05:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.send-validation", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Message validation (`F06.send-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-12-message-validation-f06-send-validation.json", "at": "2026-10-13T03:01:59Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.send-while-running", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: A message while the agent runs (`F06.send-while-running`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-27-a-message-while-the-agent-runs-f06-send-.json", "at": "2026-10-13T03:05:35Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.unknown-event", "family": "F06", "result": "xfail", "label": "map run F06-conversation-events.md: Unknown event id (`F06.unknown-event`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-09-unknown-event-id-f06-unknown-event-known.json", "at": "2026-10-13T03:01:35Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 4: control-agent-server api GET \"/$CID/events/00000000-0000-4000-8000-0000000000e6\" --expect 404 --save F06.unknown-event/get-one" }, { "feature_id": "F06.ws-bad-frame", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Bad frames on the socket (`F06.ws-bad-frame`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-20-bad-frames-on-the-socket-f06-ws-bad-fram.json", "at": "2026-10-13T03:03:29Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.ws-delete-close", "family": "F06", "result": "xfail", "label": "map run F06-conversation-events.md: Deleting a conversation closes its sockets (`F06.ws-delete-close`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-17-deleting-a-conversation-closes-its-socke.json", "at": "2026-10-13T03:03:03Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 8: false" }, { "feature_id": "F06.ws-handshake", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Handshake: auth and close codes (`F06.ws-handshake`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-16-handshake-auth-and-close-codes-f06-ws-ha.json", "at": "2026-10-13T03:02:48Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.ws-live", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Run a turn and watch it live (`F06.send-run`, `F06.ws-live`).", "run": "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/map-run/F06-conversation-events-03-run-a-turn-and-watch-it-live-f06-send-ru.json", "at": "2026-10-13T03:31:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.ws-replay-all", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Replay everything (`F06.ws-replay-all`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-18-replay-everything-f06-ws-replay-all.json", "at": "2026-10-13T03:03:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.ws-replay-since", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Replay since a timestamp (`F06.ws-replay-since`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-19-replay-since-a-timestamp-f06-ws-replay-s.json", "at": "2026-10-13T03:03:23Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.ws-send", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: Send a message over the socket (`F06.ws-send`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-21-send-a-message-over-the-socket-f06-ws-se.json", "at": "2026-10-13T03:03:33Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F06.ws-snapshot", "family": "F06", "result": "pass", "label": "map run F06-conversation-events.md: First frame is a state snapshot (`F06.ws-snapshot`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-15-first-frame-is-a-state-snapshot-f06-ws-s.json", "at": "2026-10-13T03:02:30Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.auth", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Authentication (`F07.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-10-authentication-f07-auth.json", "at": "2026-10-13T02:51:03Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.auth-frame-ignored", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Redundant auth frame (`F07.auth-frame-ignored`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-09-redundant-auth-frame-f07-auth-frame-igno.json", "at": "2026-10-13T02:50:59Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.bad-handshake", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Malformed handshake (`F07.bad-handshake`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-12-malformed-handshake-f07-bad-handshake.json", "at": "2026-10-13T02:51:10Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.deleted-close", "family": "F07", "result": "xfail", "label": "map run F07-session-socket.md: Deleting closes the socket (`F07.deleted-close`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025834-053c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-025834-053c-map-f07/evidence/map-run/F07-session-socket-02-deleting-closes-the-socket-f07-deleted-c.json", "at": "2026-10-13T02:59:26Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 7: false" }, { "feature_id": "F07.error-frame", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Invalid message (`F07.error-frame`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-06-invalid-message-f07-error-frame.json", "at": "2026-10-13T02:50:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.error-non-json", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Non-JSON and binary frames (`F07.error-non-json`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-07-non-json-and-binary-frames-f07-error-non.json", "at": "2026-10-13T02:50:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.frame-too-large", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Oversized event (`F07.frame-too-large`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-21-oversized-event-f07-frame-too-large.json", "at": "2026-10-13T02:53:22Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.full-replay", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Full replay (`F07.full-replay`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-03-full-replay-f07-full-replay.json", "at": "2026-10-13T02:50:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.item-aborted", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Interrupted stream (`F07.item-aborted`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-20-interrupted-stream-f07-item-aborted.json", "at": "2026-10-13T02:53:07Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.live-durable", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Live durable frame (`F07.live-durable`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-13-live-durable-frame-f07-live-durable.json", "at": "2026-10-13T02:51:24Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.live-order", "family": "F07", "result": "xfail", "label": "map run F07-session-socket.md: Live frames in seq order (`F07.live-order`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025834-053c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-025834-053c-map-f07/evidence/map-run/F07-session-socket-01-live-frames-in-seq-order-f07-live-order-.json", "at": "2026-10-13T02:59:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 11: control-agent-server state cat \"evidence/F07.live-order/order-$ORD.json\" --check frames.3.frame.seq eq 0 --check frames.4.frame.seq eq 1 --check frames.5.frame.seq eq 2" }, { "feature_id": "F07.progress-frames", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Streaming progress frames (`F07.progress-frames`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-18-streaming-progress-frames-f07-progress-f.json", "at": "2026-10-13T02:52:41Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.progress-not-replayed", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Progress is never replayed (`F07.progress-not-replayed`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-19-progress-is-never-replayed-f07-progress-.json", "at": "2026-10-13T02:52:45Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.reject-non-user", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Non-user role (`F07.reject-non-user`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-08-non-user-role-f07-reject-non-user.json", "at": "2026-10-13T02:50:52Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.replay-then-live", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Replay and live tail, each event once (`F07.replay-then-live`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-15-replay-and-live-tail-each-event-once-f07.json", "at": "2026-10-13T02:51:45Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.restart-resume", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Cursor survives a restart (`F07.restart-resume`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-17-cursor-survives-a-restart-f07-restart-re.json", "at": "2026-10-13T02:52:34Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.resume-cursor", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Resume from a cursor (`F07.resume-cursor`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-05-resume-from-a-cursor-f07-resume-cursor.json", "at": "2026-10-13T02:50:32Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.send-message", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Send a message on the socket (`F07.send-message`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-16-send-a-message-on-the-socket-f07-send-me.json", "at": "2026-10-13T02:51:54Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.seq-on-disk", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Seq is the on-disk index (`F07.seq-on-disk`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-04-seq-is-the-on-disk-index-f07-seq-on-disk.json", "at": "2026-10-13T02:50:23Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.slow-consumer", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Stalled reader (`F07.slow-consumer`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-22-stalled-reader-f07-slow-consumer.json", "at": "2026-10-13T02:53:55Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.snapshot-transient", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: State snapshot (`F07.snapshot-transient`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-02-state-snapshot-f07-snapshot-transient.json", "at": "2026-10-13T02:50:11Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.subscriber-limit", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Subscriber cap (`F07.subscriber-limit`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-23-subscriber-cap-f07-subscriber-limit.json", "at": "2026-10-13T02:54:02Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.sync-first", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Sync comes first (`F07.sync-first`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-01-sync-comes-first-f07-sync-first.json", "at": "2026-10-13T02:50:07Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F07.unknown-conversation", "family": "F07", "result": "pass", "label": "map run F07-session-socket.md: Unknown conversation (`F07.unknown-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-11-unknown-conversation-f07-unknown-convers.json", "at": "2026-10-13T02:51:06Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.always-confirm-wait", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: Wait for confirmation (`F08.always-confirm-wait`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-03-wait-for-confirmation-f08-always-confirm.json", "at": "2026-10-13T02:17:41Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.analyzer-errors", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: Analyzer errors (`F08.analyzer-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-10-analyzer-errors-f08-analyzer-errors.json", "at": "2026-10-13T02:19:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.analyzer-set-clear", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: Set and clear the analyzer (`F08.analyzer-set-clear`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-09-set-and-clear-the-analyzer-f08-analyzer-.json", "at": "2026-10-13T02:19:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.confirm-accept", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: Accept (`F08.confirm-accept`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-04-accept-f08-confirm-accept.json", "at": "2026-10-13T02:17:44Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.confirm-capacity", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: Accept with the run capacity full (`F08.confirm-capacity`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-07-accept-with-the-run-capacity-full-f08-co.json", "at": "2026-10-13T02:18:33Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.confirm-errors", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: Confirmation errors (`F08.confirm-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-06-confirmation-errors-f08-confirm-errors.json", "at": "2026-10-13T02:17:54Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.confirm-real", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: A real model asks first (`F08.confirm-real`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-08-a-real-model-asks-first-f08-confirm-real.json", "at": "2026-10-13T02:18:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.confirm-reject", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: Reject (`F08.confirm-reject`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-05-reject-f08-confirm-reject.json", "at": "2026-10-13T02:17:53Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.confirm-risky", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: Risk-based confirmation (`F08.confirm-risky`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-11-risk-based-confirmation-f08-confirm-risk.json", "at": "2026-10-13T02:19:26Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.policy-errors", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: Policy errors (`F08.policy-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-02-policy-errors-f08-policy-errors.json", "at": "2026-10-13T02:17:34Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.policy-set", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: Set the confirmation policy (`F08.policy-set`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-01-set-the-confirmation-policy-f08-policy-s.json", "at": "2026-10-13T02:17:31Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.secrets-bad-value", "family": "F08", "result": "xfail", "label": "map run F08-confirmation-security-secrets.md: Non-string secret value (`F08.secrets-bad-value`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034314-5b2d-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-034314-5b2d-map-f08/evidence/map-run/F08-confirmation-security-secrets-01-non-string-secret-value-f08-secrets-bad-.json", "at": "2026-10-13T03:43:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 7: test \"$CASE_FAILURE\" -eq 0" }, { "feature_id": "F08.secrets-errors", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: Secret errors (`F08.secrets-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-14-secret-errors-f08-secrets-errors.json", "at": "2026-10-13T02:20:03Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.secrets-lookup", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: Lookup secret (`F08.secrets-lookup`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-18-lookup-secret-f08-secrets-lookup.json", "at": "2026-10-13T02:21:24Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.secrets-lookup-self", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: Lookup of the server's own secret (`F08.secrets-lookup-self`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-19-lookup-of-the-server-s-own-secret-f08-se.json", "at": "2026-10-13T02:22:06Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.secrets-no-cipher", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: No cipher (`F08.secrets-no-cipher`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-21-no-cipher-f08-secrets-no-cipher.json", "at": "2026-10-13T02:25:31Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.secrets-override", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: Override a start-time secret (`F08.secrets-override`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-17-override-a-start-time-secret-f08-secrets.json", "at": "2026-10-13T02:20:48Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.secrets-profile-scope", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: Agent Profile secret scope (`F08.secrets-profile-scope`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-20-agent-profile-secret-scope-f08-secrets-p.json", "at": "2026-10-13T02:22:29Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.secrets-restart", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: Restart with a saved secret (`F08.secrets-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-025113-2057-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-025113-2057-map-f08/evidence/map-run/F08-confirmation-security-secrets-04-restart-with-a-saved-secret-f08-secrets-.json", "at": "2026-10-13T02:52:54Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.secrets-terminal", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: Secrets in the terminal (`F08.secrets-terminal`).", "run": "/workspace/verification-artifacts/runs/20261013-025113-2057-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-025113-2057-map-f08/evidence/map-run/F08-confirmation-security-secrets-03-secrets-in-the-terminal-f08-secrets-term.json", "at": "2026-10-13T02:52:01Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.secrets-unsaved-restart", "family": "F08", "result": "xfail", "label": "map run F08-confirmation-security-secrets.md: Unsaved secret restart (`F08.secrets-unsaved-restart`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025113-2057-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-025113-2057-map-f08/evidence/map-run/F08-confirmation-security-secrets-05-unsaved-secret-restart-f08-secrets-unsav.json", "at": "2026-10-13T02:53:38Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 9: control-agent-server api GET \"/$UNS\" --check secret_registry.secret_sources.QA_F08_UNSAVED.kind eq StaticSecret --save F08.secrets-unsaved-restart/get" }, { "feature_id": "F08.secrets-update", "family": "F08", "result": "pass", "label": "map run F08-confirmation-security-secrets.md: Update secrets (`F08.secrets-update`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-12-update-secrets-f08-secrets-update.json", "at": "2026-10-13T02:19:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F08.secrets-visible", "family": "F08", "result": "xfail", "label": "map run F08-confirmation-security-secrets.md: Runtime secret visible to GET (`F08.secrets-visible`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025113-2057-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-025113-2057-map-f08/evidence/map-run/F08-confirmation-security-secrets-01-runtime-secret-visible-to-get-f08-secret.json", "at": "2026-10-13T02:51:51Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 5: control-agent-server api GET \"/$KID\" --check secret_registry.secret_sources.QA_F08_FRESH.kind eq StaticSecret --save F08.secrets-visible/get" }, { "feature_id": "F09.acp-model-blank", "family": "F09", "result": "xfail", "label": "map run F09-model-switching-plugins.md: Blank ACP model (`F09.acp-model-blank`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-07-blank-acp-model-f09-acp-model-blank-know.json", "at": "2026-10-13T03:05:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 3: control-agent-server api POST \"/$AID2/switch_acp_model\" --json '{\"model\": \" \"}' --expect 400,422 --save F09.acp-model-blank/blank" }, { "feature_id": "F09.acp-model-deferred", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: ACP model before the first run (`F09.acp-model-deferred`).", "run": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-06-acp-model-before-the-first-run-f09-acp-m.json", "at": "2026-10-13T03:05:14Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.acp-model-live", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: Live ACP switch (`F09.acp-model-live`).", "run": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-09-live-acp-switch-f09-acp-model-live.json", "at": "2026-10-13T03:05:46Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.acp-model-live-timeout", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: Live switch that times out (`F09.acp-model-live-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-20-live-switch-that-times-out-f09-acp-model.json", "at": "2026-10-13T02:29:22Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.acp-model-live-unknown", "family": "F09", "result": "xfail", "label": "map run F09-model-switching-plugins.md: Unknown model on a live session (`F09.acp-model-live-unknown`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-10-unknown-model-on-a-live-session-f09-acp-.json", "at": "2026-10-13T03:05:49Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 3: control-agent-server api POST \"/$LID/switch_acp_model\" --json '{\"model\": \"qa-f09-nope\"}' --expect 400,422 --save F09.acp-model-live-unknown/switch" }, { "feature_id": "F09.acp-model-native", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: ACP switch on a regular conversation (`F09.acp-model-native`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-14-acp-switch-on-a-regular-conversation-f09.json", "at": "2026-10-13T02:25:24Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.auth-required", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: Key required (`F09.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-01-key-required-f09-auth-required.json", "at": "2026-10-13T02:18:32Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.load-plugin", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: Load a plugin (`F09.load-plugin`).", "run": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-12-load-a-plugin-f09-load-plugin.json", "at": "2026-10-13T03:05:59Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.load-plugin-errors", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: Plugin load errors (`F09.load-plugin-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-13-plugin-load-errors-f09-load-plugin-error.json", "at": "2026-10-13T03:06:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.load-plugin-hook-runs", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: The loaded hook runs (`F09.load-plugin-hook-runs`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-24-the-loaded-hook-runs-f09-load-plugin-hoo.json", "at": "2026-10-13T02:30:57Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.load-plugin-sdk", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: Load a plugin from the Python SDK (`F09.load-plugin-sdk`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-27-load-a-plugin-from-the-python-sdk-f09-lo.json", "at": "2026-10-13T02:34:29Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.load-plugin-twice", "family": "F09", "result": "xfail", "label": "map run F09-model-switching-plugins.md: Load the same plugin twice (`F09.load-plugin-twice`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-14-load-the-same-plugin-twice-f09-load-plug.json", "at": "2026-10-13T03:06:11Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 4: control-agent-server api GET \"/$QID\" --quiet --check hook_config.user_prompt_submit len-eq 1 --save F09.load-plugin-twice/get" }, { "feature_id": "F09.restart-persist", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: Restart (`F09.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-28-restart-f09-restart-persist.json", "at": "2026-10-13T02:36:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.switch-llm", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: Switch to a caller-supplied LLM (`F09.switch-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-02-switch-to-a-caller-supplied-llm-f09-swit.json", "at": "2026-10-13T02:18:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.switch-llm-acp-refused", "family": "F09", "result": "xfail", "label": "map run F09-model-switching-plugins.md: LLM switches on an ACP conversation (`F09.switch-llm-acp-refused`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-08-llm-switches-on-an-acp-conversation-f09-.json", "at": "2026-10-13T03:05:19Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 9: test \"$CASE_FAILURE\" -eq 0" }, { "feature_id": "F09.switch-llm-before-first-run", "family": "F09", "result": "xfail", "label": "map run F09-model-switching-plugins.md: Switch before the first run (`F09.switch-llm-before-first-run`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-03-switch-before-the-first-run-f09-switch-l.json", "at": "2026-10-13T03:03:56Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 7: control-agent-server api GET \"/$BID\" --until-ok 20 --quiet --check stats.usage_to_metrics.qa-f09-early.accumulated_token_usage.prompt_tokens gt 0 --save F09.switch-llm-before-first-run/stats" }, { "feature_id": "F09.switch-llm-encrypted-key", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: Mid-conversation switch with a cipher token (`F09.switch-llm-encrypted-key`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-05-mid-conversation-switch-with-a-cipher-to.json", "at": "2026-10-13T02:19:07Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.switch-llm-errors", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: Switch errors (`F09.switch-llm-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-04-switch-errors-f09-switch-llm-errors.json", "at": "2026-10-13T02:18:58Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.switch-llm-mid-run", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: Switch during a running turn (`F09.switch-llm-mid-run`).", "run": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-04-switch-during-a-running-turn-f09-switch-.json", "at": "2026-10-13T03:04:35Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.switch-llm-mid-run-get", "family": "F09", "result": "xfail", "label": "map run F09-model-switching-plugins.md: GET right after a mid-run switch (`F09.switch-llm-mid-run-get`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-05-get-right-after-a-mid-run-switch-f09-swi.json", "at": "2026-10-13T03:05:10Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 12: control-agent-server api GET \"/$GID\" --quiet --check agent.llm.usage_id eq qa-f09-midget --save F09.switch-llm-mid-run-get/get" }, { "feature_id": "F09.switch-llm-provider-connection", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: Switch through a provider connection (`F09.switch-llm-provider-connection`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-06-switch-through-a-provider-connection-f09.json", "at": "2026-10-13T02:19:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.switch-llm-registered-usage-id", "family": "F09", "result": "xfail", "label": "map run F09-model-switching-plugins.md: Switch onto a registered usage id (`F09.switch-llm-registered-usage-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-02-switch-onto-a-registered-usage-id-f09-sw.json", "at": "2026-10-13T03:03:30Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 9: control-agent-server api GET \"/$KID\" --quiet --check agent.llm.model eq deepseek/deepseek-v4-pro --save F09.switch-llm-registered-usage-id/get" }, { "feature_id": "F09.switch-llm-ws", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: Socket view of a switch (`F09.switch-llm-ws`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-03-socket-view-of-a-switch-f09-switch-llm-w.json", "at": "2026-10-13T02:18:53Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.switch-profile", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: Switch to a saved profile (`F09.switch-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-07-switch-to-a-saved-profile-f09-switch-pro.json", "at": "2026-10-13T02:19:27Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.switch-profile-edited", "family": "F09", "result": "xfail", "label": "map run F09-model-switching-plugins.md: Switch to an edited profile (`F09.switch-profile-edited`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-01-switch-to-an-edited-profile-f09-switch-p.json", "at": "2026-10-13T03:03:22Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 9: control-agent-server api GET \"/$EID\" --quiet --check agent.llm.model eq openai/qa-f09-edit-2 --save F09.switch-profile-edited/get" }, { "feature_id": "F09.switch-profile-errors", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: Profile switch errors (`F09.switch-profile-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-08-profile-switch-errors-f09-switch-profile.json", "at": "2026-10-13T02:19:35Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.ts-client", "family": "F09", "result": "pass", "label": "map run F09-model-switching-plugins.md: TypeScript client (`F09.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-21-typescript-client-f09-ts-client.json", "at": "2026-10-13T02:29:56Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F09.ts-client-acp-docs", "family": "F09", "result": "xfail", "label": "map run F09-model-switching-plugins.md: TypeScript docstrings for a pre-run ACP switch (`F09.ts-client-acp-docs`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-11-typescript-docstrings-for-a-pre-run-acp-.json", "at": "2026-10-13T03:05:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 3: false" }, { "feature_id": "F10.ask-agent", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: Ask a side question (`F10.ask-agent`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-20-ask-a-side-question-f10-ask-agent.json", "at": "2026-10-13T02:27:57Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F10.ask-agent-errors", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: Bad input, no key, unreachable model (`F10.ask-agent-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-21-bad-input-no-key-unreachable-model-f10-a.json", "at": "2026-10-13T02:29:29Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F10.ask-agent-running", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: Side question during a run (`F10.ask-agent-running`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-22-side-question-during-a-run-f10-ask-agent.json", "at": "2026-10-13T02:30:15Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F10.ask-agent-unknown", "family": "F10", "result": "xfail", "label": "map run F10-fork-navigate-condense-ask.md: Unknown conversation (`F10.ask-agent-unknown`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-11-unknown-conversation-f10-ask-agent-unkno.json", "at": "2026-10-13T02:45:07Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 3: control-agent-server api POST \"/$NOPE/ask_agent\" --json '{\"question\": \"hi\"}' --expect 404 --save F10.ask-agent-unknown/unknown" }, { "feature_id": "F10.condense-errors", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: Unknown, malformed and unauthenticated (`F10.condense-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-19-unknown-malformed-and-unauthenticated-f1.json", "at": "2026-10-13T02:27:24Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F10.condense-llm", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: Condense with DeepSeek (`F10.condense-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-15-condense-with-deepseek-f10-condense-llm.json", "at": "2026-10-13T02:24:58Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F10.condense-llm-failure", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: Unreachable condenser model (`F10.condense-llm-failure`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-18-unreachable-condenser-model-f10-condense.json", "at": "2026-10-13T02:27:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F10.condense-no-condenser", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: No request-handling condenser (`F10.condense-no-condenser`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-16-no-request-handling-condenser-f10-conden.json", "at": "2026-10-13T02:26:01Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F10.condense-status", "family": "F10", "result": "xfail", "label": "map run F10-fork-navigate-condense-ask.md: Refusal status (`F10.condense-status`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-10-refusal-status-f10-condense-status-known.json", "at": "2026-10-13T02:45:03Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 2: control-agent-server api POST \"/$NC/condense\" --expect 4xx --save F10.condense-status/disabled" }, { "feature_id": "F10.fork-errors", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: Unknown, taken and invalid input (`F10.fork-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-06-unknown-taken-and-invalid-input-f10-fork.json", "at": "2026-10-13T02:22:04Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F10.fork-from-event", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: Fork from an event (`F10.fork-from-event`).", "run": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-03-fork-from-an-event-f10-fork-from-event.json", "at": "2026-10-13T02:44:23Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F10.fork-independent", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: Fork and source diverge (`F10.fork-independent`).", "run": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-04-fork-and-source-diverge-f10-fork-indepen.json", "at": "2026-10-13T02:44:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F10.fork-keeps-credentials", "family": "F10", "result": "xfail", "label": "map run F10-fork-navigate-condense-ask.md: The fork keeps the model keys (`F10.fork-keeps-credentials`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-06-the-fork-keeps-the-model-keys-f10-fork-k.json", "at": "2026-10-13T02:44:49Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 3: control-agent-server state cat base_state.json --conversation \"$KF\" --max-chars 200 --check agent.llm.api_key exists --check agent.condenser.llm.api_key exists" }, { "feature_id": "F10.fork-metrics", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: Metrics on a fork (`F10.fork-metrics`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-04-metrics-on-a-fork-f10-fork-metrics.json", "at": "2026-10-13T02:21:10Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F10.fork-named", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: Named fork (`F10.fork-named`).", "run": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-02-named-fork-f10-fork-named.json", "at": "2026-10-13T02:44:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F10.fork-run-memory", "family": "F10", "result": "xfail", "label": "map run F10-fork-navigate-condense-ask.md: A fork runs with the source's memory (`F10.fork-run-memory`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-07-a-fork-runs-with-the-source-s-memory-f10.json", "at": "2026-10-13T02:44:56Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 4: control-agent-server conversation events \"$AF\" --kinds ConversationErrorEvent --contains LLMAuthenticationError --expect-count 0 --save F10.fork-run-memory/auth-errors" }, { "feature_id": "F10.fork-tags-consistent", "family": "F10", "result": "xfail", "label": "map run F10-fork-navigate-condense-ask.md: Fork tags agree with the stored metadata (`F10.fork-tags-consistent`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-05-fork-tags-agree-with-the-stored-metadata.json", "at": "2026-10-13T02:44:47Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 7: control-agent-server state cat meta.json --conversation \"$TF\" --check title eq 'QA F10 tag fork' --check tags.qa eq \"$TF_QA\"" }, { "feature_id": "F10.fork-whole", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: Fork the whole log (`F10.fork-whole`).", "run": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-01-fork-the-whole-log-f10-fork-whole.json", "at": "2026-10-13T02:44:12Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F10.navigate-branch", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: The next event branches (`F10.navigate-branch`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-11-the-next-event-branches-f10-navigate-bra.json", "at": "2026-10-13T02:23:33Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F10.navigate-empty", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: Navigate to the empty tree (`F10.navigate-empty`).", "run": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-08-navigate-to-the-empty-tree-f10-navigate-.json", "at": "2026-10-13T02:44:59Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F10.navigate-errors", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: Unknown, missing and unauthenticated (`F10.navigate-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-13-unknown-missing-and-unauthenticated-f10-.json", "at": "2026-10-13T02:24:07Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F10.navigate-event", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: Navigate to an earlier event (`F10.navigate-event`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-10-navigate-to-an-earlier-event-f10-navigat.json", "at": "2026-10-13T02:23:09Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F10.navigate-final-response", "family": "F10", "result": "xfail", "label": "map run F10-fork-navigate-condense-ask.md: Final response after navigating back (`F10.navigate-final-response`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-09-final-response-after-navigating-back-f10.json", "at": "2026-10-13T02:45:01Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 5: control-agent-server api GET \"/$AN/agent_final_response\" --check response eq '' --save F10.navigate-final-response/after-navigate" }, { "feature_id": "F10.restart-cold-calls", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: Condense and ask on a cold conversation (`F10.restart-cold-calls`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-25-condense-and-ask-on-a-cold-conversation-.json", "at": "2026-10-13T02:34:45Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F10.restart-persist", "family": "F10", "result": "pass", "label": "map run F10-fork-navigate-condense-ask.md: State after a restart (`F10.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-12-state-after-a-restart-f10-restart-persis.json", "at": "2026-10-13T02:46:20Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-auth", "family": "F11", "result": "pass", "label": "map run F11-goals.md: Authentication (`F11.goal-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-03-authentication-f11-goal-auth.json", "at": "2026-10-13T02:22:47Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-busy", "family": "F11", "result": "pass", "label": "map run F11-goals.md: Busy while a round is in flight (`F11.goal-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-12-busy-while-a-round-is-in-flight-f11-goal.json", "at": "2026-10-13T02:25:49Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-busy-run", "family": "F11", "result": "pass", "label": "map run F11-goals.md: Busy with an ordinary run (`F11.goal-busy-run`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-17-busy-with-an-ordinary-run-f11-goal-busy-.json", "at": "2026-10-13T02:27:56Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-cap-overrun", "family": "F11", "result": "xfail", "label": "map run F11-goals.md: A resume after a failed last audit overruns the cap (`F11.goal-cap-overrun`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-031830-28b2-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-031830-28b2-map-f11/evidence/map-run/F11-goals-02-a-resume-after-a-failed-last-audit-overr.json", "at": "2026-10-13T03:19:14Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 10: jq -e '[.events[].value | select(.iteration > .max_iterations)] | length == 0'" }, { "feature_id": "F11.goal-capacity", "family": "F11", "result": "pass", "label": "map run F11-goals.md: Capacity (`F11.goal-capacity`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-22-capacity-f11-goal-capacity.json", "at": "2026-10-13T02:34:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-capped", "family": "F11", "result": "pass", "label": "map run F11-goals.md: Round cap (`F11.goal-capped`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-10-round-cap-f11-goal-capped.json", "at": "2026-10-13T02:24:54Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-complete", "family": "F11", "result": "pass", "label": "map run F11-goals.md: A goal on a real model (`F11.goal-complete`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-06-a-goal-on-a-real-model-f11-goal-complete.json", "at": "2026-10-13T02:24:00Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-delete-active", "family": "F11", "result": "pass", "label": "map run F11-goals.md: Delete a conversation with an active loop (`F11.goal-delete-active`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-23-delete-a-conversation-with-an-active-loo.json", "at": "2026-10-13T02:35:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-followup", "family": "F11", "result": "pass", "label": "map run F11-goals.md: Incomplete verdict re-prompts (`F11.goal-followup`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-09-incomplete-verdict-re-prompts-f11-goal-f.json", "at": "2026-10-13T02:24:32Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-in-place", "family": "F11", "result": "pass", "label": "map run F11-goals.md: Same conversation, no new one (`F11.goal-in-place`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-08-same-conversation-no-new-one-f11-goal-in.json", "at": "2026-10-13T02:24:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-interrupt-run", "family": "F11", "result": "pass", "label": "map run F11-goals.md: Interrupt the round (`F11.goal-interrupt-run`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-16-interrupt-the-round-f11-goal-interrupt-r.json", "at": "2026-10-13T02:27:35Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-judge-error", "family": "F11", "result": "pass", "label": "map run F11-goals.md: The judge fails (`F11.goal-judge-error`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-21-the-judge-fails-f11-goal-judge-error.json", "at": "2026-10-13T02:32:30Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-restart-active", "family": "F11", "result": "xfail", "label": "map run F11-goals.md: A loop active at a restart stays \"running\" (`F11.goal-restart-active`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-031830-28b2-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-031830-28b2-map-f11/evidence/map-run/F11-goals-03-a-loop-active-at-a-restart-stays-running.json", "at": "2026-10-13T03:19:37Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 7: jq -e '.events[-1].value.active == false and .events[-1].value.status == \"interrupted\"'" }, { "feature_id": "F11.goal-restart-resume", "family": "F11", "result": "pass", "label": "map run F11-goals.md: Resume after a restart (`F11.goal-restart-resume`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-19-resume-after-a-restart-f11-goal-restart-.json", "at": "2026-10-13T02:30:14Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-resume", "family": "F11", "result": "pass", "label": "map run F11-goals.md: Resume a stopped loop (`F11.goal-resume`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-18-resume-a-stopped-loop-f11-goal-resume.json", "at": "2026-10-13T02:28:22Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-resume-finished", "family": "F11", "result": "pass", "label": "map run F11-goals.md: Nothing to resume after the end (`F11.goal-resume-finished`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-11-nothing-to-resume-after-the-end-f11-goal.json", "at": "2026-10-13T02:25:19Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-resume-none", "family": "F11", "result": "pass", "label": "map run F11-goals.md: Resume with no goal (`F11.goal-resume-none`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-05-resume-with-no-goal-f11-goal-resume-none.json", "at": "2026-10-13T02:23:07Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-run-error", "family": "F11", "result": "pass", "label": "map run F11-goals.md: The round's provider fails (`F11.goal-run-error`).", "run": "/workspace/verification-artifacts/runs/20261013-031830-28b2-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-031830-28b2-map-f11/evidence/map-run/F11-goals-01-the-round-s-provider-fails-f11-goal-run-.json", "at": "2026-10-13T03:19:04Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-stop", "family": "F11", "result": "pass", "label": "map run F11-goals.md: Stop an active loop (`F11.goal-stop`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-13-stop-an-active-loop-f11-goal-stop.json", "at": "2026-10-13T02:26:22Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-stop-noop", "family": "F11", "result": "pass", "label": "map run F11-goals.md: Stop with nothing to stop (`F11.goal-stop-noop`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-04-stop-with-nothing-to-stop-f11-goal-stop-.json", "at": "2026-10-13T02:22:57Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-superseded", "family": "F11", "result": "pass", "label": "map run F11-goals.md: A user message supersedes the loop (`F11.goal-superseded`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-14-a-user-message-supersedes-the-loop-f11-g.json", "at": "2026-10-13T02:26:41Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-superseded-socket", "family": "F11", "result": "pass", "label": "map run F11-goals.md: A socket message supersedes the loop (`F11.goal-superseded-socket`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-15-a-socket-message-supersedes-the-loop-f11.json", "at": "2026-10-13T02:27:09Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-unknown-id", "family": "F11", "result": "pass", "label": "map run F11-goals.md: Unknown and malformed ids (`F11.goal-unknown-id`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-02-unknown-and-malformed-ids-f11-goal-unkno.json", "at": "2026-10-13T02:22:29Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-validation", "family": "F11", "result": "pass", "label": "map run F11-goals.md: Invalid goal requests (`F11.goal-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-01-invalid-goal-requests-f11-goal-validatio.json", "at": "2026-10-13T02:22:14Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F11.goal-ws", "family": "F11", "result": "pass", "label": "map run F11-goals.md: Goal status on the socket (`F11.goal-ws`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-07-goal-status-on-the-socket-f11-goal-ws.json", "at": "2026-10-13T02:24:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.binding-live-acp", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Bind a loaded ACP Codex conversation (`F12.binding-live-acp`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-18-bind-a-loaded-acp-codex-conversation-f12.json", "at": "2026-10-13T02:30:47Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.binding-native-cold", "family": "F12", "result": "xfail", "label": "map run F12-runtime-and-credentials.md: Regular conversation, not loaded (`F12.binding-native-cold`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12/evidence/map-run/F12-runtime-and-credentials-02-regular-conversation-not-loaded-f12-bind.json", "at": "2026-10-13T03:06:20Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 6: control-agent-server api GET \"/$N/events/count\" --expect 200 --save F12.binding-native-cold/events-after-pause" }, { "feature_id": "F12.binding-native-too-late", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Regular conversation, loaded (`F12.binding-native-too-late`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-16-regular-conversation-loaded-f12-binding-.json", "at": "2026-10-13T02:29:51Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.binding-pending", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Pending binding for an unknown id (`F12.binding-pending`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-13-pending-binding-for-an-unknown-id-f12-bi.json", "at": "2026-10-13T02:27:12Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.binding-pending-cleared", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: A pause drops pending bindings (`F12.binding-pending-cleared`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-23-a-pause-drops-pending-bindings-f12-bindi.json", "at": "2026-10-13T02:35:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.binding-pending-on-create", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Pending binding consumed at create (`F12.binding-pending-on-create`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-22-pending-binding-consumed-at-create-f12-b.json", "at": "2026-10-13T02:35:33Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.binding-profile-scope", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Profile excludes the secret (`F12.binding-profile-scope`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-24-profile-excludes-the-secret-f12-binding-.json", "at": "2026-10-13T02:36:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.binding-rebind", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Re-bind (`F12.binding-rebind`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-19-re-bind-f12-binding-rebind.json", "at": "2026-10-13T02:31:19Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.binding-required-after-pause", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Required after a pause (`F12.binding-required-after-pause`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-20-required-after-a-pause-f12-binding-requi.json", "at": "2026-10-13T02:32:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.binding-required-after-restart", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Required after a restart (`F12.binding-required-after-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-21-required-after-a-restart-f12-binding-req.json", "at": "2026-10-13T02:35:14Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.binding-source-statuses", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Source answers (`F12.binding-source-statuses`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-14-source-answers-f12-binding-source-status.json", "at": "2026-10-13T02:29:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.binding-source-unavailable", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Unavailable source (`F12.binding-source-unavailable`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-15-unavailable-source-f12-binding-source-un.json", "at": "2026-10-13T02:29:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.binding-unsupported-secret", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Unsupported secret names (`F12.binding-unsupported-secret`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-11-unsupported-secret-names-f12-binding-uns.json", "at": "2026-10-13T02:26:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.binding-validation", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Body validation (`F12.binding-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-12-body-validation-f12-binding-validation.json", "at": "2026-10-13T02:26:53Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.discovery", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Discovery (`F12.discovery`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-01-discovery-f12-discovery.json", "at": "2026-10-13T02:22:01Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.docker-runtime", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Docker runtime with a daemon (`F12.docker-runtime`).", "run": "/workspace/verification-artifacts/runs/20261013-040101-d29e-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-040101-d29e-map-f12/evidence/map-run/F12-runtime-and-credentials-01-docker-runtime-with-a-daemon-f12-docker-.json", "at": "2026-10-13T04:08:00Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.docker-runtime-contract", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Docker runtime mode without a daemon (`F12.docker-runtime-contract`).", "run": "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12/evidence/map-run/F12-runtime-and-credentials-06-docker-runtime-mode-without-a-daemon-f12.json", "at": "2026-10-13T03:08:52Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.managed-key-refresh", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Managed key refresh on a 401 (`F12.managed-key-refresh`).", "run": "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12/evidence/map-run/F12-runtime-and-credentials-03-managed-key-refresh-on-a-401-f12-managed.json", "at": "2026-10-13T03:06:56Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.managed-key-scope", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Refresh scope (`F12.managed-key-scope`).", "run": "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12/evidence/map-run/F12-runtime-and-credentials-04-refresh-scope-f12-managed-key-scope.json", "at": "2026-10-13T03:07:52Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.managed-key-switch", "family": "F12", "result": "xfail", "label": "map run F12-runtime-and-credentials.md: Refresh after a model switch (`F12.managed-key-switch`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12/evidence/map-run/F12-runtime-and-credentials-05-refresh-after-a-model-switch-f12-managed.json", "at": "2026-10-13T03:08:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 19: control-agent-server sink read --name qa-f12-keysrc-switch --expect-min 2 --expect-max 2 --save F12.managed-key-switch/key-fetch" }, { "feature_id": "F12.runtime-cold", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Runtime of an unloaded conversation (`F12.runtime-cold`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-07-runtime-of-an-unloaded-conversation-f12-.json", "at": "2026-10-13T02:24:10Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.runtime-errors", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Runtime errors (`F12.runtime-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-04-runtime-errors-f12-runtime-errors.json", "at": "2026-10-13T02:22:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.runtime-info", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Runtime info (`F12.runtime-info`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-02-runtime-info-f12-runtime-info.json", "at": "2026-10-13T02:22:11Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.runtime-launched-profile", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Launched profile from the runtime (`F12.runtime-launched-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-25-launched-profile-from-the-runtime-f12-ru.json", "at": "2026-10-13T02:37:24Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.runtime-reprovision", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Reprovision (`F12.runtime-reprovision`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-03-reprovision-f12-runtime-reprovision.json", "at": "2026-10-13T02:22:28Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.sandbox-pause-rehydrate", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Rehydrate after a pause (`F12.sandbox-pause-rehydrate`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-08-rehydrate-after-a-pause-f12-sandbox-paus.json", "at": "2026-10-13T02:24:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.sandbox-pause-releases", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Pause releases everything (`F12.sandbox-pause-releases`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-06-pause-releases-everything-f12-sandbox-pa.json", "at": "2026-10-13T02:23:03Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.sandbox-pause-running", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: Pause a running conversation (`F12.sandbox-pause-running`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-09-pause-a-running-conversation-f12-sandbox.json", "at": "2026-10-13T02:25:15Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F12.sandbox-pause-websocket", "family": "F12", "result": "xfail", "label": "map run F12-runtime-and-credentials.md: Open sockets across a pause (`F12.sandbox-pause-websocket`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12/evidence/map-run/F12-runtime-and-credentials-01-open-sockets-across-a-pause-f12-sandbox-.json", "at": "2026-10-13T03:06:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 17: jq -e '.close != null or ((.kinds.MessageEvent // 0) >= 1)'" }, { "feature_id": "F12.ts-client", "family": "F12", "result": "pass", "label": "map run F12-runtime-and-credentials.md: TypeScript client (`F12.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12", "transcript": "/workspace/verification-artifacts/runs/20261013-022033-bf39-map-f12/evidence/map-run/F12-runtime-and-credentials-05-typescript-client-f12-ts-client.json", "at": "2026-10-13T02:22:46Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.auth", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Auth (`F13.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-01-auth-f13-auth.json", "at": "2026-10-13T02:17:20Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.clear", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Clear (`F13.clear`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-25-clear-f13-clear.json", "at": "2026-10-13T02:23:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.env-config-credentials", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Other credential variables (`F13.env-config-credentials`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-05-other-credential-variables-f13-env-confi.json", "at": "2026-10-13T02:18:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.env-sanitized", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Sanitized environment (`F13.env-sanitized`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-04-sanitized-environment-f13-env-sanitized.json", "at": "2026-10-13T02:17:45Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.execute-large-output", "family": "F13", "result": "xfail", "label": "map run F13-bash.md: Large output through execute (`F13.execute-large-output`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13/evidence/map-run/F13-bash-04-large-output-through-execute-f13-execute.json", "at": "2026-10-13T03:12:47Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 1: control-agent-server api POST /api/bash/execute_bash_command --json '{\"command\": \"yes a | head -c 1100000; echo QA_F13_TAIL\", \"timeout\": 60}' --expect 200 --check exit_code eq 0 --check stdout len-eq 1100012 --quiet --save F13.execute-large-output/execute" }, { "feature_id": "F13.execute-sync", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Synchronous execute (`F13.execute-sync`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-02-synchronous-execute-f13-execute-sync.json", "at": "2026-10-13T02:17:22Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.get-events", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Single and batch reads (`F13.get-events`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-11-single-and-batch-reads-f13-get-events.json", "at": "2026-10-13T02:18:44Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.global-cwd", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Global cwd (`F13.global-cwd`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-03-global-cwd-f13-global-cwd.json", "at": "2026-10-13T02:17:23Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.order-gt-paging", "family": "F13", "result": "xfail", "label": "map run F13-bash.md: Order filter and paging (`F13.order-gt-paging`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13/evidence/map-run/F13-bash-03-order-filter-and-paging-f13-order-gt-pag.json", "at": "2026-10-13T03:12:47Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 4: control-agent-server api GET /api/bash/bash_events/search --query command_id__eq=\"$BIG_CMD\" --query order__gt=0 --query limit=1 --check items len-eq 1 --check items.0.order eq 1 --quiet --save F13.order-gt-paging/limit-1" }, { "feature_id": "F13.output-chunks", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Output chunks (`F13.output-chunks`).", "run": "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13/evidence/map-run/F13-bash-02-output-chunks-f13-output-chunks.json", "at": "2026-10-13T03:12:46Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.persist-restart", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Restart (`F13.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-24-restart-f13-persist-restart.json", "at": "2026-10-13T02:22:45Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.retention", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Retention (`F13.retention`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-26-retention-f13-retention.json", "at": "2026-10-13T02:25:44Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.retention-rolling", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Rolling retention (`F13.retention-rolling`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-27-rolling-retention-f13-retention-rolling.json", "at": "2026-10-13T02:29:28Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.scoped-bad-id", "family": "F13", "result": "xfail", "label": "map run F13-bash.md: Malformed conversation id (`F13.scoped-bad-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13/evidence/map-run/F13-bash-05-malformed-conversation-id-f13-scoped-bad.json", "at": "2026-10-13T03:12:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 7: test \"$CASE_FAILURE\" -eq 0" }, { "feature_id": "F13.scoped-cwd", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Scoped cwd (`F13.scoped-cwd`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-20-scoped-cwd-f13-scoped-cwd.json", "at": "2026-10-13T02:20:25Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.scoped-idle-eviction", "family": "F13", "result": "xfail", "label": "map run F13-bash.md: Idle eviction kills a running command (`F13.scoped-idle-eviction`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13/evidence/map-run/F13-bash-06-idle-eviction-kills-a-running-command-f1.json", "at": "2026-10-13T03:14:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 19: pgrep -f \"$EV_MARK\"" }, { "feature_id": "F13.scoped-isolation", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Scope isolation (`F13.scoped-isolation`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-22-scope-isolation-f13-scoped-isolation.json", "at": "2026-10-13T02:21:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.scoped-routes", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Scoped routes (`F13.scoped-routes`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-21-scoped-routes-f13-scoped-routes.json", "at": "2026-10-13T02:20:43Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.scoped-unknown", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Unknown conversation (`F13.scoped-unknown`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-23-unknown-conversation-f13-scoped-unknown.json", "at": "2026-10-13T02:21:34Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.search", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Search (`F13.search`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-12-search-f13-search.json", "at": "2026-10-13T02:19:00Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.start-async", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Asynchronous start (`F13.start-async`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-06-asynchronous-start-f13-start-async.json", "at": "2026-10-13T02:18:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.stop", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Stop one command (`F13.stop`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-18-stop-one-command-f13-stop.json", "at": "2026-10-13T02:19:31Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.stop-idempotent", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Stop is idempotent (`F13.stop-idempotent`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-19-stop-is-idempotent-f13-stop-idempotent.json", "at": "2026-10-13T02:19:41Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.timeout", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Timeout (`F13.timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-17-timeout-f13-timeout.json", "at": "2026-10-13T02:19:22Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.timestamp-offset", "family": "F13", "result": "xfail", "label": "map run F13-bash.md: Offset timestamps (`F13.timestamp-offset`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13/evidence/map-run/F13-bash-01-offset-timestamps-f13-timestamp-offset-k.json", "at": "2026-10-13T03:12:40Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 7: control-agent-server api GET /api/bash/bash_events/search --query kind__eq=BashCommand --query timestamp__gte=\"$GTE_PLUS2\" --check items contains \"$TZ_CMD\" --save F13.timestamp-offset/plus-two" }, { "feature_id": "F13.ws-command", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Commands over the socket (`F13.ws-command`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-08-commands-over-the-socket-f13-ws-command.json", "at": "2026-10-13T02:18:25Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.ws-live", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Live and replayed socket frames (`F13.ws-live`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-07-live-and-replayed-socket-frames-f13-ws-l.json", "at": "2026-10-13T02:18:23Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.ws-resend-all", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Deprecated replay flag (`F13.ws-resend-all`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-09-deprecated-replay-flag-f13-ws-resend-all.json", "at": "2026-10-13T02:18:28Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F13.ws-subscriber-cap", "family": "F13", "result": "pass", "label": "map run F13-bash.md: Subscriber cap (`F13.ws-subscriber-cap`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-10-subscriber-cap-f13-ws-subscriber-cap.json", "at": "2026-10-13T02:18:34Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.archive-auto-base", "family": "F14", "result": "pass", "label": "map run F14-files.md: Automatic base with a remote (`F14.archive-auto-base`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-17-automatic-base-with-a-remote-f14-archive.json", "at": "2026-10-13T02:28:33Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.archive-errors", "family": "F14", "result": "pass", "label": "map run F14-files.md: Archive validation (`F14.archive-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-20-archive-validation-f14-archive-errors.json", "at": "2026-10-13T02:29:38Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.archive-excludes", "family": "F14", "result": "pass", "label": "map run F14-files.md: Exclude controls (`F14.archive-excludes`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-15-exclude-controls-f14-archive-excludes.json", "at": "2026-10-13T02:28:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.archive-format-doc", "family": "F14", "result": "xfail", "label": "map run F14-files.md: What `format` promises (`F14.archive-format-doc`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/map-run/F14-files-06-what-format-promises-f14-archive-format-.json", "at": "2026-10-13T02:49:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 12: false" }, { "feature_id": "F14.archive-git-delta", "family": "F14", "result": "pass", "label": "map run F14-files.md: git-delta patch (`F14.archive-git-delta`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-16-git-delta-patch-f14-archive-git-delta.json", "at": "2026-10-13T02:28:26Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.archive-nested-repo", "family": "F14", "result": "pass", "label": "map run F14-files.md: Nested repository (`F14.archive-nested-repo`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-18-nested-repository-f14-archive-nested-rep.json", "at": "2026-10-13T02:28:45Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.archive-tar", "family": "F14", "result": "pass", "label": "map run F14-files.md: tar.gz archive (`F14.archive-tar`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-14-tar-gz-archive-f14-archive-tar.json", "at": "2026-10-13T02:28:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.auth", "family": "F14", "result": "pass", "label": "map run F14-files.md: Session key required (`F14.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-01-session-key-required-f14-auth.json", "at": "2026-10-13T02:23:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.download-errors", "family": "F14", "result": "pass", "label": "map run F14-files.md: Download errors (`F14.download-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-07-download-errors-f14-download-errors.json", "at": "2026-10-13T02:26:48Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.home", "family": "F14", "result": "pass", "label": "map run F14-files.md: Home and favorites (`F14.home`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-09-home-and-favorites-f14-home.json", "at": "2026-10-13T02:27:20Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.mkdir", "family": "F14", "result": "pass", "label": "map run F14-files.md: Create directories (`F14.mkdir`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-08-create-directories-f14-mkdir.json", "at": "2026-10-13T02:27:15Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.restart", "family": "F14", "result": "pass", "label": "map run F14-files.md: Restart (`F14.restart`).", "run": "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/map-run/F14-files-10-restart-f14-restart.json", "at": "2026-10-13T02:50:27Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.scoped-conversation-errors", "family": "F14", "result": "pass", "label": "map run F14-files.md: Unknown or malformed conversation (`F14.scoped-conversation-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-26-unknown-or-malformed-conversation-f14-sc.json", "at": "2026-10-13T02:33:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.scoped-path-guard", "family": "F14", "result": "pass", "label": "map run F14-files.md: Workspace path guard (`F14.scoped-path-guard`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-25-workspace-path-guard-f14-scoped-path-gua.json", "at": "2026-10-13T02:32:57Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.scoped-trajectory-archive", "family": "F14", "result": "pass", "label": "map run F14-files.md: Scoped trajectory and archive (`F14.scoped-trajectory-archive`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-27-scoped-trajectory-and-archive-f14-scoped.json", "at": "2026-10-13T02:34:09Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.scoped-transfer", "family": "F14", "result": "pass", "label": "map run F14-files.md: Conversation-scoped transfer (`F14.scoped-transfer`).", "run": "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/map-run/F14-files-09-conversation-scoped-transfer-f14-scoped-.json", "at": "2026-10-13T02:49:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.sdk-workspace", "family": "F14", "result": "pass", "label": "map run F14-files.md: SDK workspace (`F14.sdk-workspace`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-28-sdk-workspace-f14-sdk-workspace.json", "at": "2026-10-13T02:35:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.subdirs", "family": "F14", "result": "pass", "label": "map run F14-files.md: List subdirectories (`F14.subdirs`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-10-list-subdirectories-f14-subdirs.json", "at": "2026-10-13T02:27:27Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.subdirs-case-pagination", "family": "F14", "result": "xfail", "label": "map run F14-files.md: Paging past a case-only twin (`F14.subdirs-case-pagination`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/map-run/F14-files-05-paging-past-a-case-only-twin-f14-subdirs.json", "at": "2026-10-13T02:49:15Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 9: control-agent-server api GET /api/file/search_subdirs --query path=\"$CASE_DIR\" --query limit=1 --query page_id=\"$CASE_NEXT\" --expect 200 --check items len-eq 1 --check items.0.name eq \"$CASE_OTHER\" --check next_page_id missing --save F14.subdirs-case-pagination/page-2" }, { "feature_id": "F14.subdirs-errors", "family": "F14", "result": "pass", "label": "map run F14-files.md: Listing errors (`F14.subdirs-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-12-listing-errors-f14-subdirs-errors.json", "at": "2026-10-13T02:27:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.subdirs-pagination", "family": "F14", "result": "pass", "label": "map run F14-files.md: Pagination (`F14.subdirs-pagination`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-11-pagination-f14-subdirs-pagination.json", "at": "2026-10-13T02:27:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.trajectory", "family": "F14", "result": "pass", "label": "map run F14-files.md: Trajectory zip (`F14.trajectory`).", "run": "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/map-run/F14-files-07-trajectory-zip-f14-trajectory.json", "at": "2026-10-13T02:49:25Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.trajectory-concurrent", "family": "F14", "result": "xfail", "label": "map run F14-files.md: Concurrent trajectory downloads (`F14.trajectory-concurrent`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/map-run/F14-files-08-concurrent-trajectory-downloads-f14-traj.json", "at": "2026-10-13T02:49:37Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 22: test -s \"${s%.status}.zip\"" }, { "feature_id": "F14.trajectory-errors", "family": "F14", "result": "pass", "label": "map run F14-files.md: Trajectory errors (`F14.trajectory-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-22-trajectory-errors-f14-trajectory-errors.json", "at": "2026-10-13T02:30:00Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.trajectory-init-path", "family": "F14", "result": "xfail", "label": "map run F14-files.md: Init-time conversations path (`F14.trajectory-init-path`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034324-233c-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-034324-233c-map-f14/evidence/map-run/F14-files-02-init-time-conversations-path-f14-traject.json", "at": "2026-10-13T03:44:31Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 15: test \"$CASE_FAILURE\" -eq 0" }, { "feature_id": "F14.upload-atomic", "family": "F14", "result": "xfail", "label": "map run F14-files.md: Failed overwrite (`F14.upload-atomic`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/map-run/F14-files-04-failed-overwrite-f14-upload-atomic-known.json", "at": "2026-10-13T02:49:12Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 16: cmp \"$F/v2.txt\" \"$AT/target.txt\"" }, { "feature_id": "F14.upload-download", "family": "F14", "result": "pass", "label": "map run F14-files.md: Upload and download round trip (`F14.upload-download`).", "run": "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/map-run/F14-files-01-upload-and-download-round-trip-f14-uploa.json", "at": "2026-10-13T02:47:46Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.upload-errors", "family": "F14", "result": "pass", "label": "map run F14-files.md: Upload validation (`F14.upload-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-022038-d620-map-f14/evidence/map-run/F14-files-03-upload-validation-f14-upload-errors.json", "at": "2026-10-13T02:23:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F14.upload-onto-directory", "family": "F14", "result": "xfail", "label": "map run F14-files.md: Upload onto a directory (`F14.upload-onto-directory`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034324-233c-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-034324-233c-map-f14/evidence/map-run/F14-files-01-upload-onto-a-directory-f14-upload-onto-.json", "at": "2026-10-13T03:44:07Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 9: test \"$CASE_FAILURE\" -eq 0" }, { "feature_id": "F14.upload-size", "family": "F14", "result": "xfail", "label": "map run F14-files.md: Upload size in the consumers' result (`F14.upload-size`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14", "transcript": "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/map-run/F14-files-03-upload-size-in-the-consumers-result-f14-.json", "at": "2026-10-13T02:48:27Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 45: test \"$(cat \"$F/sdk-size.txt\")\" = 70000" }, { "feature_id": "F15.agent-changes", "family": "F15", "result": "pass", "label": "map run F15-git.md: What the agent changed (`F15.agent-changes`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-23-what-the-agent-changed-f15-agent-changes.json", "at": "2026-10-13T02:19:09Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.auth", "family": "F15", "result": "pass", "label": "map run F15-git.md: Session key required (`F15.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-01-session-key-required-f15-auth.json", "at": "2026-10-13T02:17:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.changes", "family": "F15", "result": "pass", "label": "map run F15-git.md: Working-tree changes (`F15.changes`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-02-working-tree-changes-f15-changes.json", "at": "2026-10-13T02:17:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.changes-base", "family": "F15", "result": "pass", "label": "map run F15-git.md: Default display base (`F15.changes-base`).", "run": "/workspace/verification-artifacts/runs/20261013-031432-480c-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-031432-480c-map-f15/evidence/map-run/F15-git-01-default-display-base-f15-changes-base.json", "at": "2026-10-13T03:14:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.changes-nested-repo", "family": "F15", "result": "pass", "label": "map run F15-git.md: Nested repository (`F15.changes-nested-repo`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-06-nested-repository-f15-changes-nested-rep.json", "at": "2026-10-13T02:17:25Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.changes-ref", "family": "F15", "result": "pass", "label": "map run F15-git.md: Explicit base and its errors (`F15.changes-ref`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-05-explicit-base-and-its-errors-f15-changes.json", "at": "2026-10-13T02:17:25Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.changes-repo-below-path", "family": "F15", "result": "xfail", "label": "map run F15-git.md: Repository below the requested directory (`F15.changes-repo-below-path`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022337-0fbc-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-022337-0fbc-map-f15/evidence/map-run/F15-git-01-repository-below-the-requested-directory.json", "at": "2026-10-13T02:24:43Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 6: control-agent-server api GET /api/git/changes --query path=\"$F/ws\" --expect 200 --check . len-eq 2 --check 0.path eq proj/README.md --check 1.path eq proj/notes.txt --save F15.changes-repo-below-path/ws" }, { "feature_id": "F15.changes-special-names", "family": "F15", "result": "xfail", "label": "map run F15-git.md: File names with a space or non-ASCII characters (`F15.changes-special-names`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-031237-2ccf-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-031237-2ccf-map-f15/evidence/map-run/F15-git-02-file-names-with-a-space-or-non-ascii-cha.json", "at": "2026-10-13T03:12:51Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 19: test \"$RC_NAMES\" = 0" }, { "feature_id": "F15.changes-statuses", "family": "F15", "result": "pass", "label": "map run F15-git.md: Staged deletions, additions and renames (`F15.changes-statuses`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-03-staged-deletions-additions-and-renames-f.json", "at": "2026-10-13T02:17:22Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.changes-subdir-paths", "family": "F15", "result": "xfail", "label": "map run F15-git.md: Subdirectory of a repository (`F15.changes-subdir-paths`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022337-0fbc-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-022337-0fbc-map-f15/evidence/map-run/F15-git-03-subdirectory-of-a-repository-f15-changes.json", "at": "2026-10-13T02:24:57Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 8: python3 - \"$F/subdir.json\" \"$F/subdir/src\" \"$F/subdir\" <<'PY'\nimport json, os, sys\nchanges = json.load(open(sys.argv[1]))\npaths = [c[\"path\"] for c in changes if c[\"status\"] != \"DELETED\"]\nbases = [b for b in sys.argv[2:] if all(os.path.exists(os.path.join(b, p)) for p in paths)]\nassert paths and bases, f\"no single base resolves every path: {changes}\"\nPY" }, { "feature_id": "F15.commit-changes", "family": "F15", "result": "pass", "label": "map run F15-git.md: Files of one commit (`F15.commit-changes`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-17-files-of-one-commit-f15-commit-changes.json", "at": "2026-10-13T02:17:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.commits", "family": "F15", "result": "pass", "label": "map run F15-git.md: Commit history (`F15.commits`).", "run": "/workspace/verification-artifacts/runs/20261013-031432-480c-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-031432-480c-map-f15/evidence/map-run/F15-git-02-commit-history-f15-commits.json", "at": "2026-10-13T03:14:44Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.diff", "family": "F15", "result": "pass", "label": "map run F15-git.md: Per-file diff (`F15.diff`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-11-per-file-diff-f15-diff.json", "at": "2026-10-13T02:17:31Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.diff-commit", "family": "F15", "result": "pass", "label": "map run F15-git.md: Diff of one commit (`F15.diff-commit`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-12-diff-of-one-commit-f15-diff-commit.json", "at": "2026-10-13T02:17:32Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.diff-deleted-file", "family": "F15", "result": "xfail", "label": "map run F15-git.md: Diff of a deleted file (`F15.diff-deleted-file`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022337-0fbc-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-022337-0fbc-map-f15/evidence/map-run/F15-git-05-diff-of-a-deleted-file-f15-diff-deleted-.json", "at": "2026-10-13T02:25:11Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 5: control-agent-server api GET /api/git/diff --query path=\"$F/deleted/src/util.py\" --expect 200 --check original eq \"$(git -C \"$REPO\" show HEAD:src/util.py)\" --check modified eq '' --save F15.diff-deleted-file/diff" }, { "feature_id": "F15.diff-errors", "family": "F15", "result": "pass", "label": "map run F15-git.md: Diff errors (`F15.diff-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-13-diff-errors-f15-diff-errors.json", "at": "2026-10-13T02:17:34Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.diff-whitespace", "family": "F15", "result": "xfail", "label": "map run F15-git.md: Phantom whitespace diff (`F15.diff-whitespace`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-031237-2ccf-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-031237-2ccf-map-f15/evidence/map-run/F15-git-03-phantom-whitespace-diff-f15-diff-whitesp.json", "at": "2026-10-13T03:12:53Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 16: test \"$RC_WHITESPACE\" = 0" }, { "feature_id": "F15.non-repo", "family": "F15", "result": "pass", "label": "map run F15-git.md: Not a repository, or no commits yet (`F15.non-repo`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-10-not-a-repository-or-no-commits-yet-f15-n.json", "at": "2026-10-13T02:17:30Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.repo-search-errors", "family": "F15", "result": "pass", "label": "map run F15-git.md: Provider and parameter validation (`F15.repo-search-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-25-provider-and-parameter-validation-f15-re.json", "at": "2026-10-13T02:19:30Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.repo-search-github", "family": "F15", "result": "pass", "label": "Live GitHub listing, pagination, and search; 401 control limited by environment", "run": "/workspace/verification-artifacts/runs/20261013-041559-37e6-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-041559-37e6-map-f15/evidence/F15.repo-search-github/page-1.json", "at": "2026-10-13T04:23:00Z", "expected": "Real GitHub200listing, two-page cursor behavior and case-insensitive AGENT-SDK search match. Invalid-token401control evaluated separately.", "actual": "All positive assertions passed. The same fake token returns 200 through Agent Server and direct GitHub; stored_token_is_fake=true. The negative branch is inconclusive.", "note": "Reviewed positive-only verdict; the original full-map failure remains preserved. This pass does not claim the 401 negative branch works. No product bug counted." }, { "feature_id": "F15.repo-search-timeout", "family": "F15", "result": "pass", "label": "map run F15-git.md: GitHub timeout (`F15.repo-search-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-031237-2ccf-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-031237-2ccf-map-f15/evidence/map-run/F15-git-06-github-timeout-f15-repo-search-timeout.json", "at": "2026-10-13T03:14:55Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.repo-search-token", "family": "F15", "result": "pass", "label": "map run F15-git.md: Token lookup (`F15.repo-search-token`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-24-token-lookup-f15-repo-search-token.json", "at": "2026-10-13T02:19:20Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.repo-search-unreachable", "family": "F15", "result": "pass", "label": "map run F15-git.md: GitHub unreachable (`F15.repo-search-unreachable`).", "run": "/workspace/verification-artifacts/runs/20261013-031237-2ccf-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-031237-2ccf-map-f15/evidence/map-run/F15-git-05-github-unreachable-f15-repo-search-unrea.json", "at": "2026-10-13T03:13:44Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.scoped", "family": "F15", "result": "pass", "label": "map run F15-git.md: Conversation-scoped views (`F15.scoped`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-18-conversation-scoped-views-f15-scoped.json", "at": "2026-10-13T02:17:41Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.scoped-guard", "family": "F15", "result": "pass", "label": "map run F15-git.md: Workspace guard (`F15.scoped-guard`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-19-workspace-guard-f15-scoped-guard.json", "at": "2026-10-13T02:17:49Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.scoped-restart", "family": "F15", "result": "pass", "label": "map run F15-git.md: After a restart (`F15.scoped-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-20-after-a-restart-f15-scoped-restart.json", "at": "2026-10-13T02:18:28Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.sdk-workspace", "family": "F15", "result": "pass", "label": "map run F15-git.md: Python SDK workspace (`F15.sdk-workspace`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-21-python-sdk-workspace-f15-sdk-workspace.json", "at": "2026-10-13T02:18:49Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F15.ts-client", "family": "F15", "result": "pass", "label": "map run F15-git.md: TypeScript client (`F15.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022337-0fbc-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-022337-0fbc-map-f15/evidence/map-run/F15-git-06-typescript-client-f15-ts-client.json", "at": "2026-10-13T02:25:32Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.agent-artifact", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: The agent's own artifact (`F16.agent-artifact`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-14-the-agent-s-own-artifact-f16-agent-artif.json", "at": "2026-10-13T02:22:59Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.auth", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: Header or cookie (`F16.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-13-header-or-cookie-f16-auth.json", "at": "2026-10-13T02:22:38Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.content-types", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: Content types (`F16.content-types`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-03-content-types-f16-content-types.json", "at": "2026-10-13T02:20:44Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.deleted-conversation", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: Deleted conversation (`F16.deleted-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-11-deleted-conversation-f16-deleted-convers.json", "at": "2026-10-13T02:22:02Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.dir-index", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: Directory index and missing files (`F16.dir-index`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-04-directory-index-and-missing-files-f16-di.json", "at": "2026-10-13T02:21:00Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.docker-root-cookie", "family": "F16", "result": "xfail", "label": "map run F16-workspace-serving-and-vscode.md: Docker-mode root and the cookie (`F16.docker-root-cookie`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030422-3050-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-030422-3050-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-07-docker-mode-root-and-the-cookie-f16-dock.json", "at": "2026-10-13T03:07:17Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 9: control-agent-server api GET \"/$Z/workspace\" --run \"$DK\" --auth none --jar qa-f16-docker --expect 404 --save F16.docker-root-cookie/slashless" }, { "feature_id": "F16.file-bytes", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: File bytes and headers (`F16.file-bytes`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-02-file-bytes-and-headers-f16-file-bytes.json", "at": "2026-10-13T02:20:26Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.live-content", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: Live content (`F16.live-content`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-06-live-content-f16-live-content.json", "at": "2026-10-13T02:21:07Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.range", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: Byte ranges (`F16.range`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-05-byte-ranges-f16-range.json", "at": "2026-10-13T02:21:03Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.restart", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: Across a restart (`F16.restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-15-across-a-restart-f16-restart.json", "at": "2026-10-13T02:24:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.root-index", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: Root index (`F16.root-index`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-01-root-index-f16-root-index.json", "at": "2026-10-13T02:20:19Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.symlink-escape", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: Symlinks (`F16.symlink-escape`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-08-symlinks-f16-symlink-escape.json", "at": "2026-10-13T02:21:26Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.traversal-rejected", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: Traversal (`F16.traversal-rejected`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-07-traversal-f16-traversal-rejected.json", "at": "2026-10-13T02:21:20Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.unknown-conversation", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: Unknown and malformed ids (`F16.unknown-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-09-unknown-and-malformed-ids-f16-unknown-co.json", "at": "2026-10-13T02:21:40Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.vscode-auth", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: VS Code routes are header-only (`F16.vscode-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-18-vs-code-routes-are-header-only-f16-vscod.json", "at": "2026-10-13T02:25:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.vscode-disabled", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: VS Code disabled (`F16.vscode-disabled`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-16-vs-code-disabled-f16-vscode-disabled.json", "at": "2026-10-13T02:24:22Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.vscode-enabled-status", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: Enabled but not running (`F16.vscode-enabled-status`).", "run": "/workspace/verification-artifacts/runs/20261013-030422-3050-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-030422-3050-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-01-enabled-but-not-running-f16-vscode-enabl.json", "at": "2026-10-13T03:05:15Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.vscode-folder-encoding", "family": "F16", "result": "xfail", "label": "map run F16-workspace-serving-and-vscode.md: Folder with `&` and `#` (`F16.vscode-folder-encoding`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030422-3050-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-030422-3050-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-04-folder-with-and-f16-vscode-folder-encodi.json", "at": "2026-10-13T03:05:52Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 7: python3 -c \"$ROUNDTRIP\" \"$OWS\"" }, { "feature_id": "F16.vscode-running", "family": "F16", "result": "blocked", "label": "map run F16-workspace-serving-and-vscode.md: VS Code actually running (`F16.vscode-running`), blocked.", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": null, "at": "2026-10-13T02:29:18Z", "expected": "prerequisite available", "actual": "blocked (see the bullet)", "note": "bullet is marked blocked in the map" }, { "feature_id": "F16.vscode-scoped-guards", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: Scoped guards (`F16.vscode-scoped-guards`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-17-scoped-guards-f16-vscode-scoped-guards.json", "at": "2026-10-13T02:24:47Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.vscode-url-after-init", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: Key delivered by init, no URL (`F16.vscode-url-after-init`).", "run": "/workspace/verification-artifacts/runs/20261013-030422-3050-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-030422-3050-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-06-key-delivered-by-init-no-url-f16-vscode-.json", "at": "2026-10-13T03:06:54Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.vscode-url-no-key", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: No key, no URL (`F16.vscode-url-no-key`).", "run": "/workspace/verification-artifacts/runs/20261013-030422-3050-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-030422-3050-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-05-no-key-no-url-f16-vscode-url-no-key.json", "at": "2026-10-13T03:06:23Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.vscode-url-not-running", "family": "F16", "result": "xfail", "label": "map run F16-workspace-serving-and-vscode.md: Not running, still advertised (`F16.vscode-url-not-running`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030422-3050-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-030422-3050-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-03-not-running-still-advertised-f16-vscode-.json", "at": "2026-10-13T03:05:49Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 2: control-agent-server api GET /api/vscode/url --expect 200 --check . eq '{\"url\": null}' --save F16.vscode-url-not-running/global" }, { "feature_id": "F16.vscode-url-shape", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: URL shape (`F16.vscode-url-shape`).", "run": "/workspace/verification-artifacts/runs/20261013-030422-3050-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-030422-3050-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-02-url-shape-f16-vscode-url-shape.json", "at": "2026-10-13T03:05:48Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.workspace-dir-missing", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: Workspace directory gone (`F16.workspace-dir-missing`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-10-workspace-directory-gone-f16-workspace-d.json", "at": "2026-10-13T02:21:54Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F16.worktree-served", "family": "F16", "result": "pass", "label": "map run F16-workspace-serving-and-vscode.md: Worktree conversation (`F16.worktree-served`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-12-worktree-conversation-f16-worktree-serve.json", "at": "2026-10-13T02:22:20Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.add-parents", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Register parents (`F17.add-parents`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-11-register-parents-f17-add-parents.json", "at": "2026-10-13T02:23:56Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.add-workspaces", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Register a repository and a plain directory (`F17.add-workspaces`).", "run": "/workspace/verification-artifacts/runs/20261013-030107-4090-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-030107-4090-map-f17/evidence/map-run/F17-workspaces-registry-02-register-a-repository-and-a-plain-direct.json", "at": "2026-10-13T03:01:26Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.auth-required", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Session key required (`F17.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-16-session-key-required-f17-auth-required.json", "at": "2026-10-13T02:26:52Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.concurrent-adds", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Concurrent writes (`F17.concurrent-adds`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-17-concurrent-writes-f17-concurrent-adds.json", "at": "2026-10-13T02:27:14Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.corrupted-file", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Corrupted file (`F17.corrupted-file`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-20-corrupted-file-f17-corrupted-file.json", "at": "2026-10-13T02:29:38Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.delete-not-found", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Unknown paths (`F17.delete-not-found`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-07-unknown-paths-f17-delete-not-found.json", "at": "2026-10-13T02:22:46Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.delete-parent", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Delete a parent (`F17.delete-parent`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-13-delete-a-parent-f17-delete-parent.json", "at": "2026-10-13T02:24:20Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.delete-workspace", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Delete a workspace (`F17.delete-workspace`).", "run": "/workspace/verification-artifacts/runs/20261013-030107-4090-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-030107-4090-map-f17/evidence/map-run/F17-workspaces-registry-05-delete-a-workspace-f17-delete-workspace.json", "at": "2026-10-13T03:01:34Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.future-schema", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Future or invalid schema (`F17.future-schema`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-21-future-or-invalid-schema-f17-future-sche.json", "at": "2026-10-13T02:30:45Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.idempotent-add", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Idempotent re-add (`F17.idempotent-add`).", "run": "/workspace/verification-artifacts/runs/20261013-030107-4090-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-030107-4090-map-f17/evidence/map-run/F17-workspaces-registry-04-idempotent-re-add-f17-idempotent-add.json", "at": "2026-10-13T03:01:32Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.io-error", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Unreadable file (`F17.io-error`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-22-unreadable-file-f17-io-error.json", "at": "2026-10-13T02:31:15Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.list-empty", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Empty registry (`F17.list-empty`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-01-empty-registry-f17-list-empty.json", "at": "2026-10-13T02:21:52Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.lists-independent", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Two independent lists (`F17.lists-independent`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-14-two-independent-lists-f17-lists-independ.json", "at": "2026-10-13T02:24:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.noop-delete-no-file", "family": "F17", "result": "xfail", "label": "map run F17-workspaces-registry.md: A refused delete on a fresh directory (`F17.noop-delete-no-file`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030107-4090-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-030107-4090-map-f17/evidence/map-run/F17-workspaces-registry-01-a-refused-delete-on-a-fresh-directory-f1.json", "at": "2026-10-13T03:01:23Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 7: control-agent-server state ls home/.openhands/workspaces.json --expect-count 0" }, { "feature_id": "F17.noop-write-readonly", "family": "F17", "result": "fail", "label": "map run F17-workspaces-registry.md: No-op writes on read-only storage (`F17.noop-write-readonly`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-23-no-op-writes-on-read-only-storage-f17-no.json", "at": "2026-10-13T02:31:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 127", "note": "automated map run; prose-only expectations were not re-judged; failed at line 3: chattr +i \"$PDIR\"" }, { "feature_id": "F17.noop-write-untouched", "family": "F17", "result": "xfail", "label": "map run F17-workspaces-registry.md: No-op writes leave the file alone (`F17.noop-write-untouched`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030107-4090-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-030107-4090-map-f17/evidence/map-run/F17-workspaces-registry-06-no-op-writes-leave-the-file-alone-f17-no.json", "at": "2026-10-13T03:01:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 6: test \"$(stat -c '%i %y' \"$WS\")\" = \"$STAMP\"" }, { "feature_id": "F17.parent-children", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Children of a parent (`F17.parent-children`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-12-children-of-a-parent-f17-parent-children.json", "at": "2026-10-13T02:24:07Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.paths-opaque", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Paths are opaque strings (`F17.paths-opaque`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-09-paths-are-opaque-strings-f17-paths-opaqu.json", "at": "2026-10-13T02:23:12Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.persist-restart", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Persistence across a restart (`F17.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-15-persistence-across-a-restart-f17-persist.json", "at": "2026-10-13T02:26:11Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.persistence-location", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Location without `OH_PERSISTENCE_DIR` (`F17.persistence-location`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-24-location-without-oh-persistence-dir-f17-.json", "at": "2026-10-13T02:35:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.shared-dir", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Two servers, one file (`F17.shared-dir`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-18-two-servers-one-file-f17-shared-dir.json", "at": "2026-10-13T02:29:03Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.ts-client", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: TypeScript client (`F17.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-19-typescript-client-f17-ts-client.json", "at": "2026-10-13T02:29:12Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.validation", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Validation (`F17.validation`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-10-validation-f17-validation.json", "at": "2026-10-13T02:23:47Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F17.wire-format", "family": "F17", "result": "pass", "label": "map run F17-workspaces-registry.md: Wire format (`F17.wire-format`).", "run": "/workspace/verification-artifacts/runs/20261013-030107-4090-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-030107-4090-map-f17/evidence/map-run/F17-workspaces-registry-03-wire-format-f17-wire-format.json", "at": "2026-10-13T03:01:28Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.active-profile-pointer", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Active LLM profile pointer (`F18.active-profile-pointer`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-12-active-llm-profile-pointer-f18-active-pr.json", "at": "2026-10-13T02:19:25Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.agent-diff-merge", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Sparse agent diff and unset (`F18.agent-diff-merge`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-08-sparse-agent-diff-and-unset-f18-agent-di.json", "at": "2026-10-13T02:18:56Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.agent-kind-switch", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Agent kind switch (`F18.agent-kind-switch`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-13-agent-kind-switch-f18-agent-kind-switch.json", "at": "2026-10-13T02:19:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.agent-schema", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Agent settings schema (`F18.agent-schema`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-02-agent-settings-schema-f18-agent-schema.json", "at": "2026-10-13T02:18:22Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.auth-required", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Key required (`F18.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-04-key-required-f18-auth-required.json", "at": "2026-10-13T02:18:33Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.conversation-diff", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Conversation settings (`F18.conversation-diff`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-09-conversation-settings-f18-conversation-d.json", "at": "2026-10-13T02:19:00Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.conversation-schema", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Conversation settings schema (`F18.conversation-schema`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-03-conversation-settings-schema-f18-convers.json", "at": "2026-10-13T02:18:23Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.corrupted-files", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Corrupted files (`F18.corrupted-files`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-23-corrupted-files-f18-corrupted-files.json", "at": "2026-10-13T02:24:41Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.dangling-pointers", "family": "F18", "result": "xfail", "label": "map run F18-settings-and-secrets.md: Dangling meta-profile pointer (`F18.dangling-pointers`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025117-43f0-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-025117-43f0-map-f18/evidence/map-run/F18-settings-and-secrets-03-dangling-meta-profile-pointer-f18-dangli.json", "at": "2026-10-13T02:51:45Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 8: control-agent-server api PATCH /api/settings --json '{\"active_meta_profile\": \"qa-f18-no-router\"}' --expect 404,422 --save F18.dangling-pointers/meta-profile" }, { "feature_id": "F18.encrypted-at-rest", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Encrypted at rest (`F18.encrypted-at-rest`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-17-encrypted-at-rest-f18-encrypted-at-rest.json", "at": "2026-10-13T02:20:34Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.expose-modes", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Exposure modes (`F18.expose-modes`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-06-exposure-modes-f18-expose-modes.json", "at": "2026-10-13T02:18:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.get-defaults", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Fresh defaults (`F18.get-defaults`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-01-fresh-defaults-f18-get-defaults.json", "at": "2026-10-13T02:18:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.misc-settings-merge", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Opaque misc settings (`F18.misc-settings-merge`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-10-opaque-misc-settings-f18-misc-settings-m.json", "at": "2026-10-13T02:19:04Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.no-cipher", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: No cipher (`F18.no-cipher`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-24-no-cipher-f18-no-cipher.json", "at": "2026-10-13T02:26:34Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.patch-llm", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Set the LLM (`F18.patch-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-05-set-the-llm-f18-patch-llm.json", "at": "2026-10-13T02:18:34Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.patch-validation", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Rejected PATCH bodies (`F18.patch-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-11-rejected-patch-bodies-f18-patch-validati.json", "at": "2026-10-13T02:19:11Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.persist-restart", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Persistence across a restart (`F18.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-22-persistence-across-a-restart-f18-persist.json", "at": "2026-10-13T02:24:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.roundtrip", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Post back what GET returned (`F18.roundtrip`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-07-post-back-what-get-returned-f18-roundtri.json", "at": "2026-10-13T02:18:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.sdk-clone-repos", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: SDK clones with the stored token (`F18.sdk-clone-repos`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-20-sdk-clones-with-the-stored-token-f18-sdk.json", "at": "2026-10-13T02:22:40Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.sdk-remote-workspace", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: SDK consumer (`F18.sdk-remote-workspace`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-19-sdk-consumer-f18-sdk-remote-workspace.json", "at": "2026-10-13T02:21:53Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.secret-delete", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Delete a secret (`F18.secret-delete`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-15-delete-a-secret-f18-secret-delete.json", "at": "2026-10-13T02:19:59Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.secret-empty-value", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Empty secret value (`F18.secret-empty-value`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-26-empty-secret-value-f18-secret-empty-valu.json", "at": "2026-10-13T02:27:17Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.secret-key-rotation", "family": "F18", "result": "xfail", "label": "map run F18-settings-and-secrets.md: Cipher key rotation (`F18.secret-key-rotation`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025117-43f0-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-025117-43f0-map-f18/evidence/map-run/F18-settings-and-secrets-07-cipher-key-rotation-f18-secret-key-rotat.json", "at": "2026-10-13T02:55:35Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 21: control-agent-server state cat home/.openhands/secrets.json --check custom_secrets.QA_F18_TOKEN.secret matches '^gAAAAA'" }, { "feature_id": "F18.secret-name-contract", "family": "F18", "result": "xfail", "label": "map run F18-settings-and-secrets.md: Documented status for an invalid name (`F18.secret-name-contract`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025117-43f0-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-025117-43f0-map-f18/evidence/map-run/F18-settings-and-secrets-06-documented-status-for-an-invalid-name-f1.json", "at": "2026-10-13T02:53:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 3: control-agent-server api GET /api/settings/secrets/1bad --expect \"${CLAIMED:-422}\" --save F18.secret-name-contract/invalid-name" }, { "feature_id": "F18.secret-name-validation", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Secret names (`F18.secret-name-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-16-secret-names-f18-secret-name-validation.json", "at": "2026-10-13T02:20:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.secret-put-list-get", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Upsert, list and read a secret (`F18.secret-put-list-get`).", "run": "/workspace/verification-artifacts/runs/20261013-025117-43f0-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-025117-43f0-map-f18/evidence/map-run/F18-settings-and-secrets-01-upsert-list-and-read-a-secret-f18-secret.json", "at": "2026-10-13T02:51:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.secret-scope-agent-profile", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: Secrets scoped by an agent profile (`F18.secret-scope-agent-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-18-secrets-scoped-by-an-agent-profile-f18-s.json", "at": "2026-10-13T02:21:15Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F18.secret-scope-profile-errors", "family": "F18", "result": "xfail", "label": "map run F18-settings-and-secrets.md: Agent-profile store busy (`F18.secret-scope-profile-errors`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025117-43f0-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-025117-43f0-map-f18/evidence/map-run/F18-settings-and-secrets-05-agent-profile-store-busy-f18-secret-scop.json", "at": "2026-10-13T02:53:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 11: control-agent-server api GET /api/settings/secrets --query agent_profile_id=00000000-0000-4000-8000-0000000000f1 --timeout 60 --expect 503 --save F18.secret-scope-profile-errors/secrets-busy" }, { "feature_id": "F18.ts-client", "family": "F18", "result": "pass", "label": "map run F18-settings-and-secrets.md: TypeScript client (`F18.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-21-typescript-client-f18-ts-client.json", "at": "2026-10-13T02:23:06Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.auth", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Auth (`F19.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-01-auth-f19-auth.json", "at": "2026-10-13T02:21:02Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.conversation-secrets-at-rest", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Credentials at rest (`F19.conversation-secrets-at-rest`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-27-credentials-at-rest-f19-conversation-sec.json", "at": "2026-10-13T02:28:58Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.conversation-tool", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: The agent uses the MCP tool (`F19.conversation-tool`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-25-the-agent-uses-the-mcp-tool-f19-conversa.json", "at": "2026-10-13T02:27:47Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.disabled-server", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Disabled servers stay off (`F19.disabled-server`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-24-disabled-servers-stay-off-f19-disabled-s.json", "at": "2026-10-13T02:27:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.oauth-callback-bad-port", "family": "F19", "result": "xfail", "label": "map run F19-mcp.md: Port out of range (`F19.oauth-callback-bad-port`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030348-f557-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-030348-f557-map-f19/evidence/map-run/F19-mcp-05-port-out-of-range-f19-oauth-callback-bad.json", "at": "2026-10-13T03:04:40Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 6: control-agent-server api POST \"/api/mcp/oauth/callback/$JB\" --json '{\"callback_url\":\"http://localhost:99999/callback?code=x\"}' --expect 400 --save F19.oauth-callback-bad-port/port-99999" }, { "feature_id": "F19.oauth-callback-replay", "family": "F19", "result": "xfail", "label": "map run F19-mcp.md: Replayed callback (`F19.oauth-callback-replay`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030348-f557-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-030348-f557-map-f19/evidence/map-run/F19-mcp-04-replayed-callback-f19-oauth-callback-rep.json", "at": "2026-10-13T03:04:29Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 3: control-agent-server api POST \"/api/mcp/oauth/callback/$J\" --json \"$RBODY\" --expect 200,4xx --save F19.oauth-callback-replay/replay" }, { "feature_id": "F19.oauth-callback-validation", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Callback URL checks on a live job (`F19.oauth-callback-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-19-callback-url-checks-on-a-live-job-f19-oa.json", "at": "2026-10-13T02:25:33Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.oauth-consent-denied", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Denied consent (`F19.oauth-consent-denied`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-20-denied-consent-f19-oauth-consent-denied.json", "at": "2026-10-13T02:25:49Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.oauth-consent-timeout", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Consent timeout (`F19.oauth-consent-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-21-consent-timeout-f19-oauth-consent-timeou.json", "at": "2026-10-13T02:25:59Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.oauth-flow", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Full OAuth consent (`F19.oauth-flow`).", "run": "/workspace/verification-artifacts/runs/20261013-030348-f557-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-030348-f557-map-f19/evidence/map-run/F19-mcp-03-full-oauth-consent-f19-oauth-flow.json", "at": "2026-10-13T03:04:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.oauth-inline-writeback", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Inline OAuth server refresh (`F19.oauth-inline-writeback`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-22-inline-oauth-server-refresh-f19-oauth-in.json", "at": "2026-10-13T02:26:23Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.oauth-refresh-writeback", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Stored OAuth token refresh (`F19.oauth-refresh-writeback`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-26-stored-oauth-token-refresh-f19-oauth-ref.json", "at": "2026-10-13T02:28:19Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.oauth-start-errors", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: OAuth start refusals and early outcomes (`F19.oauth-start-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-14-oauth-start-refusals-and-early-outcomes-.json", "at": "2026-10-13T02:24:06Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.oauth-stored-state", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Stored OAuth state (`F19.oauth-stored-state`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-23-stored-oauth-state-f19-oauth-stored-stat.json", "at": "2026-10-13T02:26:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.oauth-unknown-job", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Unknown jobs (`F19.oauth-unknown-job`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-15-unknown-jobs-f19-oauth-unknown-job.json", "at": "2026-10-13T02:24:15Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.restart", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Restart (`F19.restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-28-restart-f19-restart.json", "at": "2026-10-13T02:32:41Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.settings-create", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Create one server (`F19.settings-create`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-02-create-one-server-f19-settings-create.json", "at": "2026-10-13T02:21:15Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.settings-delete", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Delete one server (`F19.settings-delete`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-06-delete-one-server-f19-settings-delete.json", "at": "2026-10-13T02:22:12Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.settings-empty-server", "family": "F19", "result": "xfail", "label": "map run F19-mcp.md: Server with neither command nor url (`F19.settings-empty-server`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030348-f557-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-030348-f557-map-f19/evidence/map-run/F19-mcp-01-server-with-neither-command-nor-url-f19-.json", "at": "2026-10-13T03:04:10Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 2: false" }, { "feature_id": "F19.settings-patch", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Sparse patch (`F19.settings-patch`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-05-sparse-patch-f19-settings-patch.json", "at": "2026-10-13T02:21:59Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.settings-validation", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Invalid servers (`F19.settings-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-03-invalid-servers-f19-settings-validation.json", "at": "2026-10-13T02:21:28Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.test-connect-failures", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Connection failures (`F19.test-connect-failures`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-10-connection-failures-f19-test-connect-fai.json", "at": "2026-10-13T02:23:12Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.test-encrypted-env", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Encrypted env from settings (`F19.test-encrypted-env`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-12-encrypted-env-from-settings-f19-test-enc.json", "at": "2026-10-13T02:23:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.test-remote-bearer", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Remote server behind a bearer token (`F19.test-remote-bearer`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-13-remote-server-behind-a-bearer-token-f19-.json", "at": "2026-10-13T02:23:47Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.test-stdio", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Test a stdio server (`F19.test-stdio`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-08-test-a-stdio-server-f19-test-stdio.json", "at": "2026-10-13T02:22:43Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.test-timeout", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Handshake timeout (`F19.test-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-11-handshake-timeout-f19-test-timeout.json", "at": "2026-10-13T02:23:19Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.test-tool-errors", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Tool failures stay in `tool_result` (`F19.test-tool-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-030348-f557-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-030348-f557-map-f19/evidence/map-run/F19-mcp-02-tool-failures-stay-in-tool-result-f19-te.json", "at": "2026-10-13T03:04:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F19.test-validation", "family": "F19", "result": "pass", "label": "map run F19-mcp.md: Connection test body validation (`F19.test-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-07-connection-test-body-validation-f19-test.json", "at": "2026-10-13T02:22:29Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.activate", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Activate (`F20.activate`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-20-activate-f20-activate.json", "at": "2026-10-13T02:53:25Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.activate-acp", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: ACP settings (`F20.activate-acp`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-22-acp-settings-f20-activate-acp.json", "at": "2026-10-13T02:53:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.activate-conversation", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Activation drives conversations (`F20.activate-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-21-activation-drives-conversations-f20-acti.json", "at": "2026-10-13T02:53:32Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.activate-dangling-provider", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Dangling provider link (`F20.activate-dangling-provider`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-25-dangling-provider-link-f20-activate-dang.json", "at": "2026-10-13T02:54:12Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.activate-linked-provider", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Linked provider (`F20.activate-linked-provider`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-26-linked-provider-f20-activate-linked-prov.json", "at": "2026-10-13T02:54:22Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.auth-required", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Key required (`F20.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-01-key-required-f20-auth-required.json", "at": "2026-10-13T02:46:19Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.corrupt-files", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Corrupted files (`F20.corrupt-files`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-09-corrupted-files-f20-corrupt-files.json", "at": "2026-10-13T02:46:51Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.delete", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Delete (`F20.delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-33-delete-f20-delete.json", "at": "2026-10-13T02:56:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.delete-guarded", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Guarded delete (`F20.delete-guarded`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-32-guarded-delete-f20-delete-guarded.json", "at": "2026-10-13T02:56:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.get-secrets", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Reading the key (`F20.get-secrets`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-04-reading-the-key-f20-get-secrets.json", "at": "2026-10-13T02:46:29Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.json-alias-guard", "family": "F20", "result": "xfail", "label": "map run F20-llm-profiles.md: Delete through the `.json` alias (`F20.json-alias-guard`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030955-5180-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-030955-5180-map-f20/evidence/map-run/F20-llm-profiles-04-delete-through-the-json-alias-f20-json-a.json", "at": "2026-10-13T03:12:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 11: jq -e '.status == 409' \"$F20/alias-delete.json\"" }, { "feature_id": "F20.key-encrypted-at-rest", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Encrypted at rest (`F20.key-encrypted-at-rest`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-03-encrypted-at-rest-f20-key-encrypted-at-r.json", "at": "2026-10-13T02:46:24Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.name-rules", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Name rules (`F20.name-rules`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-08-name-rules-f20-name-rules.json", "at": "2026-10-13T02:46:46Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.no-cipher", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: No cipher (`F20.no-cipher`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-28-no-cipher-f20-no-cipher.json", "at": "2026-10-13T02:55:30Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.not-found", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Unknown names (`F20.not-found`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-07-unknown-names-f20-not-found.json", "at": "2026-10-13T02:46:40Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.persist-restart", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Restart (`F20.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-29-restart-f20-persist-restart.json", "at": "2026-10-13T02:56:22Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.profile-limit", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Profile limit (`F20.profile-limit`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-12-profile-limit-f20-profile-limit.json", "at": "2026-10-13T02:49:46Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.rename", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Rename (`F20.rename`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-30-rename-f20-rename.json", "at": "2026-10-13T02:56:28Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.rename-errors", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Rename errors (`F20.rename-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-31-rename-errors-f20-rename-errors.json", "at": "2026-10-13T02:56:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.save", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Save (`F20.save`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-02-save-f20-save.json", "at": "2026-10-13T02:46:23Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.save-encrypted-roundtrip", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Encrypted round trip (`F20.save-encrypted-roundtrip`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-05-encrypted-round-trip-f20-save-encrypted-.json", "at": "2026-10-13T02:46:34Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.save-without-secrets", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Save without secrets (`F20.save-without-secrets`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-06-save-without-secrets-f20-save-without-se.json", "at": "2026-10-13T02:46:37Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.sdk-get-llm", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: SDK consumer (`F20.sdk-get-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-23-sdk-consumer-f20-sdk-get-llm.json", "at": "2026-10-13T02:54:01Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.sdk-get-llm-linked", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: SDK on a linked profile (`F20.sdk-get-llm-linked`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-27-sdk-on-a-linked-profile-f20-sdk-get-llm-.json", "at": "2026-10-13T02:54:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.secret-key-rotation", "family": "F20", "result": "xfail", "label": "map run F20-llm-profiles.md: Secret key rotation (`F20.secret-key-rotation`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030955-5180-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-030955-5180-map-f20/evidence/map-run/F20-llm-profiles-05-secret-key-rotation-f20-secret-key-rotat.json", "at": "2026-10-13T03:13:15Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 8: control-agent-server api POST /api/profiles/qa-rotate/activate --expect 4xx --save F20.secret-key-rotation/activate" }, { "feature_id": "F20.store-busy", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Store busy (`F20.store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-10-store-busy-f20-store-busy.json", "at": "2026-10-13T02:47:27Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.store-busy-responsive", "family": "F20", "result": "xfail", "label": "map run F20-llm-profiles.md: Lock wait blocks the server (`F20.store-busy-responsive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-11-lock-wait-blocks-the-server-f20-store-bu.json", "at": "2026-10-13T02:48:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 16: jq -e '.elapsed_ms <= 3000' \"$F20/busy-alive.json\"" }, { "feature_id": "F20.ts-client", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: TypeScript client (`F20.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-24-typescript-client-f20-ts-client.json", "at": "2026-10-13T02:54:03Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.validate-bad-key-type", "family": "F20", "result": "xfail", "label": "map run F20-llm-profiles.md: DeepSeek 401 type (`F20.validate-bad-key-type`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030955-5180-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-030955-5180-map-f20/evidence/map-run/F20-llm-profiles-01-deepseek-401-type-f20-validate-bad-key-t.json", "at": "2026-10-13T03:10:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 5: jq -e '.response.body.error.type == \"LLMAuthenticationError\"' \"$F20/bad-key.json\"" }, { "feature_id": "F20.validate-hang-bounded", "family": "F20", "result": "xfail", "label": "map run F20-llm-profiles.md: Hanging provider is slower (`F20.validate-hang-bounded`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030955-5180-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-030955-5180-map-f20/evidence/map-run/F20-llm-profiles-03-hanging-provider-is-slower-f20-validate-.json", "at": "2026-10-13T03:12:47Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 11: test \"$RC\" = 0" }, { "feature_id": "F20.validate-live", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Validate a working config (`F20.validate-live`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-13-validate-a-working-config-f20-validate-l.json", "at": "2026-10-13T02:49:53Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.validate-provider-errors", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Provider failures (`F20.validate-provider-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-16-provider-failures-f20-validate-provider-.json", "at": "2026-10-13T02:50:14Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.validate-rejects", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Validate rejects (`F20.validate-rejects`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-14-validate-rejects-f20-validate-rejects.json", "at": "2026-10-13T02:49:56Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.validate-subscription-login", "family": "F20", "result": "pass", "label": "map run F20-llm-profiles.md: Subscription without a login (`F20.validate-subscription-login`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-17-subscription-without-a-login-f20-validat.json", "at": "2026-10-13T02:50:17Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F20.validate-unreachable-bounded", "family": "F20", "result": "xfail", "label": "map run F20-llm-profiles.md: Unreachable endpoint is slow (`F20.validate-unreachable-bounded`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030955-5180-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-030955-5180-map-f20/evidence/map-run/F20-llm-profiles-02-unreachable-endpoint-is-slow-f20-validat.json", "at": "2026-10-13T03:12:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 5: jq -e '.elapsed_ms <= 30000' \"$F20/unreachable.json\"" }, { "feature_id": "F21.acp-kind", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: ACP profile (`F21.acp-kind`).", "run": "/workspace/verification-artifacts/runs/20261013-030902-92d4-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-030902-92d4-map-f21/evidence/map-run/F21-agent-profiles-02-acp-profile-f21-acp-kind.json", "at": "2026-10-13T03:09:14Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.acp-skill-sourcing", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: ACP skill sourcing (`F21.acp-skill-sourcing`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-26-acp-skill-sourcing-f21-acp-skill-sourcin.json", "at": "2026-10-13T02:53:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.activate", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Activate by id (`F21.activate`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-11-activate-by-id-f21-activate.json", "at": "2026-10-13T02:48:34Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.activate-corrupt-settings", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Activate over a corrupt settings file (`F21.activate-corrupt-settings`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-12-activate-over-a-corrupt-settings-file-f2.json", "at": "2026-10-13T02:48:41Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.auth-required", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: No key, no access (`F21.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-01-no-key-no-access-f21-auth-required.json", "at": "2026-10-13T02:46:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.busy-blocks-server", "family": "F21", "result": "xfail", "label": "map run F21-agent-profiles.md: Lock wait stalls the server (`F21.busy-blocks-server`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030902-92d4-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-030902-92d4-map-f21/evidence/map-run/F21-agent-profiles-03-lock-wait-stalls-the-server-f21-busy-blo.json", "at": "2026-10-13T03:09:44Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 7: control-agent-server api GET /alive --auth none --expect 200 --timeout 60 --expect-max-ms 2000 --save F21.busy-blocks-server/alive" }, { "feature_id": "F21.conversation-start", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Start a conversation from a profile (`F21.conversation-start`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-21-start-a-conversation-from-a-profile-f21-.json", "at": "2026-10-13T02:51:19Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.corrupt-file", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Corrupt file (`F21.corrupt-file`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-20-corrupt-file-f21-corrupt-file.json", "at": "2026-10-13T02:50:57Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.dangling-llm-report", "family": "F21", "result": "xfail", "label": "map run F21-agent-profiles.md: How a dangling LLM reference is reported (`F21.dangling-llm-report`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-30-how-a-dangling-llm-reference-is-reported.json", "at": "2026-10-13T02:55:31Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 10: jq -e '.dangling_llm_profile_ref == \"qa-f21-missing\"' \"$RAW\" > /dev/null" }, { "feature_id": "F21.delete", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Delete (`F21.delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-14-delete-f21-delete.json", "at": "2026-10-13T02:49:02Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.delete-active-acp-reset", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Active ACP reset (`F21.delete-active-acp-reset`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-15-active-acp-reset-f21-delete-active-acp-r.json", "at": "2026-10-13T02:49:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.json-suffix-identity", "family": "F21", "result": "xfail", "label": "map run F21-agent-profiles.md: `.json` names lose their identity (`F21.json-suffix-identity`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030902-92d4-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-030902-92d4-map-f21/evidence/map-run/F21-agent-profiles-05-json-names-lose-their-identity-f21-json-.json", "at": "2026-10-13T03:09:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 8: control-agent-server api GET /api/agent-profiles/qa-f21-suffix.json --check profile.id eq \"$J1\" --check profile.revision eq 1" }, { "feature_id": "F21.launch-additions", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Launch additions (`F21.launch-additions`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-23-launch-additions-f21-launch-additions.json", "at": "2026-10-13T02:51:41Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.launch-store-busy", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Busy store at conversation start (`F21.launch-store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-28-busy-store-at-conversation-start-f21-lau.json", "at": "2026-10-13T02:54:56Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.lazy-seed", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Lazy seed (`F21.lazy-seed`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-02-lazy-seed-f21-lazy-seed.json", "at": "2026-10-13T02:46:22Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.limit", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: At most 50 (`F21.limit`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-19-at-most-50-f21-limit.json", "at": "2026-10-13T02:50:54Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.materialize-dangling", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Dangling references (`F21.materialize-dangling`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-17-dangling-references-f21-materialize-dang.json", "at": "2026-10-13T02:49:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.materialize-draft", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Draft preview (`F21.materialize-draft`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-18-draft-preview-f21-materialize-draft.json", "at": "2026-10-13T02:49:29Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.materialize-valid", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Materialize a resolvable profile (`F21.materialize-valid`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-16-materialize-a-resolvable-profile-f21-mat.json", "at": "2026-10-13T02:49:15Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.name-rules", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Name rules (`F21.name-rules`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-09-name-rules-f21-name-rules.json", "at": "2026-10-13T02:48:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.not-found", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Unknown names and ids (`F21.not-found`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-10-unknown-names-and-ids-f21-not-found.json", "at": "2026-10-13T02:48:23Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.rename", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Rename (`F21.rename`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-13-rename-f21-rename.json", "at": "2026-10-13T02:48:54Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.restart", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Restart (`F21.restart`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-25-restart-f21-restart.json", "at": "2026-10-13T02:52:19Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.save-create", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Create (`F21.save-create`).", "run": "/workspace/verification-artifacts/runs/20261013-030902-92d4-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-030902-92d4-map-f21/evidence/map-run/F21-agent-profiles-01-create-f21-save-create.json", "at": "2026-10-13T03:09:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.save-overwrite", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Overwrite (`F21.save-overwrite`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-06-overwrite-f21-save-overwrite.json", "at": "2026-10-13T02:47:29Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.save-validation", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Save validation (`F21.save-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-08-save-validation-f21-save-validation.json", "at": "2026-10-13T02:47:44Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.secret-scope", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Secret scope (`F21.secret-scope`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-22-secret-scope-f21-secret-scope.json", "at": "2026-10-13T02:51:32Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.seed-acp", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: ACP seed (`F21.seed-acp`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-04-acp-seed-f21-seed-acp.json", "at": "2026-10-13T02:47:26Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.seed-no-ghost-llm", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: No ghost LLM profile (`F21.seed-no-ghost-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-03-no-ghost-llm-profile-f21-seed-no-ghost-l.json", "at": "2026-10-13T02:46:57Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.store-busy", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: Store busy (`F21.store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-27-store-busy-f21-store-busy.json", "at": "2026-10-13T02:54:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.ts-client", "family": "F21", "result": "pass", "label": "map run F21-agent-profiles.md: TypeScript client (`F21.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-24-typescript-client-f21-ts-client.json", "at": "2026-10-13T02:51:45Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F21.ts-payload-types", "family": "F21", "result": "xfail", "label": "map run F21-agent-profiles.md: TypeScript payload types (`F21.ts-payload-types`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030902-92d4-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-030902-92d4-map-f21/evidence/map-run/F21-agent-profiles-04-typescript-payload-types-f21-ts-payload-.json", "at": "2026-10-13T03:09:48Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 2", "note": "known bug reproduced (expected failure); failed at line 26: $TSC \"$D/exported.ts\"" }, { "feature_id": "F22.acp-clears-pointer", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: ACP clears the pointer (`F22.acp-clears-pointer`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-16-acp-clears-the-pointer-f22-acp-clears-po.json", "at": "2026-10-13T02:20:15Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F22.activate", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: Activate (`F22.activate`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-09-activate-f22-activate.json", "at": "2026-10-13T02:18:10Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F22.activate-acp", "family": "F22", "result": "xfail", "label": "map run F22-meta-profiles.md: Activation on ACP settings (`F22.activate-acp`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22/evidence/map-run/F22-meta-profiles-04-activation-on-acp-settings-f22-activate-.json", "at": "2026-10-13T03:43:37Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 17: jq -e '.status >= 400 and .status < 500' \"$F22/acp-activate.json\"" }, { "feature_id": "F22.auth-required", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: Key required (`F22.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-01-key-required-f22-auth-required.json", "at": "2026-10-13T02:17:52Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F22.corrupt-files", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: Corrupted files (`F22.corrupt-files`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-18-corrupted-files-f22-corrupt-files.json", "at": "2026-10-13T02:20:57Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F22.corrupt-settings", "family": "F22", "result": "xfail", "label": "map run F22-meta-profiles.md: Unreadable settings (`F22.corrupt-settings`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22/evidence/map-run/F22-meta-profiles-05-unreadable-settings-f22-corrupt-settings.json", "at": "2026-10-13T03:43:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 15: jq -e '.status == 409' \"$F22/corrupt-settings-activate.json\"" }, { "feature_id": "F22.delete", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: Delete (`F22.delete`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-14-delete-f22-delete.json", "at": "2026-10-13T02:19:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F22.delete-active-clears", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: Delete the active one (`F22.delete-active-clears`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-15-delete-the-active-one-f22-delete-active-.json", "at": "2026-10-13T02:20:01Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F22.empty-router", "family": "F22", "result": "xfail", "label": "map run F22-meta-profiles.md: Router with no route (`F22.empty-router`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22/evidence/map-run/F22-meta-profiles-03-router-with-no-route-f22-empty-router-kn.json", "at": "2026-10-13T03:43:33Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 4: control-agent-server api POST /api/meta-profiles/qa-empty --json '{\"classifier_model\": \"deepseek-flash\"}' --expect 422 --save F22.empty-router/save" }, { "feature_id": "F22.limit", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: Limit (`F22.limit`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-22-limit-f22-limit.json", "at": "2026-10-13T02:28:09Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F22.name-rules", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: Name rules (`F22.name-rules`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-07-name-rules-f22-name-rules.json", "at": "2026-10-13T02:18:02Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F22.not-found", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: Unknown names (`F22.not-found`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-08-unknown-names-f22-not-found.json", "at": "2026-10-13T02:18:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F22.overwrite", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: Overwrite (`F22.overwrite`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-06-overwrite-f22-overwrite.json", "at": "2026-10-13T02:17:59Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F22.persist-restart", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: Restart (`F22.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-13-restart-f22-persist-restart.json", "at": "2026-10-13T02:19:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F22.persistence-dir", "family": "F22", "result": "xfail", "label": "map run F22-meta-profiles.md: Persistence directory with `~` (`F22.persistence-dir`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22/evidence/map-run/F22-meta-profiles-06-persistence-directory-with-f22-persisten.json", "at": "2026-10-13T03:44:02Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 9: control-agent-server state ls 'home/qa-f22-tilde/meta-profiles/qa-tilde.json' --run \"$T\" --expect-count 1" }, { "feature_id": "F22.route-classes", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: Classes-mode routing (`F22.route-classes`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-10-classes-mode-routing-f22-route-classes.json", "at": "2026-10-13T02:18:28Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F22.route-direct", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: Direct routing (`F22.route-direct`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-11-direct-routing-f22-route-direct.json", "at": "2026-10-13T02:18:41Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F22.route-missing-profile", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: Missing LLM profile (`F22.route-missing-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-12-missing-llm-profile-f22-route-missing-pr.json", "at": "2026-10-13T02:18:56Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F22.save-classes", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: Save in classes mode (`F22.save-classes`).", "run": "/workspace/verification-artifacts/runs/20261013-034935-6069-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-034935-6069-map-f22/evidence/map-run/F22-meta-profiles-01-save-in-classes-mode-f22-save-classes.json", "at": "2026-10-13T03:49:54Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F22.save-direct", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: Save in direct mode (`F22.save-direct`).", "run": "/workspace/verification-artifacts/runs/20261013-034935-6069-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-034935-6069-map-f22/evidence/map-run/F22-meta-profiles-02-save-in-direct-mode-f22-save-direct.json", "at": "2026-10-13T03:49:56Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F22.store-busy", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: Store busy (`F22.store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-034935-6069-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-034935-6069-map-f22/evidence/map-run/F22-meta-profiles-03-store-busy-f22-store-busy.json", "at": "2026-10-13T03:51:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F22.store-busy-responsive", "family": "F22", "result": "xfail", "label": "map run F22-meta-profiles.md: Lock wait blocks the server (`F22.store-busy-responsive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-023058-b5d0-f22-lock-confirm-2", "transcript": "/workspace/verification-artifacts/runs/20261013-023058-b5d0-f22-lock-confirm-2/evidence/map-run/F22-meta-profiles-02-lock-wait-blocks-the-server-f22-store-bu.json", "at": "2026-10-13T02:35:07Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 21: test \"$SLOW\" = 0" }, { "feature_id": "F22.ts-client", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: TypeScript client (`F22.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-23-typescript-client-f22-ts-client.json", "at": "2026-10-13T02:28:14Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F22.validation", "family": "F22", "result": "pass", "label": "map run F22-meta-profiles.md: Validation (`F22.validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-04-validation-f22-validation.json", "at": "2026-10-13T02:17:56Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.acp-discovery-errors", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: ACP discovery errors (`F23.acp-discovery-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-27-acp-discovery-errors-f23-acp-discovery-e.json", "at": "2026-10-13T02:47:38Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.auth-required", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: Key required (`F23.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-01-key-required-f23-auth-required.json", "at": "2026-10-13T02:42:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.catalog-ts-client", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: TypeScript client (`F23.catalog-ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-07-typescript-client-f23-catalog-ts-client.json", "at": "2026-10-13T02:42:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.conn-create", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: Create (`F23.conn-create`).", "run": "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/map-run/F23-llm-catalog-and-connections-02-create-f23-conn-create.json", "at": "2026-10-13T03:43:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.conn-create-blank-key", "family": "F23", "result": "xfail", "label": "map run F23-llm-catalog-and-connections.md: Blank keys on create (`F23.conn-create-blank-key`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/map-run/F23-llm-catalog-and-connections-03-blank-keys-on-create-f23-conn-create-bla.json", "at": "2026-10-13T03:43:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 12: test \"$S_SPACE $S_MASK\" = '422 422'" }, { "feature_id": "F23.conn-create-validation", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: Create validation (`F23.conn-create-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-11-create-validation-f23-conn-create-valida.json", "at": "2026-10-13T02:42:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.conn-delete", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: Delete (`F23.conn-delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-25-delete-f23-conn-delete.json", "at": "2026-10-13T02:47:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.conn-key-encrypted", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: Key hidden and encrypted (`F23.conn-key-encrypted`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-09-key-hidden-and-encrypted-f23-conn-key-en.json", "at": "2026-10-13T02:42:19Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.conn-limit", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: Limit (`F23.conn-limit`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-26-limit-f23-conn-limit.json", "at": "2026-10-13T02:47:30Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.conn-patch-blank-key", "family": "F23", "result": "xfail", "label": "map run F23-llm-catalog-and-connections.md: Blank keys on update (`F23.conn-patch-blank-key`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/map-run/F23-llm-catalog-and-connections-04-blank-keys-on-update-f23-conn-patch-blan.json", "at": "2026-10-13T03:43:47Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 21: test \"$APPLIED\" = '[sk-qa-f23-default][sk-qa-f23-default][sk-qa-f23-default]'" }, { "feature_id": "F23.conn-rotate-key", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: Rotate the key (`F23.conn-rotate-key`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-17-rotate-the-key-f23-conn-rotate-key.json", "at": "2026-10-13T02:43:59Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.conn-update", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: Update (`F23.conn-update`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-14-update-f23-conn-update.json", "at": "2026-10-13T02:43:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.conn-update-validation", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: Update validation (`F23.conn-update-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-15-update-validation-f23-conn-update-valida.json", "at": "2026-10-13T02:43:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.corrupt-file", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: Corrupted file (`F23.corrupt-file`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-21-corrupted-file-f23-corrupt-file.json", "at": "2026-10-13T02:46:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.delete-guard", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: Delete guard (`F23.delete-guard`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-24-delete-guard-f23-delete-guard.json", "at": "2026-10-13T02:47:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.linked-run", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: Linked profile runs on the connection (`F23.linked-run`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-13-linked-profile-runs-on-the-connection-f2.json", "at": "2026-10-13T02:42:59Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.models", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: All models (`F23.models`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-03-all-models-f23-models.json", "at": "2026-10-13T02:42:06Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.models-filter", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: Provider filter (`F23.models-filter`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-04-provider-filter-f23-models-filter.json", "at": "2026-10-13T02:42:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.models-no-bedrock", "family": "F23", "result": "xfail", "label": "map run F23-llm-catalog-and-connections.md: No Bedrock ids under a provider (`F23.models-no-bedrock`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/map-run/F23-llm-catalog-and-connections-01-no-bedrock-ids-under-a-provider-f23-mode.json", "at": "2026-10-13T03:43:38Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 5: jq -e 'map(select(test(\"^deepseek\\\\.\"))) | debug | length == 0'" }, { "feature_id": "F23.no-cipher", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: No cipher (`F23.no-cipher`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-10-no-cipher-f23-no-cipher.json", "at": "2026-10-13T02:42:46Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.persist-restart", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: Restart (`F23.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-19-restart-f23-persist-restart.json", "at": "2026-10-13T02:44:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.providers", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: Providers (`F23.providers`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-02-providers-f23-providers.json", "at": "2026-10-13T02:42:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.store-busy", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: Store busy (`F23.store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-22-store-busy-f23-store-busy.json", "at": "2026-10-13T02:46:52Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.store-busy-responsive", "family": "F23", "result": "xfail", "label": "map run F23-llm-catalog-and-connections.md: Lock wait blocks the server (`F23.store-busy-responsive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/map-run/F23-llm-catalog-and-connections-06-lock-wait-blocks-the-server-f23-store-bu.json", "at": "2026-10-13T03:44:06Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 14: control-agent-server api GET /alive --auth none --expect 200 --expect-max-ms 3000 --timeout 60 --save F23.store-busy-responsive/alive" }, { "feature_id": "F23.strict-decrypt", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: Secret key changed (`F23.strict-decrypt`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-20-secret-key-changed-f23-strict-decrypt.json", "at": "2026-10-13T02:46:12Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F23.validate-linked-draft", "family": "F23", "result": "xfail", "label": "map run F23-llm-catalog-and-connections.md: Validate a linked draft (`F23.validate-linked-draft`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/map-run/F23-llm-catalog-and-connections-05-validate-a-linked-draft-f23-validate-lin.json", "at": "2026-10-13T03:43:49Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 3: control-agent-server api POST /api/profiles/qa-f23-draft/validate --timeout 120 --expect 200 --json \"{\\\"llm\\\": {\\\"model\\\": \\\"deepseek/deepseek-flash\\\", \\\"provider_connection_id\\\": \\\"$CONN\\\", \\\"num_retries\\\": 0}}\" --save F23.validate-linked-draft/linked --check valid eq true --check error eq null" }, { "feature_id": "F23.verified-models", "family": "F23", "result": "pass", "label": "map run F23-llm-catalog-and-connections.md: Verified models (`F23.verified-models`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-06-verified-models-f23-verified-models.json", "at": "2026-10-13T02:42:11Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F24.auth-required", "family": "F24", "result": "pass", "label": "map run F24-openai-subscription.md: Key required (`F24.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-04-key-required-f24-auth-required.json", "at": "2026-10-13T02:51:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F24.device-poll-invalid", "family": "F24", "result": "pass", "label": "map run F24-openai-subscription.md: Malformed polls (`F24.device-poll-invalid`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-09-malformed-polls-f24-device-poll-invalid.json", "at": "2026-10-13T02:52:43Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F24.device-poll-outage", "family": "F24", "result": "pass", "label": "map run F24-openai-subscription.md: Outage during a pending login (`F24.device-poll-outage`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-17-outage-during-a-pending-login-f24-device.json", "at": "2026-10-13T02:55:10Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F24.device-poll-pending", "family": "F24", "result": "pass", "label": "map run F24-openai-subscription.md: Pending login (`F24.device-poll-pending`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-12-pending-login-f24-device-poll-pending.json", "at": "2026-10-13T02:52:58Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F24.device-poll-unknown", "family": "F24", "result": "pass", "label": "map run F24-openai-subscription.md: Unknown token (`F24.device-poll-unknown`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-10-unknown-token-f24-device-poll-unknown.json", "at": "2026-10-13T02:52:46Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F24.device-start", "family": "F24", "result": "pass", "label": "map run F24-openai-subscription.md: Start a login (`F24.device-start`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-11-start-a-login-f24-device-start.json", "at": "2026-10-13T02:52:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F24.device-start-offline", "family": "F24", "result": "xfail", "label": "map run F24-openai-subscription.md: Start and poll a login offline (`F24.device-start-offline`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034748-623a-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-034748-623a-map-f24/evidence/map-run/F24-openai-subscription-02-start-and-poll-a-login-offline-f24-devic.json", "at": "2026-10-13T03:49:17Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 9: control-agent-server api POST /api/llm/subscription/openai/device/start --run \"$OUTAGE\" --expect 502,503,504 --save F24.device-start-offline/start" }, { "feature_id": "F24.login-complete", "family": "F24", "result": "blocked", "label": "map run F24-openai-subscription.md: Complete a login (`F24.login-complete`), blocked.", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": null, "at": "2026-10-13T02:56:50Z", "expected": "prerequisite available", "actual": "blocked (see the bullet)", "note": "bullet is marked blocked in the map" }, { "feature_id": "F24.logout", "family": "F24", "result": "pass", "label": "map run F24-openai-subscription.md: Logout (`F24.logout`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-06-logout-f24-logout.json", "at": "2026-10-13T02:52:27Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F24.logout-cancels-pending", "family": "F24", "result": "pass", "label": "map run F24-openai-subscription.md: Logout cancels pending logins (`F24.logout-cancels-pending`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-13-logout-cancels-pending-logins-f24-logout.json", "at": "2026-10-13T02:53:00Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F24.models", "family": "F24", "result": "pass", "label": "map run F24-openai-subscription.md: Subscription models (`F24.models`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-01-subscription-models-f24-models.json", "at": "2026-10-13T02:51:40Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F24.offline-safe", "family": "F24", "result": "pass", "label": "map run F24-openai-subscription.md: Offline-safe routes (`F24.offline-safe`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-16-offline-safe-routes-f24-offline-safe.json", "at": "2026-10-13T02:54:30Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F24.pending-lost-on-restart", "family": "F24", "result": "pass", "label": "map run F24-openai-subscription.md: Pending logins and restarts (`F24.pending-lost-on-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-14-pending-logins-and-restarts-f24-pending-.json", "at": "2026-10-13T02:53:34Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F24.status-connected", "family": "F24", "result": "pass", "label": "map run F24-openai-subscription.md: Status with a stored login (`F24.status-connected`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-03-status-with-a-stored-login-f24-status-co.json", "at": "2026-10-13T02:51:43Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F24.status-disconnected", "family": "F24", "result": "pass", "label": "map run F24-openai-subscription.md: Status with no login (`F24.status-disconnected`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-02-status-with-no-login-f24-status-disconne.json", "at": "2026-10-13T02:51:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F24.status-invalid-file", "family": "F24", "result": "pass", "label": "map run F24-openai-subscription.md: Invalid credentials file (`F24.status-invalid-file`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-08-invalid-credentials-file-f24-status-inva.json", "at": "2026-10-13T02:52:38Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F24.status-offline", "family": "F24", "result": "xfail", "label": "map run F24-openai-subscription.md: Status with an expired login offline (`F24.status-offline`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034748-623a-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-034748-623a-map-f24/evidence/map-run/F24-openai-subscription-01-status-with-an-expired-login-offline-f24.json", "at": "2026-10-13T03:48:41Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 8: control-agent-server api GET /api/llm/subscription/openai/status --run \"$OFF\" --expect 200 --check connected eq false --save F24.status-offline/status" }, { "feature_id": "F24.status-refresh-rejected", "family": "F24", "result": "pass", "label": "map run F24-openai-subscription.md: Rejected refresh (`F24.status-refresh-rejected`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-07-rejected-refresh-f24-status-refresh-reje.json", "at": "2026-10-13T02:52:33Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F24.status-restart", "family": "F24", "result": "pass", "label": "map run F24-openai-subscription.md: Login across a restart (`F24.status-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-05-login-across-a-restart-f24-status-restar.json", "at": "2026-10-13T02:52:25Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F24.ts-client", "family": "F24", "result": "pass", "label": "map run F24-openai-subscription.md: TypeScript client (`F24.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-15-typescript-client-f24-ts-client.json", "at": "2026-10-13T02:53:38Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.auth", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Session key required (`F25.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-01-session-key-required-f25-auth.json", "at": "2026-10-13T02:58:26Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.catalog-project", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Project skills (`F25.catalog-project`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-04-project-skills-f25-catalog-project.json", "at": "2026-10-13T02:58:30Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.catalog-registered-marketplace", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Registered marketplace skills (`F25.catalog-registered-marketplace`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-25-registered-marketplace-skills-f25-catalo.json", "at": "2026-10-13T03:00:03Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.catalog-server-marketplaces", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Server-default marketplaces (`F25.catalog-server-marketplaces`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-29-server-default-marketplaces-f25-catalog-.json", "at": "2026-10-13T03:03:07Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.catalog-server-sources", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Sandbox and organization sources (`F25.catalog-server-sources`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-06-sandbox-and-organization-sources-f25-cat.json", "at": "2026-10-13T02:58:34Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.catalog-user-precedence", "family": "F25", "result": "pass", "label": "map run F25-skills.md: User skills and precedence (`F25.catalog-user-precedence`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-05-user-skills-and-precedence-f25-catalog-u.json", "at": "2026-10-13T02:58:32Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.catalog-validation", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Request validation (`F25.catalog-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-07-request-validation-f25-catalog-validatio.json", "at": "2026-10-13T02:58:38Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.conversation-activation", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Installed skill in a conversation (`F25.conversation-activation`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-26-installed-skill-in-a-conversation-f25-co.json", "at": "2026-10-13T03:00:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.enable-disable", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Disable and enable (`F25.enable-disable`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-09-disable-and-enable-f25-enable-disable.json", "at": "2026-10-13T02:58:49Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.install-bad-frontmatter", "family": "F25", "result": "xfail", "label": "map run F25-skills.md: Malformed frontmatter (`F25.install-bad-frontmatter`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/map-run/F25-skills-03-malformed-frontmatter-f25-install-bad-fr.json", "at": "2026-10-13T03:44:34Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 1: control-agent-server api POST /api/skills/install --json '{\"source\": \"'\"$F/qa-f25-badyaml\"'\"}' --expect 422 --save F25.install-bad-frontmatter/install" }, { "feature_id": "F25.install-conflict", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Already installed (`F25.install-conflict`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-10-already-installed-f25-install-conflict.json", "at": "2026-10-13T02:58:53Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.install-documented-sources", "family": "F25", "result": "xfail", "label": "map run F25-skills.md: Documented install sources (`F25.install-documented-sources`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/map-run/F25-skills-06-documented-install-sources-f25-install-d.json", "at": "2026-10-13T03:44:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 5: control-agent-server api POST /api/skills/install --json '{\"source\": \"'\"$SRC\"'\"}' --timeout 240 --expect 200 --check name eq github --save F25.install-documented-sources/documented" }, { "feature_id": "F25.install-errors", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Install errors (`F25.install-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-11-install-errors-f25-install-errors.json", "at": "2026-10-13T02:59:02Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.install-git", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Install from a git source (`F25.install-git`).", "run": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/map-run/F25-skills-04-install-from-a-git-source-f25-install-gi.json", "at": "2026-10-13T03:44:35Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.install-github", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Install from GitHub (`F25.install-github`).", "run": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/map-run/F25-skills-05-install-from-github-f25-install-github.json", "at": "2026-10-13T03:44:37Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.install-local", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Install from a local directory (`F25.install-local`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-08-install-from-a-local-directory-f25-insta.json", "at": "2026-10-13T02:58:43Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.install-ref", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Install at a ref (`F25.install-ref`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-18-install-at-a-ref-f25-install-ref.json", "at": "2026-10-13T02:59:35Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.install-unknown-ref", "family": "F25", "result": "xfail", "label": "map run F25-skills.md: Unknown ref (`F25.install-unknown-ref`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/map-run/F25-skills-07-unknown-ref-f25-install-unknown-ref-know.json", "at": "2026-10-13T03:44:40Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 7: control-agent-server api POST /api/skills/install --json '{\"source\": \"file://'\"$G\"'\", \"repo_path\": \"skills/qa-f25-src-skill\", \"ref\": \"qa-f25-no-such-ref\", \"force\": true}' --expect 400 --save F25.install-unknown-ref/cached" }, { "feature_id": "F25.invoke-skill", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Skill invoked through `invoke_skill` (`F25.invoke-skill`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-27-skill-invoked-through-invoke-skill-f25-i.json", "at": "2026-10-13T03:00:48Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.marketplace-catalog", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Marketplace catalog (`F25.marketplace-catalog`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-30-marketplace-catalog-f25-marketplace-cata.json", "at": "2026-10-13T03:03:12Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.marketplace-public", "family": "F25", "result": "xfail", "label": "map run F25-skills.md: Public marketplace catalog (`F25.marketplace-public`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/map-run/F25-skills-02-public-marketplace-catalog-f25-marketpla.json", "at": "2026-10-13T03:44:33Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 1: control-agent-server api GET /api/skills/marketplace --timeout 180 --expect 200 --check skills len-ge 1 --save F25.marketplace-public/catalog" }, { "feature_id": "F25.marketplace-stale-cache", "family": "F25", "result": "xfail", "label": "map run F25-skills.md: Catalog after a sync (`F25.marketplace-stale-cache`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/map-run/F25-skills-11-catalog-after-a-sync-f25-marketplace-sta.json", "at": "2026-10-13T03:44:46Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 11: control-agent-server api GET /api/skills/marketplace --expect 200 --check skills contains '\"name\": \"qa-f25-pub-extra\"' --save F25.marketplace-stale-cache/after-sync" }, { "feature_id": "F25.name-errors", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Unknown and invalid names (`F25.name-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-21-unknown-and-invalid-names-f25-name-error.json", "at": "2026-10-13T02:59:49Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.public-repo", "family": "F25", "result": "pass", "label": "map run F25-skills.md: The real public repository (`F25.public-repo`).", "run": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/map-run/F25-skills-01-the-real-public-repository-f25-public-re.json", "at": "2026-10-13T03:44:32Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.refresh", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Refresh from the source (`F25.refresh`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-17-refresh-from-the-source-f25-refresh.json", "at": "2026-10-13T02:59:32Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.refresh-unfetchable", "family": "F25", "result": "xfail", "label": "map run F25-skills.md: Refresh of a vanished source (`F25.refresh-unfetchable`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/map-run/F25-skills-08-refresh-of-a-vanished-source-f25-refresh.json", "at": "2026-10-13T03:44:41Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 6: control-agent-server api POST /api/skills/installed/qa-f25-gone/refresh --expect 400,422 --save F25.refresh-unfetchable/refresh" }, { "feature_id": "F25.restart-persist", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Restart (`F25.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-28-restart-f25-restart-persist.json", "at": "2026-10-13T03:01:31Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.sdk-load-skills", "family": "F25", "result": "pass", "label": "map run F25-skills.md: SDK consumer (`F25.sdk-load-skills`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-14-sdk-consumer-f25-sdk-load-skills.json", "at": "2026-10-13T02:59:20Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.self-heal", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Hand-copied and hand-deleted directories (`F25.self-heal`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-22-hand-copied-and-hand-deleted-directories.json", "at": "2026-10-13T02:59:52Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.sync", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Sync the public repository (`F25.sync`).", "run": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/map-run/F25-skills-09-sync-the-public-repository-f25-sync.json", "at": "2026-10-13T03:44:44Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.sync-offline", "family": "F25", "result": "xfail", "label": "map run F25-skills.md: Sync without a reachable origin (`F25.sync-offline`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/map-run/F25-skills-10-sync-without-a-reachable-origin-f25-sync.json", "at": "2026-10-13T03:44:45Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 4: control-agent-server api POST /api/skills/sync --expect 200 --check status eq error --save F25.sync-offline/sync" }, { "feature_id": "F25.ts-client", "family": "F25", "result": "pass", "label": "map run F25-skills.md: TypeScript client (`F25.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-31-typescript-client-f25-ts-client.json", "at": "2026-10-13T03:03:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F25.uninstall", "family": "F25", "result": "pass", "label": "map run F25-skills.md: Uninstall (`F25.uninstall`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-33-uninstall-f25-uninstall.json", "at": "2026-10-13T03:03:27Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.auth", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Session key required (`F26.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-01-session-key-required-f26-auth.json", "at": "2026-10-13T03:01:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.catalog-project", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Project plugins (`F26.catalog-project`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-02-project-plugins-f26-catalog-project.json", "at": "2026-10-13T03:01:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.catalog-user-precedence", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: User plugins and precedence (`F26.catalog-user-precedence`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-03-user-plugins-and-precedence-f26-catalog-.json", "at": "2026-10-13T03:01:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.conversation-ambient", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Ambient load into a conversation (`F26.conversation-ambient`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-18-ambient-load-into-a-conversation-f26-con.json", "at": "2026-10-13T03:03:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.conversation-disabled", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Disabled plugin in a new conversation (`F26.conversation-disabled`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-19-disabled-plugin-in-a-new-conversation-f2.json", "at": "2026-10-13T03:03:58Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.conversation-resume-disabled", "family": "F26", "result": "xfail", "label": "map run F26-plugins.md: Resume after disabling (`F26.conversation-resume-disabled`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26/evidence/map-run/F26-plugins-04-resume-after-disabling-f26-conversation-.json", "at": "2026-10-13T03:46:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 12: control-agent-server ws stop qa-f26-resume --kinds MessageEvent --contains '\"activated_skills\": [\"qa-f26-inst:qa-hello\"]' --expect-none --save F26.conversation-resume-disabled/frames" }, { "feature_id": "F26.enable-disable", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Enable and disable (`F26.enable-disable`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-12-enable-and-disable-f26-enable-disable.json", "at": "2026-10-13T03:02:28Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.install-agent-plugins", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Agent Plugins layout (`F26.install-agent-plugins`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-05-agent-plugins-layout-f26-install-agent-p.json", "at": "2026-10-13T03:01:25Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.install-conflict", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Duplicate install (`F26.install-conflict`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-08-duplicate-install-f26-install-conflict.json", "at": "2026-10-13T03:01:38Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.install-documented-source", "family": "F26", "result": "xfail", "label": "map run F26-plugins.md: Documented install source (`F26.install-documented-source`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26/evidence/map-run/F26-plugins-02-documented-install-source-f26-install-do.json", "at": "2026-10-13T03:45:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 7: control-agent-server api POST /api/plugins/install --json '{\"source\": \"github:/plugins/city-weather\"}' --timeout 240 --expect 200 --check name eq city-weather --save F26.install-documented-source/documented" }, { "feature_id": "F26.install-errors", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Install errors (`F26.install-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-09-install-errors-f26-install-errors.json", "at": "2026-10-13T03:01:59Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.install-git", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Install from a git source (`F26.install-git`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-06-install-from-a-git-source-f26-install-gi.json", "at": "2026-10-13T03:01:27Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.install-github", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Install from GitHub (`F26.install-github`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-10-install-from-github-f26-install-github.json", "at": "2026-10-13T03:02:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.install-local", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Install from a local path (`F26.install-local`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-04-install-from-a-local-path-f26-install-lo.json", "at": "2026-10-13T03:01:22Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.install-repo-root-name", "family": "F26", "result": "xfail", "label": "map run F26-plugins.md: Repository root name (`F26.install-repo-root-name`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26/evidence/map-run/F26-plugins-01-repository-root-name-f26-install-repo-ro.json", "at": "2026-10-13T03:45:19Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 5: test \"$N\" = qa-f26-src" }, { "feature_id": "F26.marketplace-cache", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Catalog cache (`F26.marketplace-cache`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-23-catalog-cache-f26-marketplace-cache.json", "at": "2026-10-13T03:05:51Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.marketplace-local", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Marketplace against a local stand-in (`F26.marketplace-local`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-22-marketplace-against-a-local-stand-in-f26.json", "at": "2026-10-13T03:05:23Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.marketplace-public", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Public marketplace (`F26.marketplace-public`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-25-public-marketplace-f26-marketplace-publi.json", "at": "2026-10-13T03:06:26Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.name-errors", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Unknown and invalid names (`F26.name-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-13-unknown-and-invalid-names-f26-name-error.json", "at": "2026-10-13T03:02:34Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.refresh", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Refresh (`F26.refresh`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-14-refresh-f26-refresh.json", "at": "2026-10-13T03:02:43Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.refresh-unfetchable", "family": "F26", "result": "xfail", "label": "map run F26-plugins.md: Unfetchable refresh (`F26.refresh-unfetchable`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26/evidence/map-run/F26-plugins-03-unfetchable-refresh-f26-refresh-unfetcha.json", "at": "2026-10-13T03:45:24Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 11: control-agent-server api POST /api/plugins/installed/qa-f26-gone/refresh --expect 400,422 --save F26.refresh-unfetchable/vanished" }, { "feature_id": "F26.restart-persist", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Persistence across a restart (`F26.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-21-persistence-across-a-restart-f26-restart.json", "at": "2026-10-13T03:05:20Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.self-heal", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Hand-edited store (`F26.self-heal`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-15-hand-edited-store-f26-self-heal.json", "at": "2026-10-13T03:02:52Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.ts-client", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: TypeScript client (`F26.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-24-typescript-client-f26-ts-client.json", "at": "2026-10-13T03:05:53Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F26.uninstall", "family": "F26", "result": "pass", "label": "map run F26-plugins.md: Uninstall (`F26.uninstall`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-17-uninstall-f26-uninstall.json", "at": "2026-10-13T03:03:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.agent-definitions", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Forwarded sub-agent definitions (`F27.agent-definitions`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-21-forwarded-sub-agent-definitions-f27-agen.json", "at": "2026-10-13T03:08:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.auth", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Session key required (`F27.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-01-session-key-required-f27-auth.json", "at": "2026-10-13T03:06:27Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.browser-disabled", "family": "F27", "result": "pass", "label": "map run F27-browser-config.md: Browser config ordinary (`F27.browser-disabled-launch`, `F27.browser-disabled`).", "run": "/workspace/verification-artifacts/runs/20261013-035121-8592-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-035121-8592-map-f27/evidence/map-run/F27-browser-config-01-browser-config-ordinary-f27-browser-disa.json", "at": "2026-10-13T03:52:02Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.browser-disabled-launch", "family": "F27", "result": "pass", "label": "map run F27-browser-config.md: Browser config ordinary (`F27.browser-disabled-launch`, `F27.browser-disabled`).", "run": "/workspace/verification-artifacts/runs/20261013-035121-8592-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-035121-8592-map-f27/evidence/map-run/F27-browser-config-01-browser-config-ordinary-f27-browser-disa.json", "at": "2026-10-13T03:52:02Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.browser-playwright-path", "family": "F27", "result": "xfail", "label": "map run F27-browser-container.md: Playwright override (`F27.browser-playwright-path`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-040554-b24c-qa-f27-browser-path-2", "transcript": "/workspace/verification-artifacts/runs/20261013-040554-b24c-qa-f27-browser-path-2/evidence/map-run/F27-browser-container-02-playwright-override-f27-browser-playwrig.json", "at": "2026-10-13T04:06:27Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 6: control-agent-server api GET /api/tools/catalog --quiet --check tools contains '{\"name\": \"browser_tool_set\", \"user_selectable\": true, \"usable\": true' --save F27.browser-playwright-path/catalog" }, { "feature_id": "F27.browser-probe", "family": "F27", "result": "pass", "label": "map run F27-browser-container.md: Home-cache positive control (`F27.browser-probe`).", "run": "/workspace/verification-artifacts/runs/20261013-040554-b24c-qa-f27-browser-path-2", "transcript": "/workspace/verification-artifacts/runs/20261013-040554-b24c-qa-f27-browser-path-2/evidence/map-run/F27-browser-container-01-home-cache-positive-control-f27-browser-.json", "at": "2026-10-13T04:06:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.catalog", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Tool catalog (`F27.catalog`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-05-tool-catalog-f27-catalog.json", "at": "2026-10-13T03:06:31Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.catalog-docker-runtime", "family": "F27", "result": "pass", "label": "map run F27-browser-config.md: Docker-runtime catalog (`F27.catalog-docker-runtime`).", "run": "/workspace/verification-artifacts/runs/20261013-035121-8592-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-035121-8592-map-f27/evidence/map-run/F27-browser-config-04-docker-runtime-catalog-f27-catalog-docke.json", "at": "2026-10-13T03:53:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.catalog-sealed", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Client tools stay out of the catalog (`F27.catalog-sealed`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-19-client-tools-stay-out-of-the-catalog-f27.json", "at": "2026-10-13T03:07:33Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.client-tool-call", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Client tool call (`F27.client-tool-call`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-20-client-tool-call-f27-client-tool-call.json", "at": "2026-10-13T03:08:10Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.custom-tool-missing-on-resume", "family": "F27", "result": "xfail", "label": "map run F27-hooks-subagents-tools.md: Tool module gone on resume (`F27.custom-tool-missing-on-resume`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034502-5a13-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-034502-5a13-map-f27/evidence/map-run/F27-hooks-subagents-tools-03-tool-module-gone-on-resume-f27-custom-to.json", "at": "2026-10-13T03:46:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 4: control-agent-server api POST \"/$MCID/events\" --json '{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"Reply with the single word ok.\"}], \"run\": false}' --expect 2xx,4xx --save F27.custom-tool-missing-on-resume/send" }, { "feature_id": "F27.custom-tool-modules", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Custom tool module (`F27.custom-tool-modules`).", "run": "/workspace/verification-artifacts/runs/20261013-034502-5a13-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-034502-5a13-map-f27/evidence/map-run/F27-hooks-subagents-tools-02-custom-tool-module-f27-custom-tool-modul.json", "at": "2026-10-13T03:46:20Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.hooks-absent", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: No hooks (`F27.hooks-absent`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-08-no-hooks-f27-hooks-absent.json", "at": "2026-10-13T03:06:37Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.hooks-block", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Blocking hooks (`F27.hooks-block`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-18-blocking-hooks-f27-hooks-block.json", "at": "2026-10-13T03:07:32Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.hooks-conversation", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Hooks in a conversation (`F27.hooks-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-16-hooks-in-a-conversation-f27-hooks-conver.json", "at": "2026-10-13T03:07:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.hooks-formats", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Formats (`F27.hooks-formats`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-07-formats-f27-hooks-formats.json", "at": "2026-10-13T03:06:33Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.hooks-invalid", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Broken hooks.json (`F27.hooks-invalid`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-09-broken-hooks-json-f27-hooks-invalid.json", "at": "2026-10-13T03:06:41Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.hooks-not-automatic", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Hooks are not loaded implicitly (`F27.hooks-not-automatic`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-17-hooks-are-not-loaded-implicitly-f27-hook.json", "at": "2026-10-13T03:07:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.hooks-project", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Project hooks (`F27.hooks-project`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-06-project-hooks-f27-hooks-project.json", "at": "2026-10-13T03:06:32Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.import-modules", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Startup imports (`F27.import-modules`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-23-startup-imports-f27-import-modules.json", "at": "2026-10-13T03:09:40Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.server-info-browser-disabled", "family": "F27", "result": "xfail", "label": "map run F27-browser-config.md: Server-info browser disabled (`F27.server-info-browser-disabled`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-035121-8592-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-035121-8592-map-f27/evidence/map-run/F27-browser-config-02-server-info-browser-disabled-f27-server-.json", "at": "2026-10-13T03:52:03Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 2: control-agent-server api GET /server_info --auth none --check usable_tools contains terminal --check usable_tools not-contains browser_tool_set --save F27.server-info-browser-disabled/server-info" }, { "feature_id": "F27.subagents-bad-skill", "family": "F27", "result": "xfail", "label": "map run F27-hooks-subagents-tools.md: Agent with a missing skill (`F27.subagents-bad-skill`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034502-5a13-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-034502-5a13-map-f27/evidence/map-run/F27-hooks-subagents-tools-01-agent-with-a-missing-skill-f27-subagents.json", "at": "2026-10-13T03:45:20Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 8: control-agent-server api POST \"/$BCID/events\" --json '{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"hi\"}], \"run\": false}' --expect 2xx,4xx --save F27.subagents-bad-skill/send" }, { "feature_id": "F27.subagents-browser-disabled", "family": "F27", "result": "xfail", "label": "map run F27-browser-config.md: Subagents browser disabled (`F27.subagents-browser-disabled`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-035121-8592-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-035121-8592-map-f27/evidence/map-run/F27-browser-config-03-subagents-browser-disabled-f27-subagents.json", "at": "2026-10-13T03:52:04Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 2: control-agent-server api POST /api/sub-agents --json '{\"load_user\":false,\"load_project\":false}' --check agents contains '\"name\": \"bash-runner\"' --check agents not-contains '\"name\": \"web-researcher\"' --save F27.subagents-browser-disabled/builtins" }, { "feature_id": "F27.subagents-builtin", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Sub-agent built-ins (`F27.subagents-builtin`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-10-sub-agent-built-ins-f27-subagents-builti.json", "at": "2026-10-13T03:06:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.subagents-invalid", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Invalid definitions (`F27.subagents-invalid`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-14-invalid-definitions-f27-subagents-invali.json", "at": "2026-10-13T03:06:44Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.subagents-precedence", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Precedence (`F27.subagents-precedence`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-13-precedence-f27-subagents-precedence.json", "at": "2026-10-13T03:06:43Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.subagents-project", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Project sub-agents (`F27.subagents-project`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-11-project-sub-agents-f27-subagents-project.json", "at": "2026-10-13T03:06:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.subagents-user", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: User sub-agents (`F27.subagents-user`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-12-user-sub-agents-f27-subagents-user.json", "at": "2026-10-13T03:06:43Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.tools-list", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Registered tools (`F27.tools-list`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-03-registered-tools-f27-tools-list.json", "at": "2026-10-13T03:06:30Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.tools-redirect", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Slash redirects (`F27.tools-redirect`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-04-slash-redirects-f27-tools-redirect.json", "at": "2026-10-13T03:06:31Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F27.validation", "family": "F27", "result": "pass", "label": "map run F27-hooks-subagents-tools.md: Bad bodies (`F27.validation`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-02-bad-bodies-f27-validation.json", "at": "2026-10-13T03:06:29Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.backend-approval-gate", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Approval gate (`F28.backend-approval-gate`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-13-approval-gate-f28-backend-approval-gate.json", "at": "2026-10-13T02:43:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.backend-inherit-environment", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Inherited environment (`F28.backend-inherit-environment`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-21-inherited-environment-f28-backend-inheri.json", "at": "2026-10-13T02:45:02Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.backend-logs", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Logs while starting (`F28.backend-logs`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-18-logs-while-starting-f28-backend-logs.json", "at": "2026-10-13T02:44:14Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.backend-prepare", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Prepare (`F28.backend-prepare`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-14-prepare-f28-backend-prepare.json", "at": "2026-10-13T02:43:09Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.backend-start", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Start (`F28.backend-start`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-15-start-f28-backend-start.json", "at": "2026-10-13T02:43:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.backend-start-fails", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Failed starts (`F28.backend-start-fails`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-20-failed-starts-f28-backend-start-fails.json", "at": "2026-10-13T02:44:57Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.backend-status", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Backend status (`F28.backend-status`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-12-backend-status-f28-backend-status.json", "at": "2026-10-13T02:42:57Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.backend-stop", "family": "F28", "result": "fail", "label": "map run F28-canvas-extensions.md: Stop (`F28.backend-stop`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-16-stop-f28-backend-stop.json", "at": "2026-10-13T02:43:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "automated map run; prose-only expectations were not re-judged; failed at line 16: control-agent-server api POST /api/canvas-extensions/installed/qa-f28-flaky/backend/stop --expect 200 --check state eq stopped --expect-max-ms 7000 --save F28.backend-stop/group" }, { "feature_id": "F28.broken-install", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Broken installs (`F28.broken-install`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-11-broken-installs-f28-broken-install.json", "at": "2026-10-13T02:42:56Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.bundle", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Bundle (`F28.bundle`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-06-bundle-f28-bundle.json", "at": "2026-10-13T02:42:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.disable-stops-backend", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Disable stops the backend (`F28.disable-stops-backend`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-22-disable-stops-the-backend-f28-disable-st.json", "at": "2026-10-13T02:45:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.enable-disable", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Enable and disable (`F28.enable-disable`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-08-enable-and-disable-f28-enable-disable.json", "at": "2026-10-13T02:42:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.force-reinstall", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Reinstall (`F28.force-reinstall`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-17-reinstall-f28-force-reinstall.json", "at": "2026-10-13T02:43:56Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.hard-restart-orphan", "family": "F28", "result": "xfail", "label": "map run F28-canvas-extensions.md: Backend outlives a crash (`F28.hard-restart-orphan`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28/evidence/map-run/F28-canvas-extensions-04-backend-outlives-a-crash-f28-hard-restar.json", "at": "2026-10-13T02:54:02Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 11: false" }, { "feature_id": "F28.icon", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Icon (`F28.icon`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-07-icon-f28-icon.json", "at": "2026-10-13T02:42:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.install-errors", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Install errors (`F28.install-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-05-install-errors-f28-install-errors.json", "at": "2026-10-13T02:42:35Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.install-git", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Install from a git URL (`F28.install-git`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-02-install-from-a-git-url-f28-install-git.json", "at": "2026-10-13T02:42:26Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.install-local", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Install from a local directory (`F28.install-local`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-01-install-from-a-local-directory-f28-insta.json", "at": "2026-10-13T02:42:24Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.install-ref", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Pin a git ref (`F28.install-ref`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-03-pin-a-git-ref-f28-install-ref.json", "at": "2026-10-13T02:42:29Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.install-ref-cached", "family": "F28", "result": "xfail", "label": "map run F28-canvas-extensions.md: Unknown ref on a cached source (`F28.install-ref-cached`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28/evidence/map-run/F28-canvas-extensions-02-unknown-ref-on-a-cached-source-f28-insta.json", "at": "2026-10-13T02:53:12Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 4: control-agent-server api POST /api/canvas-extensions/install --json \"{\\\"source\\\": \\\"file://$GX\\\", \\\"ref\\\": \\\"qa-no-such-ref\\\", \\\"force\\\": true}\" --expect 400 --check detail contains 'Could not read canvas extension source' --save F28.install-ref-cached/unknown-ref" }, { "feature_id": "F28.install-repo-path", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Install a subdirectory (`F28.install-repo-path`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-04-install-a-subdirectory-f28-install-repo-.json", "at": "2026-10-13T02:42:30Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.list-discovery", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Discovery by listing (`F28.list-discovery`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-10-discovery-by-listing-f28-list-discovery.json", "at": "2026-10-13T02:42:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.prepare-corrupt-archive", "family": "F28", "result": "xfail", "label": "map run F28-canvas-extensions.md: Non-gzip artifact (`F28.prepare-corrupt-archive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28/evidence/map-run/F28-canvas-extensions-01-non-gzip-artifact-f28-prepare-corrupt-ar.json", "at": "2026-10-13T02:53:01Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 6: control-agent-server api POST /api/canvas-extensions/installed/qa-f28-notgz/backend/prepare --json \"{\\\"revision\\\": \\\"$NREV\\\"}\" --expect 409 --save F28.prepare-corrupt-archive/prepare" }, { "feature_id": "F28.restart-persistence", "family": "F28", "result": "pass", "label": "map run F28-core-supplement.md: Persistence (`F28.restart-persistence`).", "run": "/workspace/verification-artifacts/runs/20261013-040412-f5ed-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-040412-f5ed-map-f28/evidence/map-run/F28-core-supplement-01-persistence-f28-restart-persistence.json", "at": "2026-10-13T04:04:48Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.status-exit-hang", "family": "F28", "result": "fail", "label": "map run F28-canvas-extensions.md: Status after the leader exits (`F28.status-exit-hang`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-28-status-after-the-leader-exits-f28-status.json", "at": "2026-10-13T02:46:43Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "automated map run; prose-only expectations were not re-judged; failed at line 18: false" }, { "feature_id": "F28.status-probe-latch", "family": "F28", "result": "xfail", "label": "map run F28-canvas-extensions.md: Health probe latch (`F28.status-probe-latch`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28/evidence/map-run/F28-canvas-extensions-03-health-probe-latch-f28-status-probe-latc.json", "at": "2026-10-13T02:53:27Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 11: control-agent-server api GET /api/canvas-extensions/installed/qa-f28-flaky/backend --until-ok 5 --check state eq ready --save F28.status-probe-latch/after-recovery" }, { "feature_id": "F28.ts-client", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: TypeScript client (`F28.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-19-typescript-client-f28-ts-client.json", "at": "2026-10-13T02:44:27Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.uninstall", "family": "F28", "result": "pass", "label": "map run F28-core-supplement.md: Uninstall (`F28.uninstall`).", "run": "/workspace/verification-artifacts/runs/20261013-040412-f5ed-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-040412-f5ed-map-f28/evidence/map-run/F28-core-supplement-02-uninstall-f28-uninstall.json", "at": "2026-10-13T04:04:51Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F28.unknown-name", "family": "F28", "result": "pass", "label": "map run F28-canvas-extensions.md: Unknown and invalid names (`F28.unknown-name`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-09-unknown-and-invalid-names-f28-unknown-na.json", "at": "2026-10-13T02:42:47Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.backend-crash-detected", "family": "F29", "result": "xfail", "label": "map run F29-app-backend-bridge.md: Crash seen by the bridge (`F29.backend-crash-detected`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-06-crash-seen-by-the-bridge-f29-backend-cra.json", "at": "2026-10-13T03:20:25Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 7: control-agent-server api GET /echo --auth none --cookie \"oh_app_backend_session=$CC\" --until-ok 15 --expect 503 --check exception eq '503: Canvas App backend is not ready' --save F29.backend-crash-detected/crashed" }, { "feature_id": "F29.backend-lifecycle-revokes", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Backend stop and disable revoke (`F29.backend-lifecycle-revokes`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-23-backend-stop-and-disable-revoke-f29-back.json", "at": "2026-10-13T02:27:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.ingress-advertised", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Ingress advertised (`F29.ingress-advertised`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-01-ingress-advertised-f29-ingress-advertise.json", "at": "2026-10-13T02:18:00Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.ingress-config-errors", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Ingress not configured or unusable (`F29.ingress-config-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-02-ingress-not-configured-or-unusable-f29-i.json", "at": "2026-10-13T02:20:24Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.proxy-cookie-required", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Cookie required (`F29.proxy-cookie-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-10-cookie-required-f29-proxy-cookie-require.json", "at": "2026-10-13T02:22:00Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.proxy-credentials-stripped", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Credentials stripped (`F29.proxy-credentials-stripped`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-14-credentials-stripped-f29-proxy-credentia.json", "at": "2026-10-13T02:23:06Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.proxy-get", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Proxied safe methods (`F29.proxy-get`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-09-proxied-safe-methods-f29-proxy-get.json", "at": "2026-10-13T02:21:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.proxy-path-traversal", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Path traversal (`F29.proxy-path-traversal`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-18-path-traversal-f29-proxy-path-traversal.json", "at": "2026-10-13T02:23:49Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.proxy-request-framing", "family": "F29", "result": "xfail", "label": "map run F29-app-backend-bridge.md: Request framing (`F29.proxy-request-framing`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-03-request-framing-f29-proxy-request-framin.json", "at": "2026-10-13T03:20:04Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 10: control-agent-server api GET /app-backends/qa-headers/echo --auth none --cookie \"oh_app_backend_session=$FR\" --expect 200 --check method eq GET --check body eq '' --check headers.transfer-encoding missing --save F29.proxy-request-framing/get" }, { "feature_id": "F29.proxy-response-filter", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Response filter (`F29.proxy-response-filter`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-16-response-filter-f29-proxy-response-filte.json", "at": "2026-10-13T02:23:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.proxy-response-headers", "family": "F29", "result": "xfail", "label": "map run F29-app-backend-bridge.md: One Date and Server header (`F29.proxy-response-headers`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-04-one-date-and-server-header-f29-proxy-res.json", "at": "2026-10-13T03:20:06Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 8: control-agent-server api GET /echo --auth none --cookie \"oh_app_backend_session=$RH\" --expect 200 --check-header date matches '^[A-Z][a-z]{2}, [0-9]{2} [A-Z][a-z]{2} [0-9]{4} [0-9:]{8} GMT$' --check-header server not-contains ', ' --all-headers --save F29.proxy-response-headers/echo" }, { "feature_id": "F29.proxy-root", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: App root (`F29.proxy-root`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-12-app-root-f29-proxy-root.json", "at": "2026-10-13T02:22:52Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.proxy-root-path-query", "family": "F29", "result": "xfail", "label": "map run F29-app-backend-bridge.md: Root `path` parameter (`F29.proxy-root-path-query`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-02-root-path-parameter-f29-proxy-root-path-.json", "at": "2026-10-13T03:20:03Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 7: control-agent-server api GET --query path=echo --auth none --cookie \"oh_app_backend_session=$RP\" --expect 200 --check path eq / --check query.path eq echo --save F29.proxy-root-path-query/get" }, { "feature_id": "F29.proxy-unsafe-methods", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Unsafe methods (`F29.proxy-unsafe-methods`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-11-unsafe-methods-f29-proxy-unsafe-methods.json", "at": "2026-10-13T02:22:27Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.session-backend-not-ready", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Backend not ready (`F29.session-backend-not-ready`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-08-backend-not-ready-f29-session-backend-no.json", "at": "2026-10-13T02:21:27Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.session-bootstrap", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Session bootstrap (`F29.session-bootstrap`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-03-session-bootstrap-f29-session-bootstrap.json", "at": "2026-10-13T02:20:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.session-configured-control-origin", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Configured control origins (`F29.session-configured-control-origin`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-25-configured-control-origins-f29-session-c.json", "at": "2026-10-13T02:32:53Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.session-forwarded-proto", "family": "F29", "result": "xfail", "label": "map run F29-app-backend-bridge.md: Forwarded proto (`F29.session-forwarded-proto`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-01-forwarded-proto-f29-session-forwarded-pr.json", "at": "2026-10-13T03:20:02Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 6: control-agent-server api POST /session --header 'Origin: http://localhost:3000' --header 'X-Forwarded-Proto: https' --expect 200 --check ingress_url eq \"$S/\" --save F29.session-forwarded-proto/session" }, { "feature_id": "F29.session-ingress-host", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Ingress host (`F29.session-ingress-host`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-05-ingress-host-f29-session-ingress-host.json", "at": "2026-10-13T02:20:58Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.session-key-required", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Session key required (`F29.session-key-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-04-session-key-required-f29-session-key-req.json", "at": "2026-10-13T02:20:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.session-origin-rules", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Origin rules (`F29.session-origin-rules`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-07-origin-rules-f29-session-origin-rules.json", "at": "2026-10-13T02:21:17Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.session-revoke", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Revoke a session (`F29.session-revoke`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-22-revoke-a-session-f29-session-revoke.json", "at": "2026-10-13T02:25:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.session-ttl", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Session expiry (`F29.session-ttl`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-28-session-expiry-f29-session-ttl.json", "at": "2026-10-13T02:43:27Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.sessions-not-persisted", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Server restart (`F29.sessions-not-persisted`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-27-server-restart-f29-sessions-not-persiste.json", "at": "2026-10-13T02:38:17Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.trusted-forwarded-headers", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Trusted forwarded headers (`F29.trusted-forwarded-headers`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-26-trusted-forwarded-headers-f29-trusted-fo.json", "at": "2026-10-13T02:36:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.uninstall-revokes", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Uninstall (`F29.uninstall-revokes`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-29-uninstall-f29-uninstall-revokes.json", "at": "2026-10-13T02:43:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.ws-bridge", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: WebSocket bridge (`F29.ws-bridge`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-19-websocket-bridge-f29-ws-bridge.json", "at": "2026-10-13T02:24:00Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.ws-handshake-rejections", "family": "F29", "result": "pass", "label": "map run F29-app-backend-bridge.md: Handshake rejections (`F29.ws-handshake-rejections`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-20-handshake-rejections-f29-ws-handshake-re.json", "at": "2026-10-13T02:24:15Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F29.ws-subprotocol", "family": "F29", "result": "xfail", "label": "map run F29-app-backend-bridge.md: Subprotocol (`F29.ws-subprotocol`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-05-subprotocol-f29-ws-subprotocol-known-bug.json", "at": "2026-10-13T03:20:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 10: control-agent-server exec --expect-output 'subprotocol=qa.v1' --expect-output '\"protocol\": \"qa.v1\"' --save F29.ws-subprotocol/bridged -- \"$PWD/.venv/bin/python\" -c 'import sys; from websockets.sync.client import connect; ws = connect(sys.argv[1], subprotocols=[\"qa.v1\"], origin=sys.argv[2], additional_headers={\"Cookie\": \"oh_app_backend_session=\" + sys.argv[3]}, proxy=None); print(\"subprotocol=\" + str(ws.subprotocol)); ws.send(\"qa-f29-proto\"); print(ws.recv(timeout=10)); ws.close()' \"ws://127.0.0." }, { "feature_id": "F30.auth-bearer-or-header", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Bearer or session header (`F30.auth-bearer-or-header`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-02-bearer-or-session-header-f30-auth-bearer.json", "at": "2026-10-13T02:18:34Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.auth-open-without-keys", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: No session keys (`F30.auth-open-without-keys`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-03-no-session-keys-f30-auth-open-without-ke.json", "at": "2026-10-13T02:19:20Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.chat-agent-tools", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: The agent runs its own tools (`F30.chat-agent-tools`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-14-the-agent-runs-its-own-tools-f30-chat-ag.json", "at": "2026-10-13T02:23:33Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.chat-basic", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Chat completion (`F30.chat-basic`).", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-01-chat-completion-f30-chat-basic.json", "at": "2026-10-13T03:27:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.chat-confirmation-409", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Confirmation stops a new conversation (`F30.chat-confirmation-409`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-18-confirmation-stops-a-new-conversation-f3.json", "at": "2026-10-13T02:26:04Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.chat-header-new-id", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Caller-chosen id (`F30.chat-header-new-id`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-10-caller-chosen-id-f30-chat-header-new-id.json", "at": "2026-10-13T02:22:02Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.chat-input-errors", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Chat input errors (`F30.chat-input-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-15-chat-input-errors-f30-chat-input-errors.json", "at": "2026-10-13T02:24:07Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.chat-latest-user-only", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Only the latest user turn (`F30.chat-latest-user-only`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-06-only-the-latest-user-turn-f30-chat-lates.json", "at": "2026-10-13T02:19:40Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.chat-reuse-after-restart", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Continue after a restart (`F30.chat-reuse-after-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-09-continue-after-a-restart-f30-chat-reuse-.json", "at": "2026-10-13T02:21:28Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.chat-reuse-concurrent", "family": "F30", "result": "xfail", "label": "map run F30-openai-gateway.md: Overlapping turns on one conversation (`F30.chat-reuse-concurrent`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-04-overlapping-turns-on-one-conversation-f3.json", "at": "2026-10-13T03:28:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 25: control-agent-server state cat fixtures/qa-f30/race-x.json --check choices.0.message.content eq QA_F30_RACE_B" }, { "feature_id": "F30.chat-reuse-confirmation", "family": "F30", "result": "xfail", "label": "map run F30-openai-gateway.md: Confirmation on a continued conversation (`F30.chat-reuse-confirmation`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-05-confirmation-on-a-continued-conversation.json", "at": "2026-10-13T03:30:12Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 20: control-agent-server state cat fixtures/qa-f30/reconf-2.json --check status eq 409 --check response.body.detail eq 'Agent run ended with status: waiting_for_confirmation' --check elapsed_ms lt 30000" }, { "feature_id": "F30.chat-reuse-header", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Continue with the header (`F30.chat-reuse-header`).", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-02-continue-with-the-header-f30-chat-reuse-.json", "at": "2026-10-13T03:27:46Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.chat-reuse-keeps-llm", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Continuing keeps the conversation's LLM (`F30.chat-reuse-keeps-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-11-continuing-keeps-the-conversation-s-llm-.json", "at": "2026-10-13T02:22:30Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.chat-reuse-run-limit-429", "family": "F30", "result": "xfail", "label": "map run F30-openai-gateway.md: Run limit on a continued conversation (`F30.chat-reuse-run-limit-429`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-06-run-limit-on-a-continued-conversation-f3.json", "at": "2026-10-13T03:31:09Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 18: control-agent-server api GET \"/$CID/events/search\" --query source=user --query body=QA_F30_429_REUSE --check items len-eq 0 --save F30.chat-reuse-run-limit-429/user-messages" }, { "feature_id": "F30.chat-run-error-500", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Run ends in error (`F30.chat-run-error-500`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-17-run-ends-in-error-f30-chat-run-error-500.json", "at": "2026-10-13T02:25:35Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.chat-run-limit-429", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Run limit (`F30.chat-run-limit-429`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-20-run-limit-f30-chat-run-limit-429.json", "at": "2026-10-13T02:32:23Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.chat-stream", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Streaming (`F30.chat-stream`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-07-streaming-f30-chat-stream.json", "at": "2026-10-13T02:19:53Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.chat-system-suffix", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: System text goes to the system prompt (`F30.chat-system-suffix`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-05-system-text-goes-to-the-system-prompt-f3.json", "at": "2026-10-13T02:19:33Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.chat-timeout-504", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Gateway timeout (`F30.chat-timeout-504`).", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-07-gateway-timeout-f30-chat-timeout-504.json", "at": "2026-10-13T03:33:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.chat-usage-per-request", "family": "F30", "result": "xfail", "label": "map run F30-openai-gateway.md: Usage per request (`F30.chat-usage-per-request`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-03-usage-per-request-f30-chat-usage-per-req.json", "at": "2026-10-13T03:27:53Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 20: control-agent-server state cat fixtures/qa-f30/usage-2.json --check usage.prompt_tokens eq 3 --check usage.completion_tokens eq 2 --check usage.total_tokens eq 5" }, { "feature_id": "F30.models-list", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Model list (`F30.models-list`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-01-model-list-f30-models-list.json", "at": "2026-10-13T02:18:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.observability-headers", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Observability headers (`F30.observability-headers`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-16-observability-headers-f30-observability-.json", "at": "2026-10-13T02:24:43Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.openai-sdk", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Official OpenAI SDK (`F30.openai-sdk`).", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-11-official-openai-sdk-f30-openai-sdk.json", "at": "2026-10-13T03:33:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.responses-basic", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Responses (`F30.responses-basic`).", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-08-responses-f30-responses-basic.json", "at": "2026-10-13T03:33:17Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.responses-errors", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Responses errors (`F30.responses-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-10-responses-errors-f30-responses-errors.json", "at": "2026-10-13T03:33:29Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F30.responses-stateless-replay", "family": "F30", "result": "pass", "label": "map run F30-openai-gateway.md: Stateless replay (`F30.responses-stateless-replay`).", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-09-stateless-replay-f30-responses-stateless.json", "at": "2026-10-13T03:33:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.config-invalid-fails-fast", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: Invalid limits stop startup (`F31.config-invalid-fails-fast`).", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-10-invalid-limits-stop-startup-f31-config-i.json", "at": "2026-10-13T03:09:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.config-null-override", "family": "F31", "result": "xfail", "label": "map run F31-persistence-and-limits.md: Null from the environment (`F31.config-null-override`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-09-null-from-the-environment-f31-config-nul.json", "at": "2026-10-13T03:09:11Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 9: control-agent-server api GET /server_info --auth none --check app_backend_ingress_url missing --check capabilities not-contains canvas_app_backend_bridge_v1 --save F31.config-null-override/server-info" }, { "feature_id": "F31.config-precedence", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: Config file and environment (`F31.config-precedence`).", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-08-config-file-and-environment-f31-config-p.json", "at": "2026-10-13T03:08:03Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.config-silent-env", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: Unrecognised booleans and choices start anyway (`F31.config-silent-env`).", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-11-unrecognised-booleans-and-choices-start-.json", "at": "2026-10-13T03:11:24Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.crash-tool-call", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: Crash during a tool call (`F31.crash-tool-call`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-05-crash-during-a-tool-call-f31-crash-tool-.json", "at": "2026-10-13T02:22:25Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.create-capacity", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: Create at capacity (`F31.create-capacity`).", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-07-create-at-capacity-f31-create-capacity.json", "at": "2026-10-13T03:07:17Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.eviction-skips-busy", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: Busy conversations are kept (`F31.eviction-skips-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-06-busy-conversations-are-kept-f31-eviction.json", "at": "2026-10-13T03:06:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.fenced-write-no-trace", "family": "F31", "result": "xfail", "label": "map run F31-persistence-and-limits.md: Refused write leaves no trace (`F31.fenced-write-no-trace`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-03-refused-write-leaves-no-trace-f31-fenced.json", "at": "2026-10-13T03:02:58Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 4: control-agent-server api GET \"/$F\" --check title ne 'QA F31 stale owner' --save F31.fenced-write-no-trace/get" }, { "feature_id": "F31.idle-eviction", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: Idle eviction (`F31.idle-eviction`).", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-05-idle-eviction-f31-idle-eviction.json", "at": "2026-10-13T03:05:55Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.lease-claimed", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: Lease claimed on load (`F31.lease-claimed`).", "run": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31/evidence/map-run/F31-persistence-and-limits-01-lease-claimed-on-load-f31-lease-claimed.json", "at": "2026-10-13T03:41:55Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.lease-dead-owner-takeover", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: Dead owner takeover (`F31.lease-dead-owner-takeover`).", "run": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31/evidence/map-run/F31-persistence-and-limits-02-dead-owner-takeover-f31-lease-dead-owner.json", "at": "2026-10-13T03:42:09Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.lease-disabled", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: Leasing disabled (`F31.lease-disabled`).", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-04-leasing-disabled-f31-lease-disabled.json", "at": "2026-10-13T03:04:12Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.lease-expiry-fencing", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: Lease expiry fences a frozen owner (`F31.lease-expiry-fencing`).", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-02-lease-expiry-fences-a-frozen-owner-f31-l.json", "at": "2026-10-13T03:02:51Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.lease-released-graceful", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: Graceful release (`F31.lease-released-graceful`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-03-graceful-release-f31-lease-released-grac.json", "at": "2026-10-13T02:20:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.lease-renewed", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: Lease renewed (`F31.lease-renewed`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-02-lease-renewed-f31-lease-renewed.json", "at": "2026-10-13T02:19:17Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.no-plaintext-at-rest", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: Nothing secret in plaintext (`F31.no-plaintext-at-rest`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-07-nothing-secret-in-plaintext-f31-no-plain.json", "at": "2026-10-13T02:25:58Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.second-instance-catalog", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: Second server reads the catalog (`F31.second-instance-catalog`).", "run": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31/evidence/map-run/F31-persistence-and-limits-03-second-server-reads-the-catalog-f31-seco.json", "at": "2026-10-13T03:42:38Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.second-instance-lease-held", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: Second server refused while the owner lives (`F31.second-instance-lease-held`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-09-second-server-refused-while-the-owner-li.json", "at": "2026-10-13T02:32:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.second-instance-takeover", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: Takeover after the owner dies (`F31.second-instance-takeover`).", "run": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31/evidence/map-run/F31-persistence-and-limits-04-takeover-after-the-owner-dies-f31-second.json", "at": "2026-10-13T03:42:57Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.state-survives-restarts", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: State survives restarts (`F31.state-survives-restarts`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-06-state-survives-restarts-f31-state-surviv.json", "at": "2026-10-13T02:25:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.storage-retention", "family": "F31", "result": "pass", "label": "map run F31-persistence-and-limits.md: Storage retention and disk budget (`F31.storage-retention`); Docker source-image adaptation.", "run": "/workspace/verification-artifacts/runs/20261013-034826-d9b8-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-034826-d9b8-map-f31/evidence/map-run/F31-persistence-and-limits-01-storage-retention-and-disk-budget-f31-st.json", "at": "2026-10-13T03:54:58Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F31.takeover-keeps-metadata", "family": "F31", "result": "xfail", "label": "map run F31-persistence-and-limits.md: Takeover keeps newer metadata (`F31.takeover-keeps-metadata`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-01-takeover-keeps-newer-metadata-f31-takeov.json", "at": "2026-10-13T03:01:10Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 24: control-agent-server state cat meta.json --conversation \"$Y\" --check title eq 'QA F31 after B'" }, { "feature_id": "F31.tmux-dir-cleanup", "family": "F31", "result": "xfail", "label": "map run F31-persistence-and-limits.md: Per-process tmux directory removed on stop (`F31.tmux-dir-cleanup`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-031758-c007-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-031758-c007-map-f31/evidence/map-run/F31-persistence-and-limits-01-per-process-tmux-directory-removed-on-st.json", "at": "2026-10-13T03:18:30Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 10: test ! -e \"$TD\"" }, { "feature_id": "F32.backpressure", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Batch and queue limits (`F32.backpressure`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-12-batch-and-queue-limits-f32-backpressure.json", "at": "2026-10-13T02:28:31Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.buffer-size", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Custom headers, path prefix and buffer size (`F32.custom-headers`, `F32.buffer-size`).", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-01-custom-headers-path-prefix-and-buffer-si.json", "at": "2026-10-13T03:25:59Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.buffer-stranded", "family": "F32", "result": "xfail", "label": "map run F32-webhooks-and-telemetry.md: Events stranded behind an in-flight batch (`F32.buffer-stranded`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-02-events-stranded-behind-an-in-flight-batc.json", "at": "2026-10-13T03:26:10Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 4: control-agent-server sink read --path \"/qa-f32-b/events/${BID//-/}\" --contains QA_F32_BUFFER --expect-min 1 --wait 10 --save F32.buffer-stranded/after-wait" }, { "feature_id": "F32.conversation-created", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Conversation created (`F32.conversation-created`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-03-conversation-created-f32-conversation-cr.json", "at": "2026-10-13T02:19:08Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.conversation-lifecycle", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Conversation lifecycle posts (`F32.conversation-lifecycle`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-05-conversation-lifecycle-posts-f32-convers.json", "at": "2026-10-13T02:20:26Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.custom-headers", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Custom headers, path prefix and buffer size (`F32.custom-headers`, `F32.buffer-size`).", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-01-custom-headers-path-prefix-and-buffer-si.json", "at": "2026-10-13T03:25:59Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.deferred-init", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Deferred pod gets telemetry and webhooks from `/api/init` (`F32.deferred-init`).", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-17-deferred-pod-gets-telemetry-and-webhooks.json", "at": "2026-10-13T03:34:02Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.deferred-init-flag", "family": "F32", "result": "xfail", "label": "map run F32-webhooks-and-telemetry.md: Deferred pods are labelled as such (`F32.deferred-init-flag`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-18-deferred-pods-are-labelled-as-such-f32-d.json", "at": "2026-10-13T03:34:03Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 3: control-agent-server sink read --name qa-f32-deferred --path /init --contains '\"deferred_init\": true' --expect-min 1 --save F32.deferred-init-flag/events" }, { "feature_id": "F32.env-config", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Webhooks from the environment (`F32.env-config`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-16-webhooks-from-the-environment-f32-env-co.json", "at": "2026-10-13T02:37:01Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.events-full-state", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Full-state snapshot on every attach (`F32.events-full-state`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-04-full-state-snapshot-on-every-attach-f32-.json", "at": "2026-10-13T02:19:57Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.events-no-deltas", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: No streaming deltas on webhooks (`F32.events-no-deltas`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-06-no-streaming-deltas-on-webhooks-f32-even.json", "at": "2026-10-13T02:20:49Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.events-post", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: A run's events reach the webhook (`F32.events-post`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-01-a-run-s-events-reach-the-webhook-f32-eve.json", "at": "2026-10-13T02:19:04Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.events-session-key", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Session key and content type on every request (`F32.events-session-key`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-02-session-key-and-content-type-on-every-re.json", "at": "2026-10-13T02:19:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.flush-delay", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Flush timer from the first event (`F32.flush-delay`).", "run": "/workspace/verification-artifacts/runs/20261013-034519-5d31-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-034519-5d31-map-f32/evidence/map-run/F32-webhooks-and-telemetry-01-flush-timer-from-the-first-event-f32-flu.json", "at": "2026-10-13T03:46:01Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.flush-on-close", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Flush on delete and on shutdown (`F32.flush-on-close`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-09-flush-on-delete-and-on-shutdown-f32-flus.json", "at": "2026-10-13T02:24:55Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.requeue", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Failed batches are requeued (`F32.requeue`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-14-failed-batches-are-requeued-f32-requeue.json", "at": "2026-10-13T02:32:58Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.retry", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Retries against a failing receiver (`F32.retry`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-13-retries-against-a-failing-receiver-f32-r.json", "at": "2026-10-13T02:30:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.slow-receiver", "family": "F32", "result": "xfail", "label": "map run F32-webhooks-and-telemetry.md: A slow receiver blocks the next run (`F32.slow-receiver`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-05-a-slow-receiver-blocks-the-next-run-f32-.json", "at": "2026-10-13T03:27:02Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 10: control-agent-server api POST \"/$WCID/run\" --expect 200 --save F32.slow-receiver/run-after-finished" }, { "feature_id": "F32.telemetry-consent-settings", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Consent from settings (`F32.telemetry-consent-settings`).", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-12-consent-from-settings-f32-telemetry-cons.json", "at": "2026-10-13T03:30:31Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.telemetry-conversation", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Conversation telemetry is sanitized (`F32.telemetry-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-08-conversation-telemetry-is-sanitized-f32-.json", "at": "2026-10-13T03:29:20Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.telemetry-conversation-failed", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Failed runs (`F32.telemetry-conversation-failed`).", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-11-failed-runs-f32-telemetry-conversation-f.json", "at": "2026-10-13T03:29:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.telemetry-env-override", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Environment override beats settings (`F32.telemetry-env-override`).", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-13-environment-override-beats-settings-f32-.json", "at": "2026-10-13T03:31:25Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.telemetry-kill-switch", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Kill switch (`F32.telemetry-kill-switch`).", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-14-kill-switch-f32-telemetry-kill-switch.json", "at": "2026-10-13T03:32:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.telemetry-lifecycle", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Server lifecycle events (`F32.telemetry-lifecycle`).", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-07-server-lifecycle-events-f32-telemetry-li.json", "at": "2026-10-13T03:29:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.telemetry-off-by-default", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Telemetry is off by default (`F32.telemetry-off-by-default`).", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-06-telemetry-is-off-by-default-f32-telemetr.json", "at": "2026-10-13T03:28:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.telemetry-request-failed", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Unhandled errors (`F32.telemetry-request-failed`).", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-09-unhandled-errors-f32-telemetry-request-f.json", "at": "2026-10-13T03:29:25Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.telemetry-salt-and-kind", "family": "F32", "result": "pass", "label": "map run F32-webhooks-and-telemetry.md: Pseudonym key and deployment kind (`F32.telemetry-salt-and-kind`).", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-15-pseudonym-key-and-deployment-kind-f32-te.json", "at": "2026-10-13T03:33:29Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F32.telemetry-salt-docs", "family": "F32", "result": "xfail", "label": "map run F32-webhooks-and-telemetry.md: Salt description (`F32.telemetry-salt-docs`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-16-salt-description-f32-telemetry-salt-docs.json", "at": "2026-10-13T03:33:31Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 2: control-agent-server api GET /openapi.json --auth none --quiet --check components.schemas.TelemetrySpec.properties.salt.description matches '(?i)secret'" }, { "feature_id": "F32.telemetry-usage-buckets", "family": "F32", "result": "xfail", "label": "map run F32-webhooks-and-telemetry.md: Token and cost buckets from real usage (`F32.telemetry-usage-buckets`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-10-token-and-cost-buckets-from-real-usage-f.json", "at": "2026-10-13T03:29:26Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 3: control-agent-server sink read --name qa-f32-telemetry --contains '\"total_tokens_bucket\": \"unknown\"' --expect-max 0 --save F32.telemetry-usage-buckets/outcome" }, { "feature_id": "F32.webhook-spec-docs", "family": "F32", "result": "xfail", "label": "map run F32-webhooks-and-telemetry.md: Webhook field descriptions (`F32.webhook-spec-docs`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-04-webhook-field-descriptions-f32-webhook-s.json", "at": "2026-10-13T03:26:34Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 2: control-agent-server api GET /openapi.json --auth none --quiet --check components.schemas.WebhookSpec.properties.flush_delay.description not-contains 'Timer is reset on each new event'" }, { "feature_id": "F33.async-workspace", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Async workspace (`F33.async-workspace`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-07-async-workspace-f33-async-workspace.json", "at": "2026-10-13T02:27:02Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.client-tools", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Client-defined tool (`F33.client-tools`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-15-client-defined-tool-f33-client-tools.json", "at": "2026-10-13T02:38:22Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.completion-logs", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Completion logs on the client (`F33.completion-logs`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-05-completion-logs-on-the-client-f33-comple.json", "at": "2026-10-13T03:48:11Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-ask-title", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Ask, generate and set a title (`F33.conversation-ask-title`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-09-ask-generate-and-set-a-title-f33-convers.json", "at": "2026-10-13T03:50:13Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-attach", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Reattach from a second program (`F33.conversation-attach`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-06-reattach-from-a-second-program-f33-conve.json", "at": "2026-10-13T03:48:37Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-attach-large", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Attach to a long history (`F33.conversation-attach-large`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-07-attach-to-a-long-history-f33-conversatio.json", "at": "2026-10-13T03:49:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-callbacks", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Send and run to completion (`F33.conversation-run`, `F33.conversation-callbacks`, `F33.conversation-stats`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-01-send-and-run-to-completion-f33-conversat.json", "at": "2026-10-13T03:47:38Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-close", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Close keeps or deletes (`F33.conversation-close`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-27-close-keeps-or-deletes-f33-conversation-.json", "at": "2026-10-13T03:58:44Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-confirmation", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Confirm or reject (`F33.conversation-confirmation`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-13-confirm-or-reject-f33-conversation-confi.json", "at": "2026-10-13T03:51:32Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-create", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Create a conversation without running it (`F33.conversation-create`).", "run": "/workspace/verification-artifacts/runs/20261013-035525-7e07-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-035525-7e07-map-f33/evidence/map-run/F33-sdk-remote-client-01-create-a-conversation-without-running-it.json", "at": "2026-10-13T03:55:51Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-create-profile", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Create from a saved agent profile (`F33.conversation-create-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-10-create-from-a-saved-agent-profile-f33-co.json", "at": "2026-10-13T02:31:41Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-create-request", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Create from a request, twice (`F33.conversation-create-request`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-09-create-from-a-request-twice-f33-conversa.json", "at": "2026-10-13T02:29:30Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-fork", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Fork (`F33.conversation-fork`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-10-fork-f33-conversation-fork.json", "at": "2026-10-13T03:50:33Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-fork-branch", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Fork from an event, then move HEAD (`F33.conversation-fork-branch`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-11-fork-from-an-event-then-move-head-f33-co.json", "at": "2026-10-13T03:51:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-interrupt", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Interrupt a blocking run (`F33.conversation-interrupt`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-15-interrupt-a-blocking-run-f33-conversatio.json", "at": "2026-10-13T03:52:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-pause", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Pause a blocking run (`F33.conversation-pause`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-14-pause-a-blocking-run-f33-conversation-pa.json", "at": "2026-10-13T03:52:00Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-plugins-tag", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Plugin source with inline credentials (`F33.conversation-plugins-tag`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-12-plugin-source-with-inline-credentials-f3.json", "at": "2026-10-13T03:51:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-reconnect", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: A live conversation across a restart (`F33.conversation-reconnect`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-22-a-live-conversation-across-a-restart-f33.json", "at": "2026-10-13T03:56:15Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-reconnect-callbacks", "family": "F33", "result": "xfail", "label": "map run F33-sdk-remote-client.md: Callbacks after a reconnect (`F33.conversation-reconnect-callbacks`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-23-callbacks-after-a-reconnect-f33-conversa.json", "at": "2026-10-13T03:56:47Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 50: test \"$(jq .callback_got_message \"$F33/gap.json\")\" = true" }, { "feature_id": "F33.conversation-restart", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Reattach after a restart and continue (`F33.conversation-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-24-reattach-after-a-restart-and-continue-f3.json", "at": "2026-10-13T03:57:27Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-run", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Send and run to completion (`F33.conversation-run`, `F33.conversation-callbacks`, `F33.conversation-stats`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-01-send-and-run-to-completion-f33-conversat.json", "at": "2026-10-13T03:47:38Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-run-error", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Provider error surfaces (`F33.conversation-run-error`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-20-provider-error-surfaces-f33-conversation.json", "at": "2026-10-13T03:54:44Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-run-join-long-tool", "family": "F33", "result": "xfail", "label": "map run F33-sdk-remote-client.md: Join a run during a long tool call (`F33.conversation-run-join-long-tool`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-17-join-a-run-during-a-long-tool-call-f33-c.json", "at": "2026-10-13T03:53:55Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 34: test \"$(jq .joined \"$F33/longtool.json\")\" = true" }, { "feature_id": "F33.conversation-run-timeout", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Time out, then wait again (`F33.conversation-run-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-16-time-out-then-wait-again-f33-conversatio.json", "at": "2026-10-13T03:52:59Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-secrets", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Secrets for the agent (`F33.conversation-secrets`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-19-secrets-for-the-agent-f33-conversation-s.json", "at": "2026-10-13T03:54:27Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-state-reread", "family": "F33", "result": "xfail", "label": "map run F33-sdk-remote-client.md: Read the state twice (`F33.conversation-state-reread`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-08-read-the-state-twice-f33-conversation-st.json", "at": "2026-10-13T03:49:53Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 26: test \"$(jq .stable \"$F33/reread.json\")\" = true" }, { "feature_id": "F33.conversation-stats", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Send and run to completion (`F33.conversation-run`, `F33.conversation-callbacks`, `F33.conversation-stats`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-01-send-and-run-to-completion-f33-conversat.json", "at": "2026-10-13T03:47:38Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-stop-hook", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: A Stop hook that refuses the first stop (`F33.conversation-stop-hook`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-14-a-stop-hook-that-refuses-the-first-stop-.json", "at": "2026-10-13T02:36:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-streaming", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Streamed tokens (`F33.conversation-streaming`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-02-streamed-tokens-f33-conversation-streami.json", "at": "2026-10-13T03:47:53Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-streaming-state", "family": "F33", "result": "xfail", "label": "map run F33-sdk-remote-client.md: Deltas kept out of `state.events` (`F33.conversation-streaming-state`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-04-deltas-kept-out-of-state-events-f33-conv.json", "at": "2026-10-13T03:47:54Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 2: test \"$(jq '.local.StreamingDeltaEvent // 0' \"$F33/stream.json\")\" -eq 0" }, { "feature_id": "F33.conversation-stuck", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: A stuck agent (`F33.conversation-stuck`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-18-a-stuck-agent-f33-conversation-stuck.json", "at": "2026-10-13T03:54:07Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.conversation-token-callbacks", "family": "F33", "result": "xfail", "label": "map run F33-sdk-remote-client.md: Token callbacks on a remote conversation (`F33.conversation-token-callbacks`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-03-token-callbacks-on-a-remote-conversation.json", "at": "2026-10-13T03:47:53Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 2: test \"$(jq .tokens \"$F33/stream.json\")\" -gt 0" }, { "feature_id": "F33.custom-tool-module", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: A custom tool module (`F33.custom-tool-module`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-25-a-custom-tool-module-f33-custom-tool-mod.json", "at": "2026-10-13T03:58:18Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.custom-tool-module-unimportable", "family": "F33", "result": "xfail", "label": "map run F33-sdk-remote-client.md: An agent tool the server cannot import (`F33.custom-tool-module-unimportable`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-26-an-agent-tool-the-server-cannot-import-f.json", "at": "2026-10-13T03:58:28Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 24: test \"$(jq .server_errors \"$F33/ghost.json\")\" -eq 0" }, { "feature_id": "F33.workspace-background", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Background command (`F33.workspace-background`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-03-background-command-f33-workspace-backgro.json", "at": "2026-10-13T02:23:20Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.workspace-bad-key", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Wrong session key (`F33.workspace-bad-key`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-02-wrong-session-key-f33-workspace-bad-key.json", "at": "2026-10-13T02:22:36Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.workspace-exec", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Commands, server info and liveness (`F33.workspace-exec`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-01-commands-server-info-and-liveness-f33-wo.json", "at": "2026-10-13T02:21:51Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.workspace-files", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: File round trip (`F33.workspace-files`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-04-file-round-trip-f33-workspace-files.json", "at": "2026-10-13T02:23:58Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.workspace-git", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Git changes and diff (`F33.workspace-git`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-05-git-changes-and-diff-f33-workspace-git.json", "at": "2026-10-13T02:24:54Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.workspace-runtime-lifecycle", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Runtime key and release (`F33.workspace-runtime-lifecycle`); Docker source-image adaptation.", "run": "/workspace/verification-artifacts/runs/20261013-035525-7e07-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-035525-7e07-map-f33/evidence/map-run/F33-sdk-remote-client-02-runtime-key-and-release-f33-workspace-ru.json", "at": "2026-10-13T03:57:15Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.workspace-runtime-scope", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: Conversation-scoped workspace (`F33.workspace-runtime-scope`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-11-conversation-scoped-workspace-f33-worksp.json", "at": "2026-10-13T02:33:21Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.workspace-settings", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: LLM profiles, secrets and MCP from the server (`F33.workspace-settings`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-06-llm-profiles-secrets-and-mcp-from-the-se.json", "at": "2026-10-13T02:25:57Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F33.ws-unavailable", "family": "F33", "result": "pass", "label": "map run F33-sdk-remote-client.md: No events socket (`F33.ws-unavailable`).", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-21-no-events-socket-f33-ws-unavailable.json", "at": "2026-10-13T03:55:33Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.bash-client", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Bash client (`F34.bash-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-09-bash-client-f34-bash-client.json", "at": "2026-10-13T02:24:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.bash-websocket", "family": "F34", "result": "xfail", "label": "map run F34-typescript-client.md: Bash events socket from Node (`F34.bash-websocket`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-02-bash-events-socket-from-node-f34-bash-we.json", "at": "2026-10-13T02:54:41Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 22: jq -e '.saw_output and .errors == []' \"$F34/bash-ws.json\"" }, { "feature_id": "F34.conversation-branching", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Fork, navigate and ask (`F34.conversation-branching`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-24-fork-navigate-and-ask-f34-conversation-b.json", "at": "2026-10-13T02:31:37Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.conversation-confirm", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Confirmation and secrets (`F34.conversation-confirm`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-25-confirmation-and-secrets-f34-conversatio.json", "at": "2026-10-13T02:33:05Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.conversation-controls", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Title and confirmation policy (`F34.conversation-controls`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-19-title-and-confirmation-policy-f34-conver.json", "at": "2026-10-13T02:28:50Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.conversation-create", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Create a conversation (`F34.conversation-create`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-12-create-a-conversation-f34-conversation-c.json", "at": "2026-10-13T02:25:22Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.conversation-run", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Send and run (`F34.conversation-run`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-13-send-and-run-f34-conversation-run.json", "at": "2026-10-13T02:26:22Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.endpoint-audit", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Endpoint audit against this checkout (`F34.endpoint-audit`).", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-10-endpoint-audit-against-this-checkout-f34.json", "at": "2026-10-13T02:57:53Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.event-stream", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Event stream on Node's WebSocket (`F34.event-stream`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-17-event-stream-on-node-s-websocket-f34-eve.json", "at": "2026-10-13T02:27:49Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.event-stream-reconnect", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Event stream reconnect (`F34.event-stream-reconnect`).", "run": "/workspace/verification-artifacts/runs/20261013-030612-9862-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-030612-9862-map-f34/evidence/map-run/F34-typescript-client-01-event-stream-reconnect-f34-event-stream-.json", "at": "2026-10-13T03:07:17Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.event-stream-reconnect-native", "family": "F34", "result": "xpass", "label": "map run F34-typescript-client.md: Reconnect on Node's own WebSocket (`F34.event-stream-reconnect-native`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030612-9862-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-030612-9862-map-f34/evidence/map-run/F34-typescript-client-02-reconnect-on-node-s-own-websocket-f34-ev.json", "at": "2026-10-13T03:07:55Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "known bug no longer reproduces; update the map" }, { "feature_id": "F34.events-list", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Events list (`F34.events-list`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-14-events-list-f34-events-list.json", "at": "2026-10-13T02:26:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.events-search-kind", "family": "F34", "result": "xfail", "label": "map run F34-typescript-client.md: Short event kinds (`F34.events-search-kind`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-04-short-event-kinds-f34-events-search-kind.json", "at": "2026-10-13T02:54:51Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 16: jq -e '.short == .full and .short_count == 1' \"$F34/kind.json\"" }, { "feature_id": "F34.events-unknown-id", "family": "F34", "result": "xfail", "label": "map run F34-typescript-client.md: Unknown event id (`F34.events-unknown-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-03-unknown-event-id-f34-events-unknown-id-k.json", "at": "2026-10-13T02:54:48Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 16: jq -e '.single.http_error and .single.status == 404' \"$F34/unknown-event.json\"" }, { "feature_id": "F34.hooks-client", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Hooks (`F34.hooks-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-26-hooks-f34-hooks-client.json", "at": "2026-10-13T02:33:37Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.http-errors", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Errors (`F34.http-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-02-errors-f34-http-errors.json", "at": "2026-10-13T02:21:58Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.manager", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Conversation manager (`F34.manager`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-20-conversation-manager-f34-manager.json", "at": "2026-10-13T02:29:26Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.manager-acp", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: ACP conversations (`F34.manager-acp`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-23-acp-conversations-f34-manager-acp.json", "at": "2026-10-13T02:30:52Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.manager-create-options", "family": "F34", "result": "xfail", "label": "map run F34-typescript-client.md: Create options (`F34.manager-create-options`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-07-create-options-f34-manager-create-option.json", "at": "2026-10-13T02:55:22Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 21: jq -e '.max_iterations == 7 and .stuck_detection == false' \"$F34/create-options.json\"" }, { "feature_id": "F34.manager-update-result", "family": "F34", "result": "xfail", "label": "map run F34-typescript-client.md: Update result (`F34.manager-update-result`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-06-update-result-f34-manager-update-result-.json", "at": "2026-10-13T02:55:16Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 14: jq -e --arg id \"$SCID\" '.id == $id and .title == \"qa-f34-renamed\"' \"$F34/update.json\"" }, { "feature_id": "F34.mcp-client", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: MCP test (`F34.mcp-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-27-mcp-test-f34-mcp-client.json", "at": "2026-10-13T02:35:06Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.restart-reload", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Restart (`F34.restart-reload`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-28-restart-f34-restart-reload.json", "at": "2026-10-13T02:36:42Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.server-client", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Server status and version gate (`F34.server-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-01-server-status-and-version-gate-f34-serve.json", "at": "2026-10-13T02:21:45Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.version-gate-old", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Old server version (`F34.version-gate-old`).", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-11-old-server-version-f34-version-gate-old.json", "at": "2026-10-13T02:58:24Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.workspace-binary-download", "family": "F34", "result": "xfail", "label": "map run F34-typescript-client.md: Binary download through RemoteWorkspace (`F34.workspace-binary-download`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024359-a3ff-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-024359-a3ff-map-f34/evidence/map-run/F34-typescript-client-01-binary-download-through-remoteworkspace-.json", "at": "2026-10-13T02:44:39Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 16: jq -e '.blob == [255, 0, 128, 254, 65] and .file_size == 5' \"$F34/binary.json\"" }, { "feature_id": "F34.workspace-exec", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Workspace commands (`F34.workspace-exec`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-03-workspace-commands-f34-workspace-exec.json", "at": "2026-10-13T02:22:12Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.workspace-files", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Workspace files (`F34.workspace-files`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-04-workspace-files-f34-workspace-files.json", "at": "2026-10-13T02:22:25Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.workspace-git", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Git through RemoteWorkspace (`F34.workspace-git`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-07-git-through-remoteworkspace-f34-workspac.json", "at": "2026-10-13T02:23:11Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.workspace-git-types", "family": "F34", "result": "xfail", "label": "map run F34-typescript-client.md: Git status type (`F34.workspace-git-types`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-01-git-status-type-f34-workspace-git-types-.json", "at": "2026-10-13T02:54:20Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 2", "note": "known bug reproduced (expected failure); failed at line 22: $TSC \"$F34/git-types.mts\"" }, { "feature_id": "F34.workspace-scoped", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Conversation-scoped workspace (`F34.workspace-scoped`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-06-conversation-scoped-workspace-f34-worksp.json", "at": "2026-10-13T02:22:49Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.workspace-session", "family": "F34", "result": "pass", "label": "map run F34-typescript-client.md: Workspace session (`F34.workspace-session`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-11-workspace-session-f34-workspace-session.json", "at": "2026-10-13T02:24:51Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 0", "note": "automated map run; prose-only expectations were not re-judged" }, { "feature_id": "F34.ws-callback-client", "family": "F34", "result": "xfail", "label": "map run F34-typescript-client.md: Conversation callback socket from Node (`F34.ws-callback-client`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-05-conversation-callback-socket-from-node-f.json", "at": "2026-10-13T02:55:09Z", "expected": "every command in the bullet exits 0 (its --expect/--check assertions hold)", "actual": "exit 1", "note": "known bug reproduced (expected failure); failed at line 20: jq -e '(.kinds | index(\"ConversationStateUpdateEvent\")) != null and .errors == []' \"$F34/callback.json\"" } ], "triage_batches": [ { "file": "triage-f01-f07.json", "complete": true, "checkout_sha": "acca181af1f353975965bcf6486e1d0a58f444c0", "server_version": "1.54.0", "coverage": [ { "family": "F01", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "feature_counts": { "pass": 11 }, "raw_summary": { "pass": 12, "fail": 0, "xfail": 0, "xpass": 0, "blocked": 0, "skipped": 0, "no-commands": 0 } }, { "family": "F02", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "feature_counts": { "blocked": 1, "pass": 18, "xfail": 2 }, "raw_summary": { "pass": 19, "fail": 1, "xfail": 2, "xpass": 0, "blocked": 0, "skipped": 0, "no-commands": 0 } }, { "family": "F03", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "feature_counts": { "pass": 22, "xfail": 1 }, "raw_summary": { "pass": 23, "fail": 0, "xfail": 1, "xpass": 0, "blocked": 0, "skipped": 0, "no-commands": 0 } }, { "family": "F04", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "feature_counts": { "pass": 25, "xfail": 7 }, "raw_summary": { "pass": 26, "fail": 0, "xfail": 7, "xpass": 0, "blocked": 0, "skipped": 0, "no-commands": 0 } }, { "family": "F05", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "feature_counts": { "fail": 2, "pass": 26, "xfail": 3 }, "raw_summary": { "pass": 27, "fail": 2, "xfail": 3, "xpass": 0, "blocked": 0, "skipped": 0, "no-commands": 0 }, "judgments": { "F05.restart-hard-running": "blocked: immediate recovery on unreaped zombie owner", "F05.pause-running-crash": "confirmed expected failure after documented lease adaptation" }, "supplemental_replay": "/workspace/verification-artifacts/family-results/F05-lease-adapted-repro.json", "judged_feature_counts": { "pass": 26, "xfail": 4, "blocked": 1 } }, { "family": "F06", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "feature_counts": { "pass": 23, "xfail": 5 }, "raw_summary": { "pass": 23, "fail": 0, "xfail": 5, "xpass": 0, "blocked": 0, "skipped": 0, "no-commands": 0 } }, { "family": "F07", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "feature_counts": { "pass": 22, "xfail": 2 }, "raw_summary": { "pass": 23, "fail": 0, "xfail": 2, "xpass": 0, "blocked": 0, "skipped": 0, "no-commands": 0 } } ], "severity_counts": { "high": 2, "medium": 10, "low": 7 }, "date_utc": "2026-10-13", "outcome": "blocked/partial: full F01\u2013F07 primary pass completed; two environment-limited branches retained", "deliberately_not_covered": [ "Immediate dead-owner lease takeover on a host that reaps SIGKILLed children", "F02.default-bind unprivileged network namespace recipe; environment refuses uid_map writes" ], "cleanup": { "audit": "/workspace/verification-artifacts/family-results/F01-F07-final-evidence-cleanup-audit.json", "own_supplemental_runs": 8, "all_ports_closed": true, "evidence_preserved": true, "private_state_preserved_for_reproduction": true }, "primary_feature_counts": { "total": 170, "pass": 147, "confirmed_expected_failure_features": 21, "blocked": 2, "distinct_confirmed_bugs": 19, "note": "Supplemental branch IDs extend existing bugs and are excluded from the 170 primary IDs. Raw primary F05 results remain preserved; its two setup failures are judged using the recorded zombie-owner lease adaptation." } }, { "file": "triage-f08-f14.json", "checkout_sha": "acca181af1f353975965bcf6486e1d0a58f444c0", "families": [ "F08", "F09", "F10", "F11", "F12", "F13", "F14" ], "confirmation_in_progress": false, "counts": { "high": 4, "low": 12, "medium": 16 }, "pending_confirmation": [], "repeat_runs": [ "/workspace/verification-artifacts/runs/20261013-024131-0603-map-f10", "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14", "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14", "/workspace/verification-artifacts/runs/20261013-024737-a894-map-f08", "/workspace/verification-artifacts/runs/20261013-025113-2057-map-f08", "/workspace/verification-artifacts/runs/20261013-025756-a01d-map-f12", "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09", "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12", "/workspace/verification-artifacts/runs/20261013-031029-c530-map-f13", "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13", "/workspace/verification-artifacts/runs/20261013-031634-2620-map-f11", "/workspace/verification-artifacts/runs/20261013-031830-28b2-map-f11" ], "adaptations": "/workspace/verification-artifacts/f08-f14-harness/ADAPTATIONS.md", "isolated_map_check": "/workspace/verification-artifacts/family-results/f08-f14-map-check.json", "models": [ "deepseek/deepseek-flash", "deepseek/deepseek-v4-pro", "ACP sonnet/haiku via https://api.deepseek.com/anthropic" ], "cost_notes": "ACP label pricing may differ from actual DeepSeek provider billing; copied fork metrics must not be double-counted.", "supplemental_results": [ "family-results/F08-siblings.json", "family-results/F14-siblings.json" ], "status": "complete", "coverage_summary": { "distinct_feature_ids": 190, "pass": 155, "confirmed_failure_feature_ids": 35, "pending_feature_ids": [], "confirmed_grouped_bugs": 32 }, "not_run_client_entrypoints": [ { "family": "F08", "entrypoints": [ "RemoteConversation.set_confirmation_policy", "RemoteConversation.set_security_analyzer", "RemoteConversation.update_secrets (including callable and SecretSource serialization)", "RemoteConversation.reject_pending_actions", "RemoteConversation.run as the SDK confirmation-accept entry point", "ConversationClient.setConfirmationPolicy", "ConversationClient.setSecurityAnalyzer", "ConversationClient.updateSecrets", "ConversationClient.respondToConfirmation", "RemoteConversation.setConfirmationPolicy", "RemoteConversation.setSecurityAnalyzer", "RemoteConversation.updateSecrets (including callable conversion to StaticSecret)", "RemoteConversation.sendConfirmationResponse" ], "reason": "F08-confirmation-security-secrets.md:75-97 lists client surfaces and says recipes use direct REST/socket drivers; no Python or TypeScript client program appears in this family.", "notes": "Direct REST/socket observations do not verify the uncalled wrappers." }, { "family": "F09", "entrypoints": [ "RemoteConversation.switchProfile", "RemoteConversation.switchLlm", "RemoteConversation.switchAcpModel" ], "reason": "F09-model-switching-plugins.md:80-86 and dynamic program at 553-596 name only ConversationClient and ConversationManager. Python plugin program is at 726-744.", "notes": "Direct REST/socket observations do not verify the uncalled wrappers." }, { "family": "F10", "entrypoints": [ "RemoteConversation.fork", "RemoteConversation.navigate_to (including cached-state refresh)", "RemoteConversation.condense", "RemoteConversation.ask_agent", "ConversationClient.forkConversation", "ConversationClient.navigateConversation", "ConversationClient.condenseConversation", "ConversationClient.askAgent", "RemoteConversation.fork", "RemoteConversation.navigateTo", "RemoteConversation.condense", "RemoteConversation.askAgent" ], "reason": "F10-fork-navigate-condense-ask.md:71-85 explicitly states every recipe drives REST directly.", "notes": "Direct REST/socket observations do not verify the uncalled wrappers." }, { "family": "F11", "entrypoints": [ "run_goal standalone synchronous driver", "ConversationClient.startGoal", "ConversationClient.stopGoal", "ConversationClient.resumeGoal", "RemoteConversation.startGoal", "RemoteConversation.stopGoal", "RemoteConversation.resumeGoal" ], "reason": "F11-goals.md:87-107 documents TypeScript goal wrappers and the separate synchronous Python run_goal driver; recipes drive REST and sockets directly.", "notes": "Direct REST/socket observations do not verify the uncalled wrappers." }, { "family": "F12", "entrypoints": [ "AsyncRemoteWorkspace.get_runtime_session_key", "AsyncRemoteWorkspace.release_runtime" ], "reason": "F12-runtime-and-credentials.md:109-115 documents local TypeScript runtime methods and Python Docker lifecycle methods; the Node program at 243-266 checks local available state only.", "notes": "The named covered-elsewhere methods were driven in the F33 Docker SDK supplement or F34 exact-byte positive control. The remaining listed methods were not independently driven; route coverage does not prove every wrapper.", "covered_elsewhere_in_this_audit": [ "RemoteWorkspace.get_runtime_session_key", "RemoteWorkspace.release_runtime" ] }, { "family": "F13", "entrypoints": [ "RemoteWorkspace.start_command as a standalone entry point", "RemoteWorkspace.get_command_output as a standalone entry point", "RemoteWorkspace.stop_command", "AsyncRemoteWorkspace.execute_command", "AsyncRemoteWorkspace.start_command", "AsyncRemoteWorkspace.get_command_output", "AsyncRemoteWorkspace.stop_command", "BashClient.searchEvents", "BashClient.getEvent", "BashClient.getEvents", "BashClient.batchGetEvents", "BashClient.startCommand", "BashClient.executeCommand", "BashClient.clearEvents", "BashClient.stopCommand", "BashWebSocketClient (including query-key auth, replay, and reconnect)", "RemoteWorkspace.executeCommand" ], "reason": "F13-bash.md:78-98 documents all wrapper surfaces but describes only two sync execute_command SDK programs plus raw REST/WebSocket drivers.", "notes": "Direct REST/socket observations do not verify the uncalled wrappers." }, { "family": "F14", "entrypoints": [ "AsyncRemoteWorkspace.file_upload (global and scoped)", "AsyncRemoteWorkspace.file_download (global and scoped)", "FileClient.uploadFile", "FileClient.uploadTextFile", "FileClient.downloadTextFile", "FileClient.downloadTrajectory", "FileClient.getHome", "FileClient.searchSubdirectories", "RemoteConversation.downloadTrajectory", "RemoteWorkspace.fileDownload", "RemoteWorkspace.fileUpload string/File input branches", "write_file tool consuming fileUpload.file_size" ], "reason": "F14-files.md:77-89 documents wrapper surfaces; upload-size at 210-261 exercises only TypeScript global Blob upload, and sdk-workspace at 776-809 exercises sync Python transfers in both scopes.", "notes": "The named covered-elsewhere methods were driven in the F33 Docker SDK supplement or F34 exact-byte positive control. The remaining listed methods were not independently driven; route coverage does not prove every wrapper.", "covered_elsewhere_in_this_audit": [ "FileClient.downloadFile" ] } ], "client_evidence_inventory": "/workspace/verification-artifacts/f08-f14-client-not-run-audit.json", "evidence_audit": "/workspace/verification-artifacts/f08-f14-evidence-audit.json", "docker_complete": { "run": "/workspace/verification-artifacts/runs/20261013-040101-d29e-map-f12", "result": "pass", "image": "qa-agent-server:acca181a-local-uid", "source_sha": "acca181af1f353975965bcf6486e1d0a58f444c0", "scope": "Local frozen-source/locked-host-venv runtime; canonical packaging remains blocked.", "artifact": "family-results/F12-docker-complete.json" } }, { "file": "triage-f15-f21.json", "checkout_sha": "acca181af1f353975965bcf6486e1d0a58f444c0", "status": "complete", "not_run_client_entrypoints": [ { "family": "F16", "entrypoints": [ "RemoteWorkspace.startWorkspaceSession/deleteWorkspaceSession", "VSCodeClient.getUrl/getStatus" ], "reason": "Map describes the wrappers but executes no TypeScript program for them; REST cookie/status/URL proof does not verify these wrappers." }, { "family": "F19", "entrypoints": [ "MCPClient.testServer/startOAuth/getOAuthStatus/submitOAuthCallback", "SettingsClient.createMcpServer/patchMcpServer/deleteMcpServer" ], "reason": "Map explicitly states these TypeScript calls are not driven; only REST lane and Python get_mcp_config are driven." } ], "verified_reruns": [ { "feature_ids": [ "F19.test-tool-errors" ], "result": "pass twice fresh", "source": "family-results/F19-repro.json" }, { "feature_ids": [ "F16.vscode-enabled-status", "F16.vscode-url-shape", "F16.vscode-url-no-key", "F16.vscode-url-after-init" ], "result": "pass twice fresh", "source": "family-results/F16-repro.json" }, { "feature_ids": [ "F15.commits" ], "result": "pass twice fresh with recipe timestamp/prerequisite corrections", "source": "family-results/F15-history.json" }, { "feature_ids": [ "F15.repo-search-unreachable", "F15.repo-search-timeout" ], "result": "pass twice fresh with owned/longer-lived local fixture and warm doctor", "source": "family-results/F15-supplemental.json" }, { "feature_ids": [ "F15.repo-search-github" ], "result": "Live listing, pagination, and search passed; the 401 branch is separately inconclusive because direct upstream also returns 200 for the fake token.", "source": "family-results/F15-github-live-supplement.json", "adjudication": "family-results/F15-github-live-adjudication.json", "evidence": [ "/workspace/verification-artifacts/runs/20261013-041559-37e6-map-f15/evidence/F15.repo-search-github/page-1.json", "/workspace/verification-artifacts/runs/20261013-041559-37e6-map-f15/evidence/F15.repo-search-github/page-2.json", "/workspace/verification-artifacts/runs/20261013-041559-37e6-map-f15/evidence/map-run/F15-git-01-live-github-listing-f15-repo-search-gith.json", "/workspace/verification-artifacts/runs/20261013-041559-37e6-map-f15/evidence/F15.repo-search-github/bad-token.json", "/workspace/verification-artifacts/runs/20261013-041559-37e6-map-f15/evidence/F15.repo-search-github/direct-negative-control.json" ] } ] }, { "file": "triage-f22-f28.json", "checkout_sha": "acca181af1f353975965bcf6486e1d0a58f444c0", "status": "complete_with_limitations", "family_results": { "F22": "/workspace/verification-artifacts/family-results/F22.json", "F23": "/workspace/verification-artifacts/family-results/F23.json", "F24": "/workspace/verification-artifacts/family-results/F24.json", "F25": "/workspace/verification-artifacts/family-results/F25.json", "F26": "/workspace/verification-artifacts/family-results/F26.json", "F27": "/workspace/verification-artifacts/family-results/F27.json", "F28": "/workspace/verification-artifacts/family-results/F28.json" }, "not_run": [ { "feature_id": "F24.device-start-offline", "branch": "Offline poll assertion after the first failing device/start assertion" }, { "feature_id": "F25.install-documented-sources", "branch": "The second advertised /tree/main URL variant after the first failing github shorthand" }, { "feature_id": "F26.conversation-resume-disabled", "branch": "include_skills GET after the failing live socket assertion" }, { "feature_id": "F28.prepare-corrupt-archive", "branch": "Later no-write assertions after the invalid-format failure" } ], "coverage": [ { "family": "F22", "primary_run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "raw_summary": { "pass": 19, "fail": 1, "xfail": 5, "xpass": 0, "blocked": 0, "skipped": 0, "no-commands": 0 }, "latest_feature_counts": { "pass": 19, "xfail": 5 }, "feature_total": 24 }, { "family": "F23", "primary_run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "raw_summary": { "pass": 23, "fail": 0, "xfail": 5, "xpass": 0, "blocked": 0, "skipped": 0, "no-commands": 0 }, "latest_feature_counts": { "pass": 22, "xfail": 5 }, "feature_total": 27 }, { "family": "F24", "primary_run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "raw_summary": { "pass": 18, "fail": 0, "xfail": 2, "xpass": 0, "blocked": 1, "skipped": 0, "no-commands": 0 }, "latest_feature_counts": { "pass": 17, "xfail": 2, "blocked": 1 }, "feature_total": 20 }, { "family": "F25", "primary_run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "raw_summary": { "pass": 27, "fail": 0, "xfail": 7, "xpass": 0, "blocked": 0, "skipped": 0, "no-commands": 0 }, "latest_feature_counts": { "pass": 26, "xfail": 7 }, "feature_total": 33 }, { "family": "F26", "primary_run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "raw_summary": { "pass": 22, "fail": 0, "xfail": 4, "xpass": 0, "blocked": 0, "skipped": 0, "no-commands": 0 }, "latest_feature_counts": { "pass": 21, "xfail": 4 }, "feature_total": 25 }, { "family": "F27", "primary_run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "raw_summary": { "pass": 23, "fail": 5, "xfail": 4, "xpass": 0, "blocked": 0, "skipped": 0, "no-commands": 0 }, "latest_feature_counts": { "pass": 26, "xfail": 5 }, "feature_total": 31 }, { "family": "F28", "primary_run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "raw_summary": { "pass": 22, "fail": 4, "xfail": 4, "xpass": 0, "blocked": 0, "skipped": 0, "no-commands": 0 }, "latest_feature_counts": { "pass": 23, "blocked": 2, "xfail": 4 }, "feature_total": 29 } ], "primary_feature_counts": { "pass": 154, "xfail": 32, "blocked": 3, "total": 189 }, "severity_counts": { "high": 2, "low": 13, "medium": 12 }, "grouping": { "shared_store_lock": [ "F22.store-busy-responsive", "F23.store-busy-responsive" ], "blank_provider_keys": [ "F23.conn-create-blank-key", "F23.conn-patch-blank-key" ], "cached_missing_ref": [ "F25.install-unknown-ref", "F28.install-ref-cached" ], "documented_sources": [ "F25.install-documented-sources", "F26.install-documented-source" ], "vanished_source_refresh": [ "F25.refresh-unfetchable", "F26.refresh-unfetchable" ] }, "complete": true, "date_utc": "2026-10-13", "server_version": "1.54.0", "models": [ "deepseek/deepseek-flash", "deepseek/deepseek-v4-pro" ], "proof_index": "/workspace/verification-artifacts/family-results/F22-F28-proof-index.json", "evidence_audit": { "candidate_files_checked": 166, "configured_credential_matches": 0 }, "cleanup": { "audit": "/workspace/verification-artifacts/family-results/F22-F28-cleanup-audit.json", "own_runs_stopped": 19, "all_ports_closed": true, "live_server_processes": 0, "docker_containers_removed": 2, "evidence_preserved": true } }, { "file": "triage-f29-f34.json", "schema_version": 1, "generated_at": "2026-10-13T04:05:26.342129+00:00", "checkout_sha": "acca181af1f353975965bcf6486e1d0a58f444c0", "checkout": "/workspace/agent-sdk-verify", "server_version": "1.54.0", "complete": true, "status": "complete", "coverage": [ { "id": "F29.ingress-advertised", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-01-ingress-advertised-f29-ingress-advertise.json", "observations": [ { "at": "2026-10-13T02:18:00Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-01-ingress-advertised-f29-ingress-advertise.json" } ] }, { "id": "F29.ingress-config-errors", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-02-ingress-not-configured-or-unusable-f29-i.json", "observations": [ { "at": "2026-10-13T02:20:24Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-02-ingress-not-configured-or-unusable-f29-i.json" } ] }, { "id": "F29.session-bootstrap", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-03-session-bootstrap-f29-session-bootstrap.json", "observations": [ { "at": "2026-10-13T02:20:36Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-03-session-bootstrap-f29-session-bootstrap.json" } ] }, { "id": "F29.session-key-required", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-04-session-key-required-f29-session-key-req.json", "observations": [ { "at": "2026-10-13T02:20:50Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-04-session-key-required-f29-session-key-req.json" } ] }, { "id": "F29.session-ingress-host", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-05-ingress-host-f29-session-ingress-host.json", "observations": [ { "at": "2026-10-13T02:20:58Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-05-ingress-host-f29-session-ingress-host.json" } ] }, { "id": "F29.session-forwarded-proto", "family": "F29", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-01-forwarded-proto-f29-session-forwarded-pr.json", "observations": [ { "at": "2026-10-13T02:21:05Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-06-forwarded-proto-f29-session-forwarded-pr.json" }, { "at": "2026-10-13T03:19:10Z", "run": "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29/evidence/map-run/F29-app-backend-bridge-01-forwarded-proto-f29-session-forwarded-pr.json" }, { "at": "2026-10-13T03:20:02Z", "run": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-01-forwarded-proto-f29-session-forwarded-pr.json" } ] }, { "id": "F29.session-origin-rules", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-07-origin-rules-f29-session-origin-rules.json", "observations": [ { "at": "2026-10-13T02:21:17Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-07-origin-rules-f29-session-origin-rules.json" } ] }, { "id": "F29.session-configured-control-origin", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-25-configured-control-origins-f29-session-c.json", "observations": [ { "at": "2026-10-13T02:32:53Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-25-configured-control-origins-f29-session-c.json" } ] }, { "id": "F29.session-backend-not-ready", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-08-backend-not-ready-f29-session-backend-no.json", "observations": [ { "at": "2026-10-13T02:21:27Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-08-backend-not-ready-f29-session-backend-no.json" } ] }, { "id": "F29.proxy-get", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-09-proxied-safe-methods-f29-proxy-get.json", "observations": [ { "at": "2026-10-13T02:21:36Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-09-proxied-safe-methods-f29-proxy-get.json" } ] }, { "id": "F29.proxy-cookie-required", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-10-cookie-required-f29-proxy-cookie-require.json", "observations": [ { "at": "2026-10-13T02:22:00Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-10-cookie-required-f29-proxy-cookie-require.json" } ] }, { "id": "F29.proxy-unsafe-methods", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-11-unsafe-methods-f29-proxy-unsafe-methods.json", "observations": [ { "at": "2026-10-13T02:22:27Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-11-unsafe-methods-f29-proxy-unsafe-methods.json" } ] }, { "id": "F29.proxy-root", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-12-app-root-f29-proxy-root.json", "observations": [ { "at": "2026-10-13T02:22:52Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-12-app-root-f29-proxy-root.json" } ] }, { "id": "F29.proxy-root-path-query", "family": "F29", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-02-root-path-parameter-f29-proxy-root-path-.json", "observations": [ { "at": "2026-10-13T02:23:04Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-13-root-path-parameter-f29-proxy-root-path-.json" }, { "at": "2026-10-13T03:19:11Z", "run": "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29/evidence/map-run/F29-app-backend-bridge-02-root-path-parameter-f29-proxy-root-path-.json" }, { "at": "2026-10-13T03:20:03Z", "run": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-02-root-path-parameter-f29-proxy-root-path-.json" } ] }, { "id": "F29.proxy-credentials-stripped", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-14-credentials-stripped-f29-proxy-credentia.json", "observations": [ { "at": "2026-10-13T02:23:06Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-14-credentials-stripped-f29-proxy-credentia.json" } ] }, { "id": "F29.proxy-request-framing", "family": "F29", "status": "xfail", "disposition": "inconclusive_contract", "run": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-03-request-framing-f29-proxy-request-framin.json", "observations": [ { "at": "2026-10-13T02:23:14Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-15-request-framing-f29-proxy-request-framin.json" }, { "at": "2026-10-13T03:19:12Z", "run": "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29/evidence/map-run/F29-app-backend-bridge-03-request-framing-f29-proxy-request-framin.json" }, { "at": "2026-10-13T03:20:04Z", "run": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-03-request-framing-f29-proxy-request-framin.json" } ] }, { "id": "F29.proxy-response-filter", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-16-response-filter-f29-proxy-response-filte.json", "observations": [ { "at": "2026-10-13T02:23:21Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-16-response-filter-f29-proxy-response-filte.json" } ] }, { "id": "F29.proxy-response-headers", "family": "F29", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-04-one-date-and-server-header-f29-proxy-res.json", "observations": [ { "at": "2026-10-13T02:23:29Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-17-one-date-and-server-header-f29-proxy-res.json" }, { "at": "2026-10-13T03:19:13Z", "run": "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29/evidence/map-run/F29-app-backend-bridge-04-one-date-and-server-header-f29-proxy-res.json" }, { "at": "2026-10-13T03:20:06Z", "run": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-04-one-date-and-server-header-f29-proxy-res.json" } ] }, { "id": "F29.proxy-path-traversal", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-18-path-traversal-f29-proxy-path-traversal.json", "observations": [ { "at": "2026-10-13T02:23:49Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-18-path-traversal-f29-proxy-path-traversal.json" } ] }, { "id": "F29.ws-bridge", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-19-websocket-bridge-f29-ws-bridge.json", "observations": [ { "at": "2026-10-13T02:24:00Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-19-websocket-bridge-f29-ws-bridge.json" } ] }, { "id": "F29.ws-handshake-rejections", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-20-handshake-rejections-f29-ws-handshake-re.json", "observations": [ { "at": "2026-10-13T02:24:15Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-20-handshake-rejections-f29-ws-handshake-re.json" } ] }, { "id": "F29.ws-subprotocol", "family": "F29", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-05-subprotocol-f29-ws-subprotocol-known-bug.json", "observations": [ { "at": "2026-10-13T02:24:29Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-21-subprotocol-f29-ws-subprotocol-known-bug.json" }, { "at": "2026-10-13T03:19:15Z", "run": "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29/evidence/map-run/F29-app-backend-bridge-05-subprotocol-f29-ws-subprotocol-known-bug.json" }, { "at": "2026-10-13T03:20:08Z", "run": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-05-subprotocol-f29-ws-subprotocol-known-bug.json" } ] }, { "id": "F29.session-revoke", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-22-revoke-a-session-f29-session-revoke.json", "observations": [ { "at": "2026-10-13T02:25:21Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-22-revoke-a-session-f29-session-revoke.json" } ] }, { "id": "F29.backend-lifecycle-revokes", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-23-backend-stop-and-disable-revoke-f29-back.json", "observations": [ { "at": "2026-10-13T02:27:21Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-23-backend-stop-and-disable-revoke-f29-back.json" } ] }, { "id": "F29.backend-crash-detected", "family": "F29", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-06-crash-seen-by-the-bridge-f29-backend-cra.json", "observations": [ { "at": "2026-10-13T02:27:51Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-24-crash-seen-by-the-bridge-f29-backend-cra.json" }, { "at": "2026-10-13T03:19:31Z", "run": "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29/evidence/map-run/F29-app-backend-bridge-06-crash-seen-by-the-bridge-f29-backend-cra.json" }, { "at": "2026-10-13T03:20:25Z", "run": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-06-crash-seen-by-the-bridge-f29-backend-cra.json" } ] }, { "id": "F29.trusted-forwarded-headers", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-26-trusted-forwarded-headers-f29-trusted-fo.json", "observations": [ { "at": "2026-10-13T02:36:42Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-26-trusted-forwarded-headers-f29-trusted-fo.json" } ] }, { "id": "F29.sessions-not-persisted", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-27-server-restart-f29-sessions-not-persiste.json", "observations": [ { "at": "2026-10-13T02:38:17Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-27-server-restart-f29-sessions-not-persiste.json" } ] }, { "id": "F29.session-ttl", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-28-session-expiry-f29-session-ttl.json", "observations": [ { "at": "2026-10-13T02:43:27Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-28-session-expiry-f29-session-ttl.json" } ] }, { "id": "F29.uninstall-revokes", "family": "F29", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-29-uninstall-f29-uninstall-revokes.json", "observations": [ { "at": "2026-10-13T02:43:39Z", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-29-uninstall-f29-uninstall-revokes.json" } ] }, { "id": "F30.models-list", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-01-model-list-f30-models-list.json", "observations": [ { "at": "2026-10-13T02:18:21Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-01-model-list-f30-models-list.json" } ] }, { "id": "F30.auth-bearer-or-header", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-02-bearer-or-session-header-f30-auth-bearer.json", "observations": [ { "at": "2026-10-13T02:18:34Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-02-bearer-or-session-header-f30-auth-bearer.json" } ] }, { "id": "F30.auth-open-without-keys", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-03-no-session-keys-f30-auth-open-without-ke.json", "observations": [ { "at": "2026-10-13T02:19:20Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-03-no-session-keys-f30-auth-open-without-ke.json" } ] }, { "id": "F30.chat-basic", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-01-chat-completion-f30-chat-basic.json", "observations": [ { "at": "2026-10-13T02:19:26Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-04-chat-completion-f30-chat-basic.json" }, { "at": "2026-10-13T03:20:54Z", "run": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30/evidence/map-run/F30-openai-gateway-01-chat-completion-f30-chat-basic.json" }, { "at": "2026-10-13T03:27:36Z", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-01-chat-completion-f30-chat-basic.json" } ] }, { "id": "F30.chat-system-suffix", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-05-system-text-goes-to-the-system-prompt-f3.json", "observations": [ { "at": "2026-10-13T02:19:33Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-05-system-text-goes-to-the-system-prompt-f3.json" } ] }, { "id": "F30.chat-latest-user-only", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-06-only-the-latest-user-turn-f30-chat-lates.json", "observations": [ { "at": "2026-10-13T02:19:40Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-06-only-the-latest-user-turn-f30-chat-lates.json" } ] }, { "id": "F30.chat-stream", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-07-streaming-f30-chat-stream.json", "observations": [ { "at": "2026-10-13T02:19:53Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-07-streaming-f30-chat-stream.json" } ] }, { "id": "F30.chat-reuse-header", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-02-continue-with-the-header-f30-chat-reuse-.json", "observations": [ { "at": "2026-10-13T02:20:14Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-08-continue-with-the-header-f30-chat-reuse-.json" }, { "at": "2026-10-13T03:21:02Z", "run": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30/evidence/map-run/F30-openai-gateway-02-continue-with-the-header-f30-chat-reuse-.json" }, { "at": "2026-10-13T03:27:46Z", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-02-continue-with-the-header-f30-chat-reuse-.json" } ] }, { "id": "F30.chat-reuse-after-restart", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-09-continue-after-a-restart-f30-chat-reuse-.json", "observations": [ { "at": "2026-10-13T02:21:28Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-09-continue-after-a-restart-f30-chat-reuse-.json" } ] }, { "id": "F30.chat-header-new-id", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-10-caller-chosen-id-f30-chat-header-new-id.json", "observations": [ { "at": "2026-10-13T02:22:02Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-10-caller-chosen-id-f30-chat-header-new-id.json" } ] }, { "id": "F30.chat-reuse-keeps-llm", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-11-continuing-keeps-the-conversation-s-llm-.json", "observations": [ { "at": "2026-10-13T02:22:30Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-11-continuing-keeps-the-conversation-s-llm-.json" } ] }, { "id": "F30.chat-usage-per-request", "family": "F30", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-03-usage-per-request-f30-chat-usage-per-req.json", "observations": [ { "at": "2026-10-13T02:22:55Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-12-usage-per-request-f30-chat-usage-per-req.json" }, { "at": "2026-10-13T03:21:08Z", "run": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30/evidence/map-run/F30-openai-gateway-03-usage-per-request-f30-chat-usage-per-req.json" }, { "at": "2026-10-13T03:27:53Z", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-03-usage-per-request-f30-chat-usage-per-req.json" } ] }, { "id": "F30.chat-reuse-concurrent", "family": "F30", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-04-overlapping-turns-on-one-conversation-f3.json", "observations": [ { "at": "2026-10-13T02:23:20Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-13-overlapping-turns-on-one-conversation-f3.json" }, { "at": "2026-10-13T03:21:21Z", "run": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30/evidence/map-run/F30-openai-gateway-04-overlapping-turns-on-one-conversation-f3.json" }, { "at": "2026-10-13T03:28:08Z", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-04-overlapping-turns-on-one-conversation-f3.json" } ] }, { "id": "F30.chat-agent-tools", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-14-the-agent-runs-its-own-tools-f30-chat-ag.json", "observations": [ { "at": "2026-10-13T02:23:33Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-14-the-agent-runs-its-own-tools-f30-chat-ag.json" } ] }, { "id": "F30.chat-input-errors", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-15-chat-input-errors-f30-chat-input-errors.json", "observations": [ { "at": "2026-10-13T02:24:07Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-15-chat-input-errors-f30-chat-input-errors.json" } ] }, { "id": "F30.observability-headers", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-16-observability-headers-f30-observability-.json", "observations": [ { "at": "2026-10-13T02:24:43Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-16-observability-headers-f30-observability-.json" } ] }, { "id": "F30.chat-run-error-500", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-17-run-ends-in-error-f30-chat-run-error-500.json", "observations": [ { "at": "2026-10-13T02:25:35Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-17-run-ends-in-error-f30-chat-run-error-500.json" } ] }, { "id": "F30.chat-confirmation-409", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-18-confirmation-stops-a-new-conversation-f3.json", "observations": [ { "at": "2026-10-13T02:26:04Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-18-confirmation-stops-a-new-conversation-f3.json" } ] }, { "id": "F30.chat-reuse-confirmation", "family": "F30", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-05-confirmation-on-a-continued-conversation.json", "observations": [ { "at": "2026-10-13T02:28:29Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-19-confirmation-on-a-continued-conversation.json" }, { "at": "2026-10-13T03:23:25Z", "run": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30/evidence/map-run/F30-openai-gateway-05-confirmation-on-a-continued-conversation.json" }, { "at": "2026-10-13T03:30:12Z", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-05-confirmation-on-a-continued-conversation.json" } ] }, { "id": "F30.chat-run-limit-429", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-20-run-limit-f30-chat-run-limit-429.json", "observations": [ { "at": "2026-10-13T02:32:23Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-20-run-limit-f30-chat-run-limit-429.json" } ] }, { "id": "F30.chat-reuse-run-limit-429", "family": "F30", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-06-run-limit-on-a-continued-conversation-f3.json", "observations": [ { "at": "2026-10-13T02:36:32Z", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-21-run-limit-on-a-continued-conversation-f3.json" }, { "at": "2026-10-13T03:24:15Z", "run": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30/evidence/map-run/F30-openai-gateway-06-run-limit-on-a-continued-conversation-f3.json" }, { "at": "2026-10-13T03:31:09Z", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-06-run-limit-on-a-continued-conversation-f3.json" } ] }, { "id": "F30.chat-timeout-504", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-07-gateway-timeout-f30-chat-timeout-504.json", "observations": [ { "at": "2026-10-13T03:26:20Z", "run": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30/evidence/map-run/F30-openai-gateway-07-gateway-timeout-f30-chat-timeout-504.json" }, { "at": "2026-10-13T03:33:13Z", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-07-gateway-timeout-f30-chat-timeout-504.json" } ] }, { "id": "F30.responses-basic", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-08-responses-f30-responses-basic.json", "observations": [ { "at": "2026-10-13T03:26:22Z", "run": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30/evidence/map-run/F30-openai-gateway-08-responses-f30-responses-basic.json" }, { "at": "2026-10-13T03:33:17Z", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-08-responses-f30-responses-basic.json" } ] }, { "id": "F30.responses-stateless-replay", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-09-stateless-replay-f30-responses-stateless.json", "observations": [ { "at": "2026-10-13T03:26:26Z", "run": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30/evidence/map-run/F30-openai-gateway-09-stateless-replay-f30-responses-stateless.json" }, { "at": "2026-10-13T03:33:21Z", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-09-stateless-replay-f30-responses-stateless.json" } ] }, { "id": "F30.responses-errors", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-10-responses-errors-f30-responses-errors.json", "observations": [ { "at": "2026-10-13T03:26:30Z", "run": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30/evidence/map-run/F30-openai-gateway-10-responses-errors-f30-responses-errors.json" }, { "at": "2026-10-13T03:33:29Z", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-10-responses-errors-f30-responses-errors.json" } ] }, { "id": "F30.openai-sdk", "family": "F30", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-11-official-openai-sdk-f30-openai-sdk.json", "observations": [ { "at": "2026-10-13T03:26:38Z", "run": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30/evidence/map-run/F30-openai-gateway-11-official-openai-sdk-f30-openai-sdk.json" }, { "at": "2026-10-13T03:33:39Z", "run": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-11-official-openai-sdk-f30-openai-sdk.json" } ] }, { "id": "F31.lease-claimed", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31/evidence/map-run/F31-persistence-and-limits-01-lease-claimed-on-load-f31-lease-claimed.json", "observations": [ { "at": "2026-10-13T02:19:06Z", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-01-lease-claimed-on-load-f31-lease-claimed.json" }, { "at": "2026-10-13T03:40:51Z", "run": "/workspace/verification-artifacts/runs/20261013-034040-cc7f-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034040-cc7f-map-f31/evidence/map-run/F31-persistence-and-limits-01-lease-claimed-on-load-f31-lease-claimed.json" }, { "at": "2026-10-13T03:41:55Z", "run": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31/evidence/map-run/F31-persistence-and-limits-01-lease-claimed-on-load-f31-lease-claimed.json" } ] }, { "id": "F31.lease-renewed", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-02-lease-renewed-f31-lease-renewed.json", "observations": [ { "at": "2026-10-13T02:19:17Z", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-02-lease-renewed-f31-lease-renewed.json" } ] }, { "id": "F31.lease-released-graceful", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-03-graceful-release-f31-lease-released-grac.json", "observations": [ { "at": "2026-10-13T02:20:18Z", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-03-graceful-release-f31-lease-released-grac.json" } ] }, { "id": "F31.lease-dead-owner-takeover", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31/evidence/map-run/F31-persistence-and-limits-02-dead-owner-takeover-f31-lease-dead-owner.json", "observations": [ { "at": "2026-10-13T02:21:04Z", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "result": "fail", "evidence": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-04-dead-owner-takeover-f31-lease-dead-owner.json" }, { "at": "2026-10-13T03:41:00Z", "run": "/workspace/verification-artifacts/runs/20261013-034040-cc7f-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034040-cc7f-map-f31/evidence/map-run/F31-persistence-and-limits-02-dead-owner-takeover-f31-lease-dead-owner.json" }, { "at": "2026-10-13T03:42:09Z", "run": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31/evidence/map-run/F31-persistence-and-limits-02-dead-owner-takeover-f31-lease-dead-owner.json" } ] }, { "id": "F31.crash-tool-call", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-05-crash-during-a-tool-call-f31-crash-tool-.json", "observations": [ { "at": "2026-10-13T02:22:25Z", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-05-crash-during-a-tool-call-f31-crash-tool-.json" } ] }, { "id": "F31.state-survives-restarts", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-06-state-survives-restarts-f31-state-surviv.json", "observations": [ { "at": "2026-10-13T02:25:08Z", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-06-state-survives-restarts-f31-state-surviv.json" } ] }, { "id": "F31.no-plaintext-at-rest", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-07-nothing-secret-in-plaintext-f31-no-plain.json", "observations": [ { "at": "2026-10-13T02:25:58Z", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-07-nothing-secret-in-plaintext-f31-no-plain.json" } ] }, { "id": "F31.second-instance-catalog", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31/evidence/map-run/F31-persistence-and-limits-03-second-server-reads-the-catalog-f31-seco.json", "observations": [ { "at": "2026-10-13T02:29:07Z", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-08-second-server-reads-the-catalog-f31-seco.json" }, { "at": "2026-10-13T03:41:22Z", "run": "/workspace/verification-artifacts/runs/20261013-034040-cc7f-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034040-cc7f-map-f31/evidence/map-run/F31-persistence-and-limits-03-second-server-reads-the-catalog-f31-seco.json" }, { "at": "2026-10-13T03:42:38Z", "run": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31/evidence/map-run/F31-persistence-and-limits-03-second-server-reads-the-catalog-f31-seco.json" } ] }, { "id": "F31.second-instance-lease-held", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-09-second-server-refused-while-the-owner-li.json", "observations": [ { "at": "2026-10-13T02:32:18Z", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-09-second-server-refused-while-the-owner-li.json" } ] }, { "id": "F31.second-instance-takeover", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31/evidence/map-run/F31-persistence-and-limits-04-takeover-after-the-owner-dies-f31-second.json", "observations": [ { "at": "2026-10-13T02:36:11Z", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "result": "fail", "evidence": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-10-takeover-after-the-owner-dies-f31-second.json" }, { "at": "2026-10-13T03:41:41Z", "run": "/workspace/verification-artifacts/runs/20261013-034040-cc7f-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034040-cc7f-map-f31/evidence/map-run/F31-persistence-and-limits-04-takeover-after-the-owner-dies-f31-second.json" }, { "at": "2026-10-13T03:42:57Z", "run": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31/evidence/map-run/F31-persistence-and-limits-04-takeover-after-the-owner-dies-f31-second.json" } ] }, { "id": "F31.takeover-keeps-metadata", "family": "F31", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-01-takeover-keeps-newer-metadata-f31-takeov.json", "observations": [ { "at": "2026-10-13T02:45:12Z", "run": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31/evidence/map-run/F31-persistence-and-limits-01-takeover-keeps-newer-metadata-f31-takeov.json" }, { "at": "2026-10-13T03:01:10Z", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-01-takeover-keeps-newer-metadata-f31-takeov.json" } ] }, { "id": "F31.lease-expiry-fencing", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-02-lease-expiry-fences-a-frozen-owner-f31-l.json", "observations": [ { "at": "2026-10-13T02:47:22Z", "run": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31/evidence/map-run/F31-persistence-and-limits-02-lease-expiry-fences-a-frozen-owner-f31-l.json" }, { "at": "2026-10-13T03:02:51Z", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-02-lease-expiry-fences-a-frozen-owner-f31-l.json" } ] }, { "id": "F31.fenced-write-no-trace", "family": "F31", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-03-refused-write-leaves-no-trace-f31-fenced.json", "observations": [ { "at": "2026-10-13T02:47:25Z", "run": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31/evidence/map-run/F31-persistence-and-limits-03-refused-write-leaves-no-trace-f31-fenced.json" }, { "at": "2026-10-13T03:02:58Z", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-03-refused-write-leaves-no-trace-f31-fenced.json" } ] }, { "id": "F31.lease-disabled", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-04-leasing-disabled-f31-lease-disabled.json", "observations": [ { "at": "2026-10-13T02:49:02Z", "run": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31/evidence/map-run/F31-persistence-and-limits-04-leasing-disabled-f31-lease-disabled.json" }, { "at": "2026-10-13T03:04:12Z", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-04-leasing-disabled-f31-lease-disabled.json" } ] }, { "id": "F31.idle-eviction", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-05-idle-eviction-f31-idle-eviction.json", "observations": [ { "at": "2026-10-13T02:50:43Z", "run": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31/evidence/map-run/F31-persistence-and-limits-05-idle-eviction-f31-idle-eviction.json" }, { "at": "2026-10-13T03:05:55Z", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-05-idle-eviction-f31-idle-eviction.json" } ] }, { "id": "F31.eviction-skips-busy", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-06-busy-conversations-are-kept-f31-eviction.json", "observations": [ { "at": "2026-10-13T02:51:39Z", "run": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31/evidence/map-run/F31-persistence-and-limits-06-busy-conversations-are-kept-f31-eviction.json" }, { "at": "2026-10-13T03:06:50Z", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-06-busy-conversations-are-kept-f31-eviction.json" } ] }, { "id": "F31.create-capacity", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-07-create-at-capacity-f31-create-capacity.json", "observations": [ { "at": "2026-10-13T02:52:15Z", "run": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31/evidence/map-run/F31-persistence-and-limits-07-create-at-capacity-f31-create-capacity.json" }, { "at": "2026-10-13T03:07:17Z", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-07-create-at-capacity-f31-create-capacity.json" } ] }, { "id": "F31.config-precedence", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-08-config-file-and-environment-f31-config-p.json", "observations": [ { "at": "2026-10-13T02:53:43Z", "run": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31/evidence/map-run/F31-persistence-and-limits-08-config-file-and-environment-f31-config-p.json" }, { "at": "2026-10-13T03:08:03Z", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-08-config-file-and-environment-f31-config-p.json" } ] }, { "id": "F31.config-null-override", "family": "F31", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-09-null-from-the-environment-f31-config-nul.json", "observations": [ { "at": "2026-10-13T02:55:20Z", "run": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31/evidence/map-run/F31-persistence-and-limits-09-null-from-the-environment-f31-config-nul.json" }, { "at": "2026-10-13T03:09:11Z", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-09-null-from-the-environment-f31-config-nul.json" } ] }, { "id": "F31.config-invalid-fails-fast", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-10-invalid-limits-stop-startup-f31-config-i.json", "observations": [ { "at": "2026-10-13T02:56:01Z", "run": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31", "result": "fail", "evidence": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31/evidence/map-run/F31-persistence-and-limits-10-invalid-limits-stop-startup-f31-config-i.json" }, { "at": "2026-10-13T03:09:50Z", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-10-invalid-limits-stop-startup-f31-config-i.json" } ] }, { "id": "F31.config-silent-env", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-11-unrecognised-booleans-and-choices-start-.json", "observations": [ { "at": "2026-10-13T02:58:51Z", "run": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31/evidence/map-run/F31-persistence-and-limits-11-unrecognised-booleans-and-choices-start-.json" }, { "at": "2026-10-13T03:11:24Z", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-11-unrecognised-booleans-and-choices-start-.json" } ] }, { "id": "F31.tmux-dir-cleanup", "family": "F31", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-031758-c007-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-031758-c007-map-f31/evidence/map-run/F31-persistence-and-limits-01-per-process-tmux-directory-removed-on-st.json", "observations": [ { "at": "2026-10-13T02:59:25Z", "run": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31", "result": "fail", "evidence": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31/evidence/map-run/F31-persistence-and-limits-12-per-process-tmux-directory-removed-on-st.json" }, { "at": "2026-10-13T03:11:48Z", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "result": "fail", "evidence": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-12-per-process-tmux-directory-removed-on-st.json" }, { "at": "2026-10-13T03:17:32Z", "run": "/workspace/verification-artifacts/runs/20261013-031657-8509-map-f31", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031657-8509-map-f31/evidence/map-run/F31-persistence-and-limits-01-per-process-tmux-directory-removed-on-st.json" }, { "at": "2026-10-13T03:18:30Z", "run": "/workspace/verification-artifacts/runs/20261013-031758-c007-map-f31", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031758-c007-map-f31/evidence/map-run/F31-persistence-and-limits-01-per-process-tmux-directory-removed-on-st.json" } ] }, { "id": "F31.storage-retention", "family": "F31", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034826-d9b8-map-f31", "evidence": "/workspace/verification-artifacts/runs/20261013-034826-d9b8-map-f31/evidence/map-run/F31-persistence-and-limits-01-storage-retention-and-disk-budget-f31-st.json", "observations": [ { "at": "2026-10-13T02:59:25Z", "run": "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31", "result": "blocked", "evidence": null }, { "at": "2026-10-13T03:11:48Z", "run": "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "result": "blocked", "evidence": null }, { "at": "2026-10-13T03:54:58Z", "run": "/workspace/verification-artifacts/runs/20261013-034826-d9b8-map-f31", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034826-d9b8-map-f31/evidence/map-run/F31-persistence-and-limits-01-storage-retention-and-disk-budget-f31-st.json" } ] }, { "id": "F32.events-post", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-01-a-run-s-events-reach-the-webhook-f32-eve.json", "observations": [ { "at": "2026-10-13T02:19:04Z", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-01-a-run-s-events-reach-the-webhook-f32-eve.json" } ] }, { "id": "F32.events-session-key", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-02-session-key-and-content-type-on-every-re.json", "observations": [ { "at": "2026-10-13T02:19:05Z", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-02-session-key-and-content-type-on-every-re.json" } ] }, { "id": "F32.events-full-state", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-04-full-state-snapshot-on-every-attach-f32-.json", "observations": [ { "at": "2026-10-13T02:19:57Z", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-04-full-state-snapshot-on-every-attach-f32-.json" } ] }, { "id": "F32.events-no-deltas", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-06-no-streaming-deltas-on-webhooks-f32-even.json", "observations": [ { "at": "2026-10-13T02:20:49Z", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-06-no-streaming-deltas-on-webhooks-f32-even.json" } ] }, { "id": "F32.conversation-created", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-03-conversation-created-f32-conversation-cr.json", "observations": [ { "at": "2026-10-13T02:19:08Z", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-03-conversation-created-f32-conversation-cr.json" } ] }, { "id": "F32.conversation-lifecycle", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-05-conversation-lifecycle-posts-f32-convers.json", "observations": [ { "at": "2026-10-13T02:20:26Z", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-05-conversation-lifecycle-posts-f32-convers.json" } ] }, { "id": "F32.custom-headers", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-01-custom-headers-path-prefix-and-buffer-si.json", "observations": [ { "at": "2026-10-13T02:22:09Z", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-07-custom-headers-path-prefix-and-buffer-si.json" }, { "at": "2026-10-13T03:17:33Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-01-custom-headers-path-prefix-and-buffer-si.json" }, { "at": "2026-10-13T03:25:59Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-01-custom-headers-path-prefix-and-buffer-si.json" } ] }, { "id": "F32.buffer-size", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-01-custom-headers-path-prefix-and-buffer-si.json", "observations": [ { "at": "2026-10-13T02:22:09Z", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-07-custom-headers-path-prefix-and-buffer-si.json" }, { "at": "2026-10-13T03:17:33Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-01-custom-headers-path-prefix-and-buffer-si.json" }, { "at": "2026-10-13T03:25:59Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-01-custom-headers-path-prefix-and-buffer-si.json" } ] }, { "id": "F32.buffer-stranded", "family": "F32", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-02-events-stranded-behind-an-in-flight-batc.json", "observations": [ { "at": "2026-10-13T02:22:28Z", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-08-events-stranded-behind-an-in-flight-batc.json" }, { "at": "2026-10-13T03:17:44Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-02-events-stranded-behind-an-in-flight-batc.json" }, { "at": "2026-10-13T03:26:10Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-02-events-stranded-behind-an-in-flight-batc.json" } ] }, { "id": "F32.flush-delay", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034519-5d31-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-034519-5d31-map-f32/evidence/map-run/F32-webhooks-and-telemetry-01-flush-timer-from-the-first-event-f32-flu.json", "observations": [ { "at": "2026-10-13T02:26:24Z", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "result": "fail", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-10-flush-timer-from-the-first-event-f32-flu.json" }, { "at": "2026-10-13T03:18:06Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "fail", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-03-flush-timer-from-the-first-event-f32-flu.json" }, { "at": "2026-10-13T03:26:32Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "fail", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-03-flush-timer-from-the-first-event-f32-flu.json" }, { "at": "2026-10-13T03:44:58Z", "run": "/workspace/verification-artifacts/runs/20261013-034422-d0ed-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034422-d0ed-map-f32/evidence/map-run/F32-webhooks-and-telemetry-01-flush-timer-from-the-first-event-f32-flu.json" }, { "at": "2026-10-13T03:46:01Z", "run": "/workspace/verification-artifacts/runs/20261013-034519-5d31-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034519-5d31-map-f32/evidence/map-run/F32-webhooks-and-telemetry-01-flush-timer-from-the-first-event-f32-flu.json" } ] }, { "id": "F32.webhook-spec-docs", "family": "F32", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-04-webhook-field-descriptions-f32-webhook-s.json", "observations": [ { "at": "2026-10-13T02:26:40Z", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-11-webhook-field-descriptions-f32-webhook-s.json" }, { "at": "2026-10-13T03:18:08Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-04-webhook-field-descriptions-f32-webhook-s.json" }, { "at": "2026-10-13T03:26:34Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-04-webhook-field-descriptions-f32-webhook-s.json" } ] }, { "id": "F32.flush-on-close", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-09-flush-on-delete-and-on-shutdown-f32-flus.json", "observations": [ { "at": "2026-10-13T02:24:55Z", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-09-flush-on-delete-and-on-shutdown-f32-flus.json" } ] }, { "id": "F32.backpressure", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-12-batch-and-queue-limits-f32-backpressure.json", "observations": [ { "at": "2026-10-13T02:28:31Z", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-12-batch-and-queue-limits-f32-backpressure.json" } ] }, { "id": "F32.retry", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-13-retries-against-a-failing-receiver-f32-r.json", "observations": [ { "at": "2026-10-13T02:30:42Z", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-13-retries-against-a-failing-receiver-f32-r.json" } ] }, { "id": "F32.requeue", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-14-failed-batches-are-requeued-f32-requeue.json", "observations": [ { "at": "2026-10-13T02:32:58Z", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-14-failed-batches-are-requeued-f32-requeue.json" } ] }, { "id": "F32.slow-receiver", "family": "F32", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-05-a-slow-receiver-blocks-the-next-run-f32-.json", "observations": [ { "at": "2026-10-13T02:35:44Z", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-15-a-slow-receiver-blocks-the-next-run-f32-.json" }, { "at": "2026-10-13T03:18:36Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-05-a-slow-receiver-blocks-the-next-run-f32-.json" }, { "at": "2026-10-13T03:27:02Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-05-a-slow-receiver-blocks-the-next-run-f32-.json" } ] }, { "id": "F32.env-config", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-16-webhooks-from-the-environment-f32-env-co.json", "observations": [ { "at": "2026-10-13T02:37:01Z", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-16-webhooks-from-the-environment-f32-env-co.json" } ] }, { "id": "F32.telemetry-off-by-default", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-06-telemetry-is-off-by-default-f32-telemetr.json", "observations": [ { "at": "2026-10-13T02:38:19Z", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "result": "fail", "evidence": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-17-telemetry-is-off-by-default-f32-telemetr.json" }, { "at": "2026-10-13T03:19:42Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-06-telemetry-is-off-by-default-f32-telemetr.json" }, { "at": "2026-10-13T03:28:21Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-06-telemetry-is-off-by-default-f32-telemetr.json" } ] }, { "id": "F32.telemetry-lifecycle", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-07-server-lifecycle-events-f32-telemetry-li.json", "observations": [ { "at": "2026-10-13T03:20:30Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-07-server-lifecycle-events-f32-telemetry-li.json" }, { "at": "2026-10-13T03:29:13Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-07-server-lifecycle-events-f32-telemetry-li.json" } ] }, { "id": "F32.telemetry-conversation", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-08-conversation-telemetry-is-sanitized-f32-.json", "observations": [ { "at": "2026-10-13T03:20:36Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-08-conversation-telemetry-is-sanitized-f32-.json" }, { "at": "2026-10-13T03:29:20Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-08-conversation-telemetry-is-sanitized-f32-.json" } ] }, { "id": "F32.telemetry-request-failed", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-09-unhandled-errors-f32-telemetry-request-f.json", "observations": [ { "at": "2026-10-13T03:20:39Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-09-unhandled-errors-f32-telemetry-request-f.json" }, { "at": "2026-10-13T03:29:25Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-09-unhandled-errors-f32-telemetry-request-f.json" } ] }, { "id": "F32.telemetry-conversation-failed", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-11-failed-runs-f32-telemetry-conversation-f.json", "observations": [ { "at": "2026-10-13T03:20:50Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-11-failed-runs-f32-telemetry-conversation-f.json" }, { "at": "2026-10-13T03:29:36Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-11-failed-runs-f32-telemetry-conversation-f.json" } ] }, { "id": "F32.telemetry-usage-buckets", "family": "F32", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-10-token-and-cost-buckets-from-real-usage-f.json", "observations": [ { "at": "2026-10-13T03:20:40Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-10-token-and-cost-buckets-from-real-usage-f.json" }, { "at": "2026-10-13T03:29:26Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-10-token-and-cost-buckets-from-real-usage-f.json" } ] }, { "id": "F32.telemetry-consent-settings", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-12-consent-from-settings-f32-telemetry-cons.json", "observations": [ { "at": "2026-10-13T03:21:40Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-12-consent-from-settings-f32-telemetry-cons.json" }, { "at": "2026-10-13T03:30:31Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-12-consent-from-settings-f32-telemetry-cons.json" } ] }, { "id": "F32.telemetry-env-override", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-13-environment-override-beats-settings-f32-.json", "observations": [ { "at": "2026-10-13T03:22:30Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-13-environment-override-beats-settings-f32-.json" }, { "at": "2026-10-13T03:31:25Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-13-environment-override-beats-settings-f32-.json" } ] }, { "id": "F32.telemetry-kill-switch", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-14-kill-switch-f32-telemetry-kill-switch.json", "observations": [ { "at": "2026-10-13T03:23:16Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-14-kill-switch-f32-telemetry-kill-switch.json" }, { "at": "2026-10-13T03:32:21Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-14-kill-switch-f32-telemetry-kill-switch.json" } ] }, { "id": "F32.telemetry-salt-and-kind", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-15-pseudonym-key-and-deployment-kind-f32-te.json", "observations": [ { "at": "2026-10-13T03:24:21Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-15-pseudonym-key-and-deployment-kind-f32-te.json" }, { "at": "2026-10-13T03:33:29Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-15-pseudonym-key-and-deployment-kind-f32-te.json" } ] }, { "id": "F32.telemetry-salt-docs", "family": "F32", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-16-salt-description-f32-telemetry-salt-docs.json", "observations": [ { "at": "2026-10-13T03:24:23Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-16-salt-description-f32-telemetry-salt-docs.json" }, { "at": "2026-10-13T03:33:31Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-16-salt-description-f32-telemetry-salt-docs.json" } ] }, { "id": "F32.deferred-init", "family": "F32", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-17-deferred-pod-gets-telemetry-and-webhooks.json", "observations": [ { "at": "2026-10-13T03:24:53Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-17-deferred-pod-gets-telemetry-and-webhooks.json" }, { "at": "2026-10-13T03:34:02Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-17-deferred-pod-gets-telemetry-and-webhooks.json" } ] }, { "id": "F32.deferred-init-flag", "family": "F32", "status": "xfail", "disposition": "inconclusive_contract", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-18-deferred-pods-are-labelled-as-such-f32-d.json", "observations": [ { "at": "2026-10-13T03:24:54Z", "run": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-18-deferred-pods-are-labelled-as-such-f32-d.json" }, { "at": "2026-10-13T03:34:03Z", "run": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-18-deferred-pods-are-labelled-as-such-f32-d.json" } ] }, { "id": "F33.workspace-exec", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-01-commands-server-info-and-liveness-f33-wo.json", "observations": [ { "at": "2026-10-13T02:21:51Z", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-01-commands-server-info-and-liveness-f33-wo.json" } ] }, { "id": "F33.workspace-bad-key", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-02-wrong-session-key-f33-workspace-bad-key.json", "observations": [ { "at": "2026-10-13T02:22:36Z", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-02-wrong-session-key-f33-workspace-bad-key.json" } ] }, { "id": "F33.workspace-background", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-03-background-command-f33-workspace-backgro.json", "observations": [ { "at": "2026-10-13T02:23:20Z", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-03-background-command-f33-workspace-backgro.json" } ] }, { "id": "F33.workspace-files", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-04-file-round-trip-f33-workspace-files.json", "observations": [ { "at": "2026-10-13T02:23:58Z", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-04-file-round-trip-f33-workspace-files.json" } ] }, { "id": "F33.workspace-git", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-05-git-changes-and-diff-f33-workspace-git.json", "observations": [ { "at": "2026-10-13T02:24:54Z", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-05-git-changes-and-diff-f33-workspace-git.json" } ] }, { "id": "F33.workspace-settings", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-06-llm-profiles-secrets-and-mcp-from-the-se.json", "observations": [ { "at": "2026-10-13T02:25:57Z", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-06-llm-profiles-secrets-and-mcp-from-the-se.json" } ] }, { "id": "F33.async-workspace", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-07-async-workspace-f33-async-workspace.json", "observations": [ { "at": "2026-10-13T02:27:02Z", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-07-async-workspace-f33-async-workspace.json" } ] }, { "id": "F33.workspace-runtime-scope", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-11-conversation-scoped-workspace-f33-worksp.json", "observations": [ { "at": "2026-10-13T02:33:21Z", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-11-conversation-scoped-workspace-f33-worksp.json" } ] }, { "id": "F33.workspace-runtime-lifecycle", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-035525-7e07-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-035525-7e07-map-f33/evidence/map-run/F33-sdk-remote-client-02-runtime-key-and-release-f33-workspace-ru.json", "observations": [ { "at": "2026-10-13T02:33:21Z", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "result": "blocked", "evidence": null }, { "at": "2026-10-13T03:57:15Z", "run": "/workspace/verification-artifacts/runs/20261013-035525-7e07-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-035525-7e07-map-f33/evidence/map-run/F33-sdk-remote-client-02-runtime-key-and-release-f33-workspace-ru.json" } ] }, { "id": "F33.conversation-create", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-035525-7e07-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-035525-7e07-map-f33/evidence/map-run/F33-sdk-remote-client-01-create-a-conversation-without-running-it.json", "observations": [ { "at": "2026-10-13T02:28:14Z", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-08-create-a-conversation-without-running-it.json" }, { "at": "2026-10-13T03:55:51Z", "run": "/workspace/verification-artifacts/runs/20261013-035525-7e07-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-035525-7e07-map-f33/evidence/map-run/F33-sdk-remote-client-01-create-a-conversation-without-running-it.json" } ] }, { "id": "F33.conversation-create-request", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-09-create-from-a-request-twice-f33-conversa.json", "observations": [ { "at": "2026-10-13T02:29:30Z", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-09-create-from-a-request-twice-f33-conversa.json" } ] }, { "id": "F33.conversation-create-profile", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-10-create-from-a-saved-agent-profile-f33-co.json", "observations": [ { "at": "2026-10-13T02:31:41Z", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-10-create-from-a-saved-agent-profile-f33-co.json" } ] }, { "id": "F33.conversation-run", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-01-send-and-run-to-completion-f33-conversat.json", "observations": [ { "at": "2026-10-13T02:35:28Z", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-13-send-and-run-to-completion-f33-conversat.json" }, { "at": "2026-10-13T03:35:30Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-01-send-and-run-to-completion-f33-conversat.json" }, { "at": "2026-10-13T03:47:38Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-01-send-and-run-to-completion-f33-conversat.json" } ] }, { "id": "F33.conversation-callbacks", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-01-send-and-run-to-completion-f33-conversat.json", "observations": [ { "at": "2026-10-13T02:35:28Z", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-13-send-and-run-to-completion-f33-conversat.json" }, { "at": "2026-10-13T03:35:30Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-01-send-and-run-to-completion-f33-conversat.json" }, { "at": "2026-10-13T03:47:38Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-01-send-and-run-to-completion-f33-conversat.json" } ] }, { "id": "F33.conversation-stats", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-01-send-and-run-to-completion-f33-conversat.json", "observations": [ { "at": "2026-10-13T02:35:28Z", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-13-send-and-run-to-completion-f33-conversat.json" }, { "at": "2026-10-13T03:35:30Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-01-send-and-run-to-completion-f33-conversat.json" }, { "at": "2026-10-13T03:47:38Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-01-send-and-run-to-completion-f33-conversat.json" } ] }, { "id": "F33.conversation-stop-hook", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-14-a-stop-hook-that-refuses-the-first-stop-.json", "observations": [ { "at": "2026-10-13T02:36:39Z", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-14-a-stop-hook-that-refuses-the-first-stop-.json" } ] }, { "id": "F33.client-tools", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-15-client-defined-tool-f33-client-tools.json", "observations": [ { "at": "2026-10-13T02:38:22Z", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-15-client-defined-tool-f33-client-tools.json" } ] }, { "id": "F33.conversation-streaming", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-02-streamed-tokens-f33-conversation-streami.json", "observations": [ { "at": "2026-10-13T03:35:47Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-02-streamed-tokens-f33-conversation-streami.json" }, { "at": "2026-10-13T03:47:53Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-02-streamed-tokens-f33-conversation-streami.json" } ] }, { "id": "F33.conversation-token-callbacks", "family": "F33", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-03-token-callbacks-on-a-remote-conversation.json", "observations": [ { "at": "2026-10-13T03:35:47Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-03-token-callbacks-on-a-remote-conversation.json" }, { "at": "2026-10-13T03:47:53Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-03-token-callbacks-on-a-remote-conversation.json" } ] }, { "id": "F33.conversation-streaming-state", "family": "F33", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-04-deltas-kept-out-of-state-events-f33-conv.json", "observations": [ { "at": "2026-10-13T03:35:47Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-04-deltas-kept-out-of-state-events-f33-conv.json" }, { "at": "2026-10-13T03:47:54Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-04-deltas-kept-out-of-state-events-f33-conv.json" } ] }, { "id": "F33.completion-logs", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-05-completion-logs-on-the-client-f33-comple.json", "observations": [ { "at": "2026-10-13T03:36:01Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-05-completion-logs-on-the-client-f33-comple.json" }, { "at": "2026-10-13T03:48:11Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-05-completion-logs-on-the-client-f33-comple.json" } ] }, { "id": "F33.conversation-attach", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-06-reattach-from-a-second-program-f33-conve.json", "observations": [ { "at": "2026-10-13T03:36:26Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-06-reattach-from-a-second-program-f33-conve.json" }, { "at": "2026-10-13T03:48:37Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-06-reattach-from-a-second-program-f33-conve.json" } ] }, { "id": "F33.conversation-attach-large", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-07-attach-to-a-long-history-f33-conversatio.json", "observations": [ { "at": "2026-10-13T03:37:05Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-07-attach-to-a-long-history-f33-conversatio.json" }, { "at": "2026-10-13T03:49:21Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-07-attach-to-a-long-history-f33-conversatio.json" } ] }, { "id": "F33.conversation-state-reread", "family": "F33", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-08-read-the-state-twice-f33-conversation-st.json", "observations": [ { "at": "2026-10-13T03:37:33Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-08-read-the-state-twice-f33-conversation-st.json" }, { "at": "2026-10-13T03:49:53Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-08-read-the-state-twice-f33-conversation-st.json" } ] }, { "id": "F33.conversation-ask-title", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-09-ask-generate-and-set-a-title-f33-convers.json", "observations": [ { "at": "2026-10-13T03:37:52Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-09-ask-generate-and-set-a-title-f33-convers.json" }, { "at": "2026-10-13T03:50:13Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-09-ask-generate-and-set-a-title-f33-convers.json" } ] }, { "id": "F33.conversation-fork", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-10-fork-f33-conversation-fork.json", "observations": [ { "at": "2026-10-13T03:38:10Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-10-fork-f33-conversation-fork.json" }, { "at": "2026-10-13T03:50:33Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-10-fork-f33-conversation-fork.json" } ] }, { "id": "F33.conversation-fork-branch", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-11-fork-from-an-event-then-move-head-f33-co.json", "observations": [ { "at": "2026-10-13T03:38:28Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-11-fork-from-an-event-then-move-head-f33-co.json" }, { "at": "2026-10-13T03:51:05Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-11-fork-from-an-event-then-move-head-f33-co.json" } ] }, { "id": "F33.conversation-plugins-tag", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-12-plugin-source-with-inline-credentials-f3.json", "observations": [ { "at": "2026-10-13T03:38:39Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-12-plugin-source-with-inline-credentials-f3.json" }, { "at": "2026-10-13T03:51:16Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-12-plugin-source-with-inline-credentials-f3.json" } ] }, { "id": "F33.conversation-confirmation", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-13-confirm-or-reject-f33-conversation-confi.json", "observations": [ { "at": "2026-10-13T03:38:49Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-13-confirm-or-reject-f33-conversation-confi.json" }, { "at": "2026-10-13T03:51:32Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-13-confirm-or-reject-f33-conversation-confi.json" } ] }, { "id": "F33.conversation-pause", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-14-pause-a-blocking-run-f33-conversation-pa.json", "observations": [ { "at": "2026-10-13T03:39:13Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-14-pause-a-blocking-run-f33-conversation-pa.json" }, { "at": "2026-10-13T03:52:00Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-14-pause-a-blocking-run-f33-conversation-pa.json" } ] }, { "id": "F33.conversation-interrupt", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-15-interrupt-a-blocking-run-f33-conversatio.json", "observations": [ { "at": "2026-10-13T03:39:26Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-15-interrupt-a-blocking-run-f33-conversatio.json" }, { "at": "2026-10-13T03:52:16Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-15-interrupt-a-blocking-run-f33-conversatio.json" } ] }, { "id": "F33.conversation-run-timeout", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-16-time-out-then-wait-again-f33-conversatio.json", "observations": [ { "at": "2026-10-13T03:39:57Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-16-time-out-then-wait-again-f33-conversatio.json" }, { "at": "2026-10-13T03:52:59Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-16-time-out-then-wait-again-f33-conversatio.json" } ] }, { "id": "F33.conversation-run-join-long-tool", "family": "F33", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-17-join-a-run-during-a-long-tool-call-f33-c.json", "observations": [ { "at": "2026-10-13T03:40:52Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-17-join-a-run-during-a-long-tool-call-f33-c.json" }, { "at": "2026-10-13T03:53:55Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-17-join-a-run-during-a-long-tool-call-f33-c.json" } ] }, { "id": "F33.conversation-stuck", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-18-a-stuck-agent-f33-conversation-stuck.json", "observations": [ { "at": "2026-10-13T03:41:04Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-18-a-stuck-agent-f33-conversation-stuck.json" }, { "at": "2026-10-13T03:54:07Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-18-a-stuck-agent-f33-conversation-stuck.json" } ] }, { "id": "F33.conversation-secrets", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-19-secrets-for-the-agent-f33-conversation-s.json", "observations": [ { "at": "2026-10-13T03:42:06Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "fail", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-19-secrets-for-the-agent-f33-conversation-s.json" }, { "at": "2026-10-13T03:47:31Z", "run": "/workspace/verification-artifacts/runs/20261013-034700-c24a-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034700-c24a-map-f33/evidence/map-run/F33-sdk-remote-client-01-secrets-for-the-agent-f33-conversation-s.json" }, { "at": "2026-10-13T03:54:27Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-19-secrets-for-the-agent-f33-conversation-s.json" } ] }, { "id": "F33.conversation-run-error", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-20-provider-error-surfaces-f33-conversation.json", "observations": [ { "at": "2026-10-13T03:42:30Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-20-provider-error-surfaces-f33-conversation.json" }, { "at": "2026-10-13T03:54:44Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-20-provider-error-surfaces-f33-conversation.json" } ] }, { "id": "F33.ws-unavailable", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-21-no-events-socket-f33-ws-unavailable.json", "observations": [ { "at": "2026-10-13T03:43:18Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-21-no-events-socket-f33-ws-unavailable.json" }, { "at": "2026-10-13T03:55:33Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-21-no-events-socket-f33-ws-unavailable.json" } ] }, { "id": "F33.conversation-reconnect", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-22-a-live-conversation-across-a-restart-f33.json", "observations": [ { "at": "2026-10-13T03:44:05Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-22-a-live-conversation-across-a-restart-f33.json" }, { "at": "2026-10-13T03:56:15Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-22-a-live-conversation-across-a-restart-f33.json" } ] }, { "id": "F33.conversation-reconnect-callbacks", "family": "F33", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-23-callbacks-after-a-reconnect-f33-conversa.json", "observations": [ { "at": "2026-10-13T03:44:33Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-23-callbacks-after-a-reconnect-f33-conversa.json" }, { "at": "2026-10-13T03:56:47Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-23-callbacks-after-a-reconnect-f33-conversa.json" } ] }, { "id": "F33.conversation-restart", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-24-reattach-after-a-restart-and-continue-f3.json", "observations": [ { "at": "2026-10-13T03:45:10Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-24-reattach-after-a-restart-and-continue-f3.json" }, { "at": "2026-10-13T03:57:27Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-24-reattach-after-a-restart-and-continue-f3.json" } ] }, { "id": "F33.custom-tool-module", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-25-a-custom-tool-module-f33-custom-tool-mod.json", "observations": [ { "at": "2026-10-13T03:46:07Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-25-a-custom-tool-module-f33-custom-tool-mod.json" }, { "at": "2026-10-13T03:58:18Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-25-a-custom-tool-module-f33-custom-tool-mod.json" } ] }, { "id": "F33.custom-tool-module-unimportable", "family": "F33", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-26-an-agent-tool-the-server-cannot-import-f.json", "observations": [ { "at": "2026-10-13T03:46:18Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-26-an-agent-tool-the-server-cannot-import-f.json" }, { "at": "2026-10-13T03:58:28Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-26-an-agent-tool-the-server-cannot-import-f.json" } ] }, { "id": "F33.conversation-close", "family": "F33", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-27-close-keeps-or-deletes-f33-conversation-.json", "observations": [ { "at": "2026-10-13T03:46:33Z", "run": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-27-close-keeps-or-deletes-f33-conversation-.json" }, { "at": "2026-10-13T03:58:44Z", "run": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-27-close-keeps-or-deletes-f33-conversation-.json" } ] }, { "id": "F34.server-client", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-01-server-status-and-version-gate-f34-serve.json", "observations": [ { "at": "2026-10-13T02:21:45Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-01-server-status-and-version-gate-f34-serve.json" } ] }, { "id": "F34.http-errors", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-02-errors-f34-http-errors.json", "observations": [ { "at": "2026-10-13T02:21:58Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-02-errors-f34-http-errors.json" } ] }, { "id": "F34.workspace-exec", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-03-workspace-commands-f34-workspace-exec.json", "observations": [ { "at": "2026-10-13T02:22:12Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-03-workspace-commands-f34-workspace-exec.json" } ] }, { "id": "F34.workspace-files", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-04-workspace-files-f34-workspace-files.json", "observations": [ { "at": "2026-10-13T02:22:25Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-04-workspace-files-f34-workspace-files.json" } ] }, { "id": "F34.workspace-binary-download", "family": "F34", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-024359-a3ff-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-024359-a3ff-map-f34/evidence/map-run/F34-typescript-client-01-binary-download-through-remoteworkspace-.json", "observations": [ { "at": "2026-10-13T02:22:36Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-05-binary-download-through-remoteworkspace-.json" }, { "at": "2026-10-13T02:43:42Z", "run": "/workspace/verification-artifacts/runs/20261013-024310-2602-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-024310-2602-map-f34/evidence/map-run/F34-typescript-client-01-binary-download-through-remoteworkspace-.json" }, { "at": "2026-10-13T02:44:39Z", "run": "/workspace/verification-artifacts/runs/20261013-024359-a3ff-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-024359-a3ff-map-f34/evidence/map-run/F34-typescript-client-01-binary-download-through-remoteworkspace-.json" } ] }, { "id": "F34.workspace-scoped", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-06-conversation-scoped-workspace-f34-worksp.json", "observations": [ { "at": "2026-10-13T02:22:49Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-06-conversation-scoped-workspace-f34-worksp.json" } ] }, { "id": "F34.workspace-git", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-07-git-through-remoteworkspace-f34-workspac.json", "observations": [ { "at": "2026-10-13T02:23:11Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-07-git-through-remoteworkspace-f34-workspac.json" } ] }, { "id": "F34.workspace-git-types", "family": "F34", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-01-git-status-type-f34-workspace-git-types-.json", "observations": [ { "at": "2026-10-13T02:23:30Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-08-git-status-type-f34-workspace-git-types-.json" }, { "at": "2026-10-13T02:48:54Z", "run": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-01-git-status-type-f34-workspace-git-types-.json" }, { "at": "2026-10-13T02:54:20Z", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-01-git-status-type-f34-workspace-git-types-.json" } ] }, { "id": "F34.bash-client", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-09-bash-client-f34-bash-client.json", "observations": [ { "at": "2026-10-13T02:24:05Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-09-bash-client-f34-bash-client.json" } ] }, { "id": "F34.bash-websocket", "family": "F34", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-02-bash-events-socket-from-node-f34-bash-we.json", "observations": [ { "at": "2026-10-13T02:24:35Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-10-bash-events-socket-from-node-f34-bash-we.json" }, { "at": "2026-10-13T02:49:13Z", "run": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-02-bash-events-socket-from-node-f34-bash-we.json" }, { "at": "2026-10-13T02:54:41Z", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-02-bash-events-socket-from-node-f34-bash-we.json" } ] }, { "id": "F34.workspace-session", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-11-workspace-session-f34-workspace-session.json", "observations": [ { "at": "2026-10-13T02:24:51Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-11-workspace-session-f34-workspace-session.json" } ] }, { "id": "F34.conversation-create", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-12-create-a-conversation-f34-conversation-c.json", "observations": [ { "at": "2026-10-13T02:25:22Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-12-create-a-conversation-f34-conversation-c.json" } ] }, { "id": "F34.conversation-run", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-13-send-and-run-f34-conversation-run.json", "observations": [ { "at": "2026-10-13T02:26:22Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-13-send-and-run-f34-conversation-run.json" } ] }, { "id": "F34.events-list", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-14-events-list-f34-events-list.json", "observations": [ { "at": "2026-10-13T02:26:42Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-14-events-list-f34-events-list.json" } ] }, { "id": "F34.events-unknown-id", "family": "F34", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-03-unknown-event-id-f34-events-unknown-id-k.json", "observations": [ { "at": "2026-10-13T02:27:13Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-15-unknown-event-id-f34-events-unknown-id-k.json" }, { "at": "2026-10-13T02:49:20Z", "run": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-03-unknown-event-id-f34-events-unknown-id-k.json" }, { "at": "2026-10-13T02:54:48Z", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-03-unknown-event-id-f34-events-unknown-id-k.json" } ] }, { "id": "F34.events-search-kind", "family": "F34", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-04-short-event-kinds-f34-events-search-kind.json", "observations": [ { "at": "2026-10-13T02:27:24Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-16-short-event-kinds-f34-events-search-kind.json" }, { "at": "2026-10-13T02:49:23Z", "run": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-04-short-event-kinds-f34-events-search-kind.json" }, { "at": "2026-10-13T02:54:51Z", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-04-short-event-kinds-f34-events-search-kind.json" } ] }, { "id": "F34.event-stream", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-17-event-stream-on-node-s-websocket-f34-eve.json", "observations": [ { "at": "2026-10-13T02:27:49Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-17-event-stream-on-node-s-websocket-f34-eve.json" } ] }, { "id": "F34.ws-callback-client", "family": "F34", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-05-conversation-callback-socket-from-node-f.json", "observations": [ { "at": "2026-10-13T02:28:22Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-18-conversation-callback-socket-from-node-f.json" }, { "at": "2026-10-13T02:49:41Z", "run": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-05-conversation-callback-socket-from-node-f.json" }, { "at": "2026-10-13T02:55:09Z", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-05-conversation-callback-socket-from-node-f.json" } ] }, { "id": "F34.conversation-controls", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-19-title-and-confirmation-policy-f34-conver.json", "observations": [ { "at": "2026-10-13T02:28:50Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-19-title-and-confirmation-policy-f34-conver.json" } ] }, { "id": "F34.conversation-confirm", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-25-confirmation-and-secrets-f34-conversatio.json", "observations": [ { "at": "2026-10-13T02:33:05Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-25-confirmation-and-secrets-f34-conversatio.json" } ] }, { "id": "F34.conversation-branching", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-24-fork-navigate-and-ask-f34-conversation-b.json", "observations": [ { "at": "2026-10-13T02:31:37Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-24-fork-navigate-and-ask-f34-conversation-b.json" } ] }, { "id": "F34.manager", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-20-conversation-manager-f34-manager.json", "observations": [ { "at": "2026-10-13T02:29:26Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-20-conversation-manager-f34-manager.json" } ] }, { "id": "F34.manager-update-result", "family": "F34", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-06-update-result-f34-manager-update-result-.json", "observations": [ { "at": "2026-10-13T02:29:45Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-21-update-result-f34-manager-update-result-.json" }, { "at": "2026-10-13T02:49:48Z", "run": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-06-update-result-f34-manager-update-result-.json" }, { "at": "2026-10-13T02:55:16Z", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-06-update-result-f34-manager-update-result-.json" } ] }, { "id": "F34.manager-create-options", "family": "F34", "status": "xfail", "disposition": "confirmed_bug", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-07-create-options-f34-manager-create-option.json", "observations": [ { "at": "2026-10-13T02:30:16Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-22-create-options-f34-manager-create-option.json" }, { "at": "2026-10-13T02:49:53Z", "run": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-07-create-options-f34-manager-create-option.json" }, { "at": "2026-10-13T02:55:22Z", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "result": "xfail", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-07-create-options-f34-manager-create-option.json" } ] }, { "id": "F34.manager-acp", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-23-acp-conversations-f34-manager-acp.json", "observations": [ { "at": "2026-10-13T02:30:52Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-23-acp-conversations-f34-manager-acp.json" } ] }, { "id": "F34.hooks-client", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-26-hooks-f34-hooks-client.json", "observations": [ { "at": "2026-10-13T02:33:37Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-26-hooks-f34-hooks-client.json" } ] }, { "id": "F34.mcp-client", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-27-mcp-test-f34-mcp-client.json", "observations": [ { "at": "2026-10-13T02:35:06Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-27-mcp-test-f34-mcp-client.json" } ] }, { "id": "F34.restart-reload", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-28-restart-f34-restart-reload.json", "observations": [ { "at": "2026-10-13T02:36:42Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-28-restart-f34-restart-reload.json" } ] }, { "id": "F34.event-stream-reconnect", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-030612-9862-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-030612-9862-map-f34/evidence/map-run/F34-typescript-client-01-event-stream-reconnect-f34-event-stream-.json", "observations": [ { "at": "2026-10-13T02:38:11Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "fail", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-29-event-stream-reconnect-f34-event-stream-.json" }, { "at": "2026-10-13T02:50:40Z", "run": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-08-event-stream-reconnect-f34-event-stream-.json" }, { "at": "2026-10-13T02:56:29Z", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "result": "fail", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-08-event-stream-reconnect-f34-event-stream-.json" }, { "at": "2026-10-13T03:05:15Z", "run": "/workspace/verification-artifacts/runs/20261013-030339-abcc-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-030339-abcc-map-f34/evidence/map-run/F34-typescript-client-01-event-stream-reconnect-f34-event-stream-.json" }, { "at": "2026-10-13T03:07:17Z", "run": "/workspace/verification-artifacts/runs/20261013-030612-9862-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-030612-9862-map-f34/evidence/map-run/F34-typescript-client-01-event-stream-reconnect-f34-event-stream-.json" } ] }, { "id": "F34.event-stream-reconnect-native", "family": "F34", "status": "xpass", "disposition": "pass_runtime_map_drift", "run": "/workspace/verification-artifacts/runs/20261013-030612-9862-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-030612-9862-map-f34/evidence/map-run/F34-typescript-client-02-reconnect-on-node-s-own-websocket-f34-ev.json", "observations": [ { "at": "2026-10-13T02:38:20Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "fail", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-30-reconnect-on-node-s-own-websocket-f34-ev.json" }, { "at": "2026-10-13T02:51:49Z", "run": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "result": "fail", "evidence": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-09-reconnect-on-node-s-own-websocket-f34-ev.json" }, { "at": "2026-10-13T02:57:15Z", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "result": "xpass", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-09-reconnect-on-node-s-own-websocket-f34-ev.json" }, { "at": "2026-10-13T03:05:56Z", "run": "/workspace/verification-artifacts/runs/20261013-030339-abcc-map-f34", "result": "xpass", "evidence": "/workspace/verification-artifacts/runs/20261013-030339-abcc-map-f34/evidence/map-run/F34-typescript-client-02-reconnect-on-node-s-own-websocket-f34-ev.json" }, { "at": "2026-10-13T03:07:55Z", "run": "/workspace/verification-artifacts/runs/20261013-030612-9862-map-f34", "result": "xpass", "evidence": "/workspace/verification-artifacts/runs/20261013-030612-9862-map-f34/evidence/map-run/F34-typescript-client-02-reconnect-on-node-s-own-websocket-f34-ev.json" } ] }, { "id": "F34.endpoint-audit", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-10-endpoint-audit-against-this-checkout-f34.json", "observations": [ { "at": "2026-10-13T02:38:20Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "fail", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-31-endpoint-audit-against-this-checkout-f34.json" }, { "at": "2026-10-13T02:52:15Z", "run": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-10-endpoint-audit-against-this-checkout-f34.json" }, { "at": "2026-10-13T02:57:53Z", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-10-endpoint-audit-against-this-checkout-f34.json" } ] }, { "id": "F34.version-gate-old", "family": "F34", "status": "pass", "disposition": "pass", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-11-old-server-version-f34-version-gate-old.json", "observations": [ { "at": "2026-10-13T02:38:20Z", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "result": "fail", "evidence": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-32-old-server-version-f34-version-gate-old.json" }, { "at": "2026-10-13T02:53:18Z", "run": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-11-old-server-version-f34-version-gate-old.json" }, { "at": "2026-10-13T02:58:24Z", "run": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "result": "pass", "evidence": "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-11-old-server-version-f34-version-gate-old.json" } ] } ], "counts": { "active_features": 183, "feature_statuses": { "pass": 148, "xfail": 34, "xpass": 1 }, "adjudicated_dispositions": { "pass": 148, "confirmed_bug": 32, "inconclusive_contract": 2, "pass_runtime_map_drift": 1 }, "confirmed_bug_groups": 31, "confirmed_bug_feature_ids": 32, "severity": { "low": 11, "medium": 19, "high": 1 } }, "reports": [ "/workspace/verification-artifacts/family-results/F29-F34-inventory.json", "/workspace/verification-artifacts/family-results/F29-repro.json", "/workspace/verification-artifacts/family-results/F29.json", "/workspace/verification-artifacts/family-results/F30-supplement-repro.json", "/workspace/verification-artifacts/family-results/F30.json", "/workspace/verification-artifacts/family-results/F31-storage-docker-supplement.json", "/workspace/verification-artifacts/family-results/F31-supplement-repro.json", "/workspace/verification-artifacts/family-results/F31-takeover-supervised-repro.json", "/workspace/verification-artifacts/family-results/F31-tmux-adapted-repro.json", "/workspace/verification-artifacts/family-results/F31.json", "/workspace/verification-artifacts/family-results/F32-supplement-repro.json", "/workspace/verification-artifacts/family-results/F32-timer-corrected-repro.json", "/workspace/verification-artifacts/family-results/F32.json", "/workspace/verification-artifacts/family-results/F33-runtime-docker-supplement.json", "/workspace/verification-artifacts/family-results/F33-secrets-synthetic-prompt.json", "/workspace/verification-artifacts/family-results/F33-supplement-repro.json", "/workspace/verification-artifacts/family-results/F33.json", "/workspace/verification-artifacts/family-results/F34-binary-repro-retry.json", "/workspace/verification-artifacts/family-results/F34-reconnect-adapted-repro.json", "/workspace/verification-artifacts/family-results/F34-supplement-repro.json", "/workspace/verification-artifacts/family-results/F34.json" ], "own_runs": [ "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "/workspace/verification-artifacts/runs/20261013-024310-2602-map-f34", "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31", "/workspace/verification-artifacts/runs/20261013-024359-a3ff-map-f34", "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34", "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31", "/workspace/verification-artifacts/runs/20261013-030339-abcc-map-f34", "/workspace/verification-artifacts/runs/20261013-030612-9862-map-f34", "/workspace/verification-artifacts/runs/20261013-031657-8509-map-f31", "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "/workspace/verification-artifacts/runs/20261013-031758-c007-map-f31", "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29", "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29", "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30", "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32", "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30", "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "/workspace/verification-artifacts/runs/20261013-034040-cc7f-map-f31", "/workspace/verification-artifacts/runs/20261013-034142-b86b-map-f31", "/workspace/verification-artifacts/runs/20261013-034422-d0ed-map-f32", "/workspace/verification-artifacts/runs/20261013-034519-5d31-map-f32", "/workspace/verification-artifacts/runs/20261013-034700-c24a-map-f33", "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33", "/workspace/verification-artifacts/runs/20261013-034826-d9b8-map-f31", "/workspace/verification-artifacts/runs/20261013-035525-7e07-map-f33" ], "models_and_recorded_cost": { "models": { "deepseek/deepseek-flash": { "cost_usd": 0.08294667599999998, "prompt_tokens": 715991, "completion_tokens": 25192, "cache_read_tokens": 551296, "metric_conversations": 124, "recorded_unique_cost_entries": 119, "recorded_unique_token_entries": 119 }, "openai/qa-stub": { "cost_usd": 0.0, "prompt_tokens": 359, "completion_tokens": 181, "cache_read_tokens": 0, "metric_conversations": 39, "recorded_unique_cost_entries": 0, "recorded_unique_token_entries": 38 }, "openai/qa-placeholder": { "cost_usd": 0.0, "prompt_tokens": 0, "completion_tokens": 0, "cache_read_tokens": 0, "metric_conversations": 86, "recorded_unique_cost_entries": 0, "recorded_unique_token_entries": 0 } }, "note": "Partial persisted provider accounting, deduplicates per-response cost timestamps and token response IDs copied into forks; excludes deleted conversations, autotitles and provider validation calls; not provider invoice." }, "product_source_changes": false, "final_doctor": { "ok": true, "evidence": "/workspace/verification-artifacts/family-results/F33-doctor-after-recovery.json", "prior_attempt_raced_an_intentional_recipe_restart": "/workspace/verification-artifacts/family-results/F33-doctor-raced-recipe-restart.json" }, "own_process_cleanup": "/workspace/verification-artifacts/F29-F34-own-process-cleanup.json", "extra_archived_proxy_checks": "/workspace/verification-artifacts/F31-archived-proxy-routes.json", "adaptation_patch": "/workspace/verification-artifacts/F29-F34-all-recipe-adaptations.patch" } ], "source_group_count": 133, "aggregate_merges": [ { "source": "F34.events-unknown-id", "target": "F06.unknown-event", "reason": "The TypeScript lookup drives the same EventLog.get_index missing-ID failure as the REST consumer." }, { "source": "F34.events-search-kind", "target": "F06.search-kind-short", "reason": "The TypeScript search drives the same server short-kind versus module.ClassName comparison." }, { "source": "F04.create-secrets-get", "target": "F08.secret-runtime-persistence", "reason": "Constructor and runtime secret updates mutate LocalConversation.update_secrets without triggering a state save." } ], "severity_counts": { "medium": 69, "low": 49, "high": 12 }, "families": [ { "family": "F01", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "ok": true, "counts": { "pass": 11 }, "latest_counts": { "pass": 11 } }, { "family": "F02", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "ok": false, "counts": { "pass": 18, "fail": 1, "xfail": 2 }, "latest_counts": { "pass": 18, "blocked": 1, "xfail": 2 } }, { "family": "F03", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "ok": true, "counts": { "pass": 22, "xfail": 1 }, "latest_counts": { "pass": 22, "xfail": 1, "retired": 1 } }, { "family": "F04", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "ok": true, "counts": { "pass": 25, "xfail": 7 }, "latest_counts": { "pass": 25, "xfail": 7 } }, { "family": "F05", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "ok": false, "counts": { "pass": 26, "xfail": 3, "fail": 2 }, "latest_counts": { "pass": 27, "xfail": 4 } }, { "family": "F06", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "ok": true, "counts": { "pass": 23, "xfail": 5 }, "latest_counts": { "pass": 23, "xfail": 5 } }, { "family": "F07", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "ok": true, "counts": { "pass": 22, "xfail": 2 }, "latest_counts": { "pass": 22, "xfail": 2 } }, { "family": "F08", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "ok": false, "counts": { "pass": 19, "xfail": 3, "fail": 1 }, "latest_counts": { "pass": 20, "xfail": 3 } }, { "family": "F09", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "ok": false, "counts": { "pass": 18, "xfail": 8, "fail": 2 }, "latest_counts": { "pass": 19, "xfail": 9 } }, { "family": "F10", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "ok": false, "counts": { "pass": 18, "xfail": 6, "fail": 1 }, "latest_counts": { "pass": 19, "xfail": 6 } }, { "family": "F11", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "ok": false, "counts": { "pass": 22, "fail": 1, "xfail": 2 }, "latest_counts": { "pass": 23, "xfail": 2 } }, { "family": "F12", "run": null, "ok": false, "primary_incomplete": true, "counts": {}, "latest_counts": { "pass": 27, "xfail": 3 } }, { "family": "F13", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "ok": true, "counts": { "pass": 24, "xfail": 5 }, "latest_counts": { "pass": 24, "xfail": 5 } }, { "family": "F14", "run": null, "ok": false, "primary_incomplete": true, "counts": {}, "latest_counts": { "pass": 23, "xfail": 7 } }, { "family": "F15", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "ok": false, "counts": { "pass": 18, "xfail": 5, "fail": 4, "blocked": 1 }, "latest_counts": { "pass": 23, "xfail": 5 } }, { "family": "F16", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "ok": false, "counts": { "pass": 18, "fail": 4, "xfail": 3, "blocked": 1 }, "latest_counts": { "pass": 22, "xfail": 3, "blocked": 1 } }, { "family": "F17", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "ok": false, "counts": { "pass": 21, "xfail": 2, "fail": 1 }, "latest_counts": { "pass": 21, "xfail": 2, "fail": 1 } }, { "family": "F18", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "ok": true, "counts": { "pass": 26, "xfail": 6 }, "latest_counts": { "pass": 25, "xfail": 4 } }, { "family": "F19", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "ok": false, "counts": { "pass": 24, "xfail": 3, "fail": 1 }, "latest_counts": { "pass": 25, "xfail": 3 } }, { "family": "F20", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "ok": true, "counts": { "pass": 29, "xfail": 6 }, "latest_counts": { "pass": 29, "xfail": 6 } }, { "family": "F21", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "ok": true, "counts": { "pass": 28, "xfail": 4 }, "latest_counts": { "pass": 28, "xfail": 4 } }, { "family": "F22", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "ok": false, "counts": { "pass": 18, "xfail": 5, "fail": 1 }, "latest_counts": { "pass": 19, "xfail": 5 } }, { "family": "F23", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "ok": true, "counts": { "pass": 22, "xfail": 5 }, "latest_counts": { "pass": 22, "xfail": 5 } }, { "family": "F24", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "ok": true, "counts": { "pass": 17, "xfail": 2, "blocked": 1 }, "latest_counts": { "pass": 17, "xfail": 2, "blocked": 1 } }, { "family": "F25", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "ok": true, "counts": { "pass": 26, "xfail": 7 }, "latest_counts": { "pass": 26, "xfail": 7 } }, { "family": "F26", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "ok": true, "counts": { "pass": 21, "xfail": 4 }, "latest_counts": { "pass": 21, "xfail": 4 } }, { "family": "F27", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "ok": false, "counts": { "pass": 22, "xfail": 4, "fail": 5 }, "latest_counts": { "pass": 26, "xfail": 5 } }, { "family": "F28", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "ok": false, "counts": { "pass": 21, "fail": 4, "xfail": 4 }, "latest_counts": { "pass": 23, "fail": 2, "xfail": 4 } }, { "family": "F29", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "ok": true, "counts": { "pass": 23, "xfail": 6 }, "latest_counts": { "pass": 23, "xfail": 6 } }, { "family": "F30", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "ok": false, "counts": { "pass": 17, "xfail": 4 }, "latest_counts": { "pass": 22, "xfail": 4 } }, { "family": "F31", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "ok": false, "counts": { "pass": 8, "fail": 2 }, "latest_counts": { "pass": 19, "xfail": 4 } }, { "family": "F32", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "ok": false, "counts": { "pass": 13, "xfail": 3, "fail": 2 }, "latest_counts": { "pass": 24, "xfail": 6 } }, { "family": "F33", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "ok": false, "counts": { "pass": 16 }, "latest_counts": { "pass": 37, "xfail": 6 } }, { "family": "F34", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "ok": false, "counts": { "pass": 20, "xfail": 8, "fail": 4 }, "latest_counts": { "pass": 23, "xfail": 8, "xpass": 1 } } ], "primary_rows": [ { "feature_id": "F01.alive", "family": "F01", "result": "pass", "label": "Liveness (`F01.alive`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-01-liveness-f01-alive.json", "failed_command": null, "variants": [ { "feature_id": "F01.alive", "family": "F01", "result": "pass", "label": "Liveness (`F01.alive`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-01-liveness-f01-alive.json", "failed_command": null } ] }, { "feature_id": "F01.debug-tracebacks", "family": "F01", "result": "pass", "label": "DEBUG tracebacks (`F01.debug-tracebacks`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-10-debug-tracebacks-f01-debug-tracebacks.json", "failed_command": null, "variants": [ { "feature_id": "F01.debug-tracebacks", "family": "F01", "result": "pass", "label": "DEBUG tracebacks (`F01.debug-tracebacks`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-10-debug-tracebacks-f01-debug-tracebacks.json", "failed_command": null } ] }, { "feature_id": "F01.health", "family": "F01", "result": "pass", "label": "Health (`F01.health`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-02-health-f01-health.json", "failed_command": null, "variants": [ { "feature_id": "F01.health", "family": "F01", "result": "pass", "label": "Health (`F01.health`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-02-health-f01-health.json", "failed_command": null } ] }, { "feature_id": "F01.idle-time", "family": "F01", "result": "pass", "label": "Idle time (`F01.idle-time`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-07-idle-time-f01-idle-time.json", "failed_command": null, "variants": [ { "feature_id": "F01.idle-time", "family": "F01", "result": "pass", "label": "Idle time (`F01.idle-time`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-07-idle-time-f01-idle-time.json", "failed_command": null } ] }, { "feature_id": "F01.openapi", "family": "F01", "result": "pass", "label": "OpenAPI document (`F01.openapi`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-08-openapi-document-f01-openapi.json", "failed_command": null, "variants": [ { "feature_id": "F01.openapi", "family": "F01", "result": "pass", "label": "OpenAPI document (`F01.openapi`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-08-openapi-document-f01-openapi.json", "failed_command": null } ] }, { "feature_id": "F01.preload-tools", "family": "F01", "result": "pass", "label": "Tool preload (`F01.preload-tools`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-11-tool-preload-f01-preload-tools.json", "failed_command": null, "variants": [ { "feature_id": "F01.preload-tools", "family": "F01", "result": "pass", "label": "Tool preload (`F01.preload-tools`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-11-tool-preload-f01-preload-tools.json", "failed_command": null } ] }, { "feature_id": "F01.ready", "family": "F01", "result": "pass", "label": "Readiness (`F01.ready`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-03-readiness-f01-ready.json", "failed_command": null, "variants": [ { "feature_id": "F01.ready", "family": "F01", "result": "pass", "label": "Readiness (`F01.ready`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-03-readiness-f01-ready.json", "failed_command": null } ] }, { "feature_id": "F01.root-info", "family": "F01", "result": "pass", "label": "Root alias (`F01.root-info`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-05-root-alias-f01-root-info.json", "failed_command": null, "variants": [ { "feature_id": "F01.root-info", "family": "F01", "result": "pass", "label": "Root alias (`F01.root-info`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-05-root-alias-f01-root-info.json", "failed_command": null } ] }, { "feature_id": "F01.server-info", "family": "F01", "result": "pass", "label": "Server info (`F01.server-info`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-04-server-info-f01-server-info.json", "failed_command": null, "variants": [ { "feature_id": "F01.server-info", "family": "F01", "result": "pass", "label": "Server info (`F01.server-info`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-04-server-info-f01-server-info.json", "failed_command": null } ] }, { "feature_id": "F01.static-files", "family": "F01", "result": "pass", "label": "Static files (`F01.static-files`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-09-static-files-f01-static-files.json", "failed_command": null, "variants": [ { "feature_id": "F01.static-files", "family": "F01", "result": "pass", "label": "Static files (`F01.static-files`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-09-static-files-f01-static-files.json", "failed_command": null } ] }, { "feature_id": "F01.uptime-restart", "family": "F01", "result": "pass", "label": "Uptime across a restart (`F01.uptime-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-06-uptime-across-a-restart-f01-uptime-resta.json", "failed_command": null, "variants": [ { "feature_id": "F01.uptime-restart", "family": "F01", "result": "pass", "label": "Uptime across a restart (`F01.uptime-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-06-uptime-across-a-restart-f01-uptime-resta.json", "failed_command": null } ] }, { "feature_id": "F02.cors-configured-origins", "family": "F02", "result": "pass", "label": "Configured CORS origins (`F02.cors-configured-origins`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-15-configured-cors-origins-f02-cors-configu.json", "failed_command": null, "variants": [ { "feature_id": "F02.cors-configured-origins", "family": "F02", "result": "pass", "label": "Configured CORS origins (`F02.cors-configured-origins`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-15-configured-cors-origins-f02-cors-configu.json", "failed_command": null } ] }, { "feature_id": "F02.cors-default-origins", "family": "F02", "result": "pass", "label": "Default CORS origins (`F02.cors-default-origins`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-14-default-cors-origins-f02-cors-default-or.json", "failed_command": null, "variants": [ { "feature_id": "F02.cors-default-origins", "family": "F02", "result": "pass", "label": "Default CORS origins (`F02.cors-default-origins`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-14-default-cors-origins-f02-cors-default-or.json", "failed_command": null } ] }, { "feature_id": "F02.cors-workspace-routes", "family": "F02", "result": "pass", "label": "Workspace routes accept any origin (`F02.cors-workspace-routes`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-16-workspace-routes-accept-any-origin-f02-c.json", "failed_command": null, "variants": [ { "feature_id": "F02.cors-workspace-routes", "family": "F02", "result": "pass", "label": "Workspace routes accept any origin (`F02.cors-workspace-routes`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-16-workspace-routes-accept-any-origin-f02-c.json", "failed_command": null } ] }, { "feature_id": "F02.default-bind", "family": "F02", "result": "fail", "label": "Default bind address (`F02.default-bind`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-19-default-bind-address-f02-default-bind.json", "failed_command": "control-agent-server exec --save F02.default-bind/no-key --expect-output 'Starting OpenHands Agent Server on 127.0.0.1:8000' --expect-output 'listening on 127.0.0.1:8000' --reject-output 'Binding to all interfaces' -- unshare -rn env -i PATH=\"$PATH\" QA_F02_PYTHON=\"$QA_F02_PYTHON\" bash \"$F02/bind.sh\" \"$F02/bind-no-key\"", "variants": [ { "feature_id": "F02.default-bind", "family": "F02", "result": "fail", "label": "Default bind address (`F02.default-bind`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-19-default-bind-address-f02-default-bind.json", "failed_command": "control-agent-server exec --save F02.default-bind/no-key --expect-output 'Starting OpenHands Agent Server on 127.0.0.1:8000' --expect-output 'listening on 127.0.0.1:8000' --reject-output 'Binding to all interfaces' -- unshare -rn env -i PATH=\"$PATH\" QA_F02_PYTHON=\"$QA_F02_PYTHON\" bash \"$F02/bind.sh\" \"$F02/bind-no-key\"" } ] }, { "feature_id": "F02.http-header-only", "family": "F02", "result": "pass", "label": "Header only (`F02.http-header-only`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-02-header-only-f02-http-header-only.json", "failed_command": null, "variants": [ { "feature_id": "F02.http-header-only", "family": "F02", "result": "pass", "label": "Header only (`F02.http-header-only`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-02-header-only-f02-http-header-only.json", "failed_command": null } ] }, { "feature_id": "F02.http-key-required", "family": "F02", "result": "pass", "label": "Key required on HTTP (`F02.http-key-required`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-01-key-required-on-http-f02-http-key-requir.json", "failed_command": null, "variants": [ { "feature_id": "F02.http-key-required", "family": "F02", "result": "pass", "label": "Key required on HTTP (`F02.http-key-required`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-01-key-required-on-http-f02-http-key-requir.json", "failed_command": null } ] }, { "feature_id": "F02.keys-hidden-from-commands", "family": "F02", "result": "pass", "label": "Keys stay out of commands (`F02.keys-hidden-from-commands`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-13-keys-stay-out-of-commands-f02-keys-hidde.json", "failed_command": null, "variants": [ { "feature_id": "F02.keys-hidden-from-commands", "family": "F02", "result": "pass", "label": "Keys stay out of commands (`F02.keys-hidden-from-commands`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-13-keys-stay-out-of-commands-f02-keys-hidde.json", "failed_command": null } ] }, { "feature_id": "F02.legacy-env-key", "family": "F02", "result": "pass", "label": "Legacy `SESSION_API_KEY` (`F02.legacy-env-key`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-18-legacy-session-api-key-f02-legacy-env-ke.json", "failed_command": null, "variants": [ { "feature_id": "F02.legacy-env-key", "family": "F02", "result": "pass", "label": "Legacy `SESSION_API_KEY` (`F02.legacy-env-key`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-18-legacy-session-api-key-f02-legacy-env-ke.json", "failed_command": null } ] }, { "feature_id": "F02.multiple-keys", "family": "F02", "result": "pass", "label": "A second key (`F02.multiple-keys`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-11-a-second-key-f02-multiple-keys.json", "failed_command": null, "variants": [ { "feature_id": "F02.multiple-keys", "family": "F02", "result": "pass", "label": "A second key (`F02.multiple-keys`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-11-a-second-key-f02-multiple-keys.json", "failed_command": null } ] }, { "feature_id": "F02.no-auth-mode", "family": "F02", "result": "pass", "label": "Unauthenticated mode (`F02.no-auth-mode`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-17-unauthenticated-mode-f02-no-auth-mode.json", "failed_command": null, "variants": [ { "feature_id": "F02.no-auth-mode", "family": "F02", "result": "pass", "label": "Unauthenticated mode (`F02.no-auth-mode`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-17-unauthenticated-mode-f02-no-auth-mode.json", "failed_command": null } ] }, { "feature_id": "F02.rotate-key", "family": "F02", "result": "pass", "label": "Rotate the key (`F02.rotate-key`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-12-rotate-the-key-f02-rotate-key.json", "failed_command": null, "variants": [ { "feature_id": "F02.rotate-key", "family": "F02", "result": "pass", "label": "Rotate the key (`F02.rotate-key`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-12-rotate-the-key-f02-rotate-key.json", "failed_command": null } ] }, { "feature_id": "F02.workspace-cookie-access", "family": "F02", "result": "pass", "label": "The cookie alone opens workspace files (`F02.workspace-cookie-access`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-09-the-cookie-alone-opens-workspace-files-f.json", "failed_command": null, "variants": [ { "feature_id": "F02.workspace-cookie-access", "family": "F02", "result": "pass", "label": "The cookie alone opens workspace files (`F02.workspace-cookie-access`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-09-the-cookie-alone-opens-workspace-files-f.json", "failed_command": null } ] }, { "feature_id": "F02.workspace-cookie-root-path", "family": "F02", "result": "xfail", "label": "Cookie path ignores a prefix (`F02.workspace-cookie-root-path`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-20-cookie-path-ignores-a-prefix-f02-workspa.json", "failed_command": "control-agent-server exec --run \"$PREFIXED\" --save F02.workspace-cookie-root-path/file-with-cookie -- bash \"$F02/jar.sh\" GET \"/qa-f02-prefix/$PCID/workspace/qa-f02.html\" nokey 200 'has:qa-f02-workspace-file'", "variants": [ { "feature_id": "F02.workspace-cookie-root-path", "family": "F02", "result": "xfail", "label": "Cookie path ignores a prefix (`F02.workspace-cookie-root-path`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-20-cookie-path-ignores-a-prefix-f02-workspa.json", "failed_command": "control-agent-server exec --run \"$PREFIXED\" --save F02.workspace-cookie-root-path/file-with-cookie -- bash \"$F02/jar.sh\" GET \"/qa-f02-prefix/$PCID/workspace/qa-f02.html\" nokey 200 'has:qa-f02-workspace-file'" } ] }, { "feature_id": "F02.workspace-session-delete", "family": "F02", "result": "pass", "label": "Delete the workspace cookie (`F02.workspace-session-delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-10-delete-the-workspace-cookie-f02-workspac.json", "failed_command": null, "variants": [ { "feature_id": "F02.workspace-session-delete", "family": "F02", "result": "pass", "label": "Delete the workspace cookie (`F02.workspace-session-delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-10-delete-the-workspace-cookie-f02-workspac.json", "failed_command": null } ] }, { "feature_id": "F02.workspace-session-ipv6-loopback", "family": "F02", "result": "xfail", "label": "IPv6 loopback is not a secure context (`F02.workspace-session-ipv6-loopback`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-21-ipv6-loopback-is-not-a-secure-context-f0.json", "failed_command": "control-agent-server api POST /api/auth/workspace-session --header 'Host: [::1]:8000' --expect 204 --save F02.workspace-session-ipv6-loopback/ipv6-host --check-header set-cookie contains SameSite=none --check-header set-cookie contains Secure --check-header set-cookie contains Partitioned", "variants": [ { "feature_id": "F02.workspace-session-ipv6-loopback", "family": "F02", "result": "xfail", "label": "IPv6 loopback is not a secure context (`F02.workspace-session-ipv6-loopback`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-21-ipv6-loopback-is-not-a-secure-context-f0.json", "failed_command": "control-agent-server api POST /api/auth/workspace-session --header 'Host: [::1]:8000' --expect 204 --save F02.workspace-session-ipv6-loopback/ipv6-host --check-header set-cookie contains SameSite=none --check-header set-cookie contains Secure --check-header set-cookie contains Partitioned" } ] }, { "feature_id": "F02.workspace-session-mint", "family": "F02", "result": "pass", "label": "Mint the workspace cookie (`F02.workspace-session-mint`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-08-mint-the-workspace-cookie-f02-workspace-.json", "failed_command": null, "variants": [ { "feature_id": "F02.workspace-session-mint", "family": "F02", "result": "pass", "label": "Mint the workspace cookie (`F02.workspace-session-mint`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-08-mint-the-workspace-cookie-f02-workspace-.json", "failed_command": null } ] }, { "feature_id": "F02.ws-auth-timeout", "family": "F02", "result": "pass", "label": "Auth timeout (`F02.ws-auth-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-05-auth-timeout-f02-ws-auth-timeout.json", "failed_command": null, "variants": [ { "feature_id": "F02.ws-auth-timeout", "family": "F02", "result": "pass", "label": "Auth timeout (`F02.ws-auth-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-05-auth-timeout-f02-ws-auth-timeout.json", "failed_command": null } ] }, { "feature_id": "F02.ws-first-frame", "family": "F02", "result": "pass", "label": "First-frame auth on every socket (`F02.ws-first-frame`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-03-first-frame-auth-on-every-socket-f02-ws-.json", "failed_command": null, "variants": [ { "feature_id": "F02.ws-first-frame", "family": "F02", "result": "pass", "label": "First-frame auth on every socket (`F02.ws-first-frame`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-03-first-frame-auth-on-every-socket-f02-ws-.json", "failed_command": null } ] }, { "feature_id": "F02.ws-first-frame-reject", "family": "F02", "result": "pass", "label": "Rejected first frames (`F02.ws-first-frame-reject`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-04-rejected-first-frames-f02-ws-first-frame.json", "failed_command": null, "variants": [ { "feature_id": "F02.ws-first-frame-reject", "family": "F02", "result": "pass", "label": "Rejected first frames (`F02.ws-first-frame-reject`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-04-rejected-first-frames-f02-ws-first-frame.json", "failed_command": null } ] }, { "feature_id": "F02.ws-legacy-key", "family": "F02", "result": "pass", "label": "Deprecated query and header keys (`F02.ws-legacy-key`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-06-deprecated-query-and-header-keys-f02-ws-.json", "failed_command": null, "variants": [ { "feature_id": "F02.ws-legacy-key", "family": "F02", "result": "pass", "label": "Deprecated query and header keys (`F02.ws-legacy-key`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-06-deprecated-query-and-header-keys-f02-ws-.json", "failed_command": null } ] }, { "feature_id": "F02.ws-legacy-reject", "family": "F02", "result": "pass", "label": "Wrong legacy key fails the handshake (`F02.ws-legacy-reject`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-07-wrong-legacy-key-fails-the-handshake-f02.json", "failed_command": null, "variants": [ { "feature_id": "F02.ws-legacy-reject", "family": "F02", "result": "pass", "label": "Wrong legacy key fails the handshake (`F02.ws-legacy-reject`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-07-wrong-legacy-key-fails-the-handshake-f02.json", "failed_command": null } ] }, { "feature_id": "F03.auth-before-gate", "family": "F03", "result": "retired", "label": "retired by #5700. It asserted that a dormant pod with boot session keys answered 401 to a missing key before the 503 gate; readiness is now checked first, see `F03.gate-before-auth`.", "run": null, "transcript": null, "variants": [ { "feature_id": "F03.auth-before-gate", "family": "F03", "result": "retired", "label": "retired by #5700. It asserted that a dormant pod with boot session keys answered 401 to a missing key before the 503 gate; readiness is now checked first, see `F03.gate-before-auth`.", "run": null, "transcript": null } ] }, { "feature_id": "F03.dormant-boot", "family": "F03", "result": "pass", "label": "Dormant boot (`F03.dormant-boot`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-01-dormant-boot-f03-dormant-boot.json", "failed_command": null, "variants": [ { "feature_id": "F03.dormant-boot", "family": "F03", "result": "pass", "label": "Dormant boot (`F03.dormant-boot`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-01-dormant-boot-f03-dormant-boot.json", "failed_command": null } ] }, { "feature_id": "F03.dormant-gates-api", "family": "F03", "result": "pass", "label": "Dormant gate (`F03.dormant-gates-api`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-02-dormant-gate-f03-dormant-gates-api.json", "failed_command": null, "variants": [ { "feature_id": "F03.dormant-gates-api", "family": "F03", "result": "pass", "label": "Dormant gate (`F03.dormant-gates-api`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-02-dormant-gate-f03-dormant-gates-api.json", "failed_command": null } ] }, { "feature_id": "F03.dormant-gates-sockets", "family": "F03", "result": "pass", "label": "Dormant sockets (`F03.dormant-gates-sockets`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-03-dormant-sockets-f03-dormant-gates-socket.json", "failed_command": null, "variants": [ { "feature_id": "F03.dormant-gates-sockets", "family": "F03", "result": "pass", "label": "Dormant sockets (`F03.dormant-gates-sockets`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-03-dormant-sockets-f03-dormant-gates-socket.json", "failed_command": null } ] }, { "feature_id": "F03.dormant-gates-v1", "family": "F03", "result": "pass", "label": "Dormant /v1 and app backends (`F03.dormant-gates-v1`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-04-dormant-v1-and-app-backends-f03-dormant-.json", "failed_command": null, "variants": [ { "feature_id": "F03.dormant-gates-v1", "family": "F03", "result": "pass", "label": "Dormant /v1 and app backends (`F03.dormant-gates-v1`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-04-dormant-v1-and-app-backends-f03-dormant-.json", "failed_command": null } ] }, { "feature_id": "F03.gate-before-auth", "family": "F03", "result": "pass", "label": "Readiness precedes auth (`F03.gate-before-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-22-readiness-precedes-auth-f03-gate-before-.json", "failed_command": null, "variants": [ { "feature_id": "F03.gate-before-auth", "family": "F03", "result": "pass", "label": "Readiness precedes auth (`F03.gate-before-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-22-readiness-precedes-auth-f03-gate-before-.json", "failed_command": null } ] }, { "feature_id": "F03.init-applies-env", "family": "F03", "result": "pass", "label": "Init env reaches bash (`F03.init-applies-env`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-12-init-env-reaches-bash-f03-init-applies-e.json", "failed_command": null, "variants": [ { "feature_id": "F03.init-applies-env", "family": "F03", "result": "pass", "label": "Init env reaches bash (`F03.init-applies-env`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-12-init-env-reaches-bash-f03-init-applies-e.json", "failed_command": null } ] }, { "feature_id": "F03.init-delivered-cors-origins", "family": "F03", "result": "xfail", "label": "Delivered CORS origins (`F03.init-delivered-cors-origins`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-18-delivered-cors-origins-f03-init-delivere.json", "failed_command": "control-agent-server api OPTIONS /api/settings --auth none --header 'Origin: https://qa-f03.example' --header 'Access-Control-Request-Method: GET' --expect 200 --check-header access-control-allow-origin eq https://qa-f03.example --save F03.init-delivered-cors-origins/preflight", "variants": [ { "feature_id": "F03.init-delivered-cors-origins", "family": "F03", "result": "xfail", "label": "Delivered CORS origins (`F03.init-delivered-cors-origins`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-18-delivered-cors-origins-f03-init-delivere.json", "failed_command": "control-agent-server api OPTIONS /api/settings --auth none --header 'Origin: https://qa-f03.example' --header 'Access-Control-Request-Method: GET' --expect 200 --check-header access-control-allow-origin eq https://qa-f03.example --save F03.init-delivered-cors-origins/preflight" } ] }, { "feature_id": "F03.init-delivered-max-runs", "family": "F03", "result": "pass", "label": "Delivered run capacity (`F03.init-delivered-max-runs`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-17-delivered-run-capacity-f03-init-delivere.json", "failed_command": null, "variants": [ { "feature_id": "F03.init-delivered-max-runs", "family": "F03", "result": "pass", "label": "Delivered run capacity (`F03.init-delivered-max-runs`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-17-delivered-run-capacity-f03-init-delivere.json", "failed_command": null } ] }, { "feature_id": "F03.init-delivered-webhooks", "family": "F03", "result": "pass", "label": "Delivered webhooks (`F03.init-delivered-webhooks`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-10-delivered-webhooks-f03-init-delivered-we.json", "failed_command": null, "variants": [ { "feature_id": "F03.init-delivered-webhooks", "family": "F03", "result": "pass", "label": "Delivered webhooks (`F03.init-delivered-webhooks`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-10-delivered-webhooks-f03-init-delivered-we.json", "failed_command": null } ] }, { "feature_id": "F03.init-delivered-worktree-root", "family": "F03", "result": "pass", "label": "Delivered worktree root (`F03.init-delivered-worktree-root`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-16-delivered-worktree-root-f03-init-deliver.json", "failed_command": null, "variants": [ { "feature_id": "F03.init-delivered-worktree-root", "family": "F03", "result": "pass", "label": "Delivered worktree root (`F03.init-delivered-worktree-root`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-16-delivered-worktree-root-f03-init-deliver.json", "failed_command": null } ] }, { "feature_id": "F03.init-enforces-session-keys", "family": "F03", "result": "pass", "label": "Delivered keys are enforced (`F03.init-enforces-session-keys`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-11-delivered-keys-are-enforced-f03-init-enf.json", "failed_command": null, "variants": [ { "feature_id": "F03.init-enforces-session-keys", "family": "F03", "result": "pass", "label": "Delivered keys are enforced (`F03.init-enforces-session-keys`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-11-delivered-keys-are-enforced-f03-init-enf.json", "failed_command": null } ] }, { "feature_id": "F03.init-failure-rollback", "family": "F03", "result": "pass", "label": "Failure rollback (`F03.init-failure-rollback`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-08-failure-rollback-f03-init-failure-rollba.json", "failed_command": null, "variants": [ { "feature_id": "F03.init-failure-rollback", "family": "F03", "result": "pass", "label": "Failure rollback (`F03.init-failure-rollback`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-08-failure-rollback-f03-init-failure-rollba.json", "failed_command": null } ] }, { "feature_id": "F03.init-key-auth", "family": "F03", "result": "pass", "label": "Init key auth (`F03.init-key-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-05-init-key-auth-f03-init-key-auth.json", "failed_command": null, "variants": [ { "feature_id": "F03.init-key-auth", "family": "F03", "result": "pass", "label": "Init key auth (`F03.init-key-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-05-init-key-auth-f03-init-key-auth.json", "failed_command": null } ] }, { "feature_id": "F03.init-keys-gate-bash-socket", "family": "F03", "result": "pass", "label": "Delivered keys gate the bash socket (`F03.init-keys-gate-bash-socket`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-13-delivered-keys-gate-the-bash-socket-f03-.json", "failed_command": null, "variants": [ { "feature_id": "F03.init-keys-gate-bash-socket", "family": "F03", "result": "pass", "label": "Delivered keys gate the bash socket (`F03.init-keys-gate-bash-socket`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-13-delivered-keys-gate-the-bash-socket-f03-.json", "failed_command": null } ] }, { "feature_id": "F03.init-open-without-secret", "family": "F03", "result": "pass", "label": "Open init without a secret (`F03.init-open-without-secret`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-20-open-init-without-a-secret-f03-init-open.json", "failed_command": null, "variants": [ { "feature_id": "F03.init-open-without-secret", "family": "F03", "result": "pass", "label": "Open init without a secret (`F03.init-open-without-secret`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-20-open-init-without-a-secret-f03-init-open.json", "failed_command": null } ] }, { "feature_id": "F03.init-opens-events-socket-and-v1", "family": "F03", "result": "pass", "label": "Events socket and /v1 open after init (`F03.init-opens-events-socket-and-v1`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-14-events-socket-and-v1-open-after-init-f03.json", "failed_command": null, "variants": [ { "feature_id": "F03.init-opens-events-socket-and-v1", "family": "F03", "result": "pass", "label": "Events socket and /v1 open after init (`F03.init-opens-events-socket-and-v1`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-14-events-socket-and-v1-open-after-init-f03.json", "failed_command": null } ] }, { "feature_id": "F03.init-redacts-secret-input", "family": "F03", "result": "pass", "label": "Input redaction (`F03.init-redacts-secret-input`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-07-input-redaction-f03-init-redacts-secret-.json", "failed_command": null, "variants": [ { "feature_id": "F03.init-redacts-secret-input", "family": "F03", "result": "pass", "label": "Input redaction (`F03.init-redacts-secret-input`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-07-input-redaction-f03-init-redacts-secret-.json", "failed_command": null } ] }, { "feature_id": "F03.init-second-call-400", "family": "F03", "result": "pass", "label": "Second init is 400 (`F03.init-second-call-400`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-19-second-init-is-400-f03-init-second-call-.json", "failed_command": null, "variants": [ { "feature_id": "F03.init-second-call-400", "family": "F03", "result": "pass", "label": "Second init is 400 (`F03.init-second-call-400`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-19-second-init-is-400-f03-init-second-call-.json", "failed_command": null } ] }, { "feature_id": "F03.init-state-not-persisted", "family": "F03", "result": "pass", "label": "Init state is not persisted (`F03.init-state-not-persisted`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-21-init-state-is-not-persisted-f03-init-sta.json", "failed_command": null, "variants": [ { "feature_id": "F03.init-state-not-persisted", "family": "F03", "result": "pass", "label": "Init state is not persisted (`F03.init-state-not-persisted`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-21-init-state-is-not-persisted-f03-init-sta.json", "failed_command": null } ] }, { "feature_id": "F03.init-success", "family": "F03", "result": "pass", "label": "Successful init (`F03.init-success`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-09-successful-init-f03-init-success.json", "failed_command": null, "variants": [ { "feature_id": "F03.init-success", "family": "F03", "result": "pass", "label": "Successful init (`F03.init-success`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-09-successful-init-f03-init-success.json", "failed_command": null } ] }, { "feature_id": "F03.init-validation", "family": "F03", "result": "pass", "label": "Init validation (`F03.init-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-06-init-validation-f03-init-validation.json", "failed_command": null, "variants": [ { "feature_id": "F03.init-validation", "family": "F03", "result": "pass", "label": "Init validation (`F03.init-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-06-init-validation-f03-init-validation.json", "failed_command": null } ] }, { "feature_id": "F03.init-web-url-root-path", "family": "F03", "result": "pass", "label": "Delivered web URL sets the root path (`F03.init-web-url-root-path`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-15-delivered-web-url-sets-the-root-path-f03.json", "failed_command": null, "variants": [ { "feature_id": "F03.init-web-url-root-path", "family": "F03", "result": "pass", "label": "Delivered web URL sets the root path (`F03.init-web-url-root-path`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-15-delivered-web-url-sets-the-root-path-f03.json", "failed_command": null } ] }, { "feature_id": "F03.not-deferred-404", "family": "F03", "result": "pass", "label": "Not deferred is 404 (`F03.not-deferred-404`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-23-not-deferred-is-404-f03-not-deferred-404.json", "failed_command": null, "variants": [ { "feature_id": "F03.not-deferred-404", "family": "F03", "result": "pass", "label": "Not deferred is 404 (`F03.not-deferred-404`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-23-not-deferred-is-404-f03-not-deferred-404.json", "failed_command": null } ] }, { "feature_id": "F04.batch-get", "family": "F04", "result": "pass", "label": "Batch get (`F04.batch-get`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-19-batch-get-f04-batch-get.json", "failed_command": null, "variants": [ { "feature_id": "F04.batch-get", "family": "F04", "result": "pass", "label": "Batch get (`F04.batch-get`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-19-batch-get-f04-batch-get.json", "failed_command": null } ] }, { "feature_id": "F04.batch-get-limit", "family": "F04", "result": "xfail", "label": "100 ids in one batch (`F04.batch-get-limit`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-20-100-ids-in-one-batch-f04-batch-get-limit.json", "failed_command": "control-agent-server exec --expect-output QA_BATCH_OK --save F04.batch-get-limit/hundred -- python3 \"$F/qa-f04-batch.py\" \"$IDS100\" 400,422", "variants": [ { "feature_id": "F04.batch-get-limit", "family": "F04", "result": "xfail", "label": "100 ids in one batch (`F04.batch-get-limit`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-20-100-ids-in-one-batch-f04-batch-get-limit.json", "failed_command": "control-agent-server exec --expect-output QA_BATCH_OK --save F04.batch-get-limit/hundred -- python3 \"$F/qa-f04-batch.py\" \"$IDS100\" 400,422" } ] }, { "feature_id": "F04.count", "family": "F04", "result": "pass", "label": "Count (`F04.count`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-23-count-f04-count.json", "failed_command": null, "variants": [ { "feature_id": "F04.count", "family": "F04", "result": "pass", "label": "Count (`F04.count`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-23-count-f04-count.json", "failed_command": null } ] }, { "feature_id": "F04.create-agent-profile", "family": "F04", "result": "pass", "label": "Stored Agent Profile (`F04.create-agent-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-03-stored-agent-profile-f04-create-agent-pr.json", "failed_command": null, "variants": [ { "feature_id": "F04.create-agent-profile", "family": "F04", "result": "pass", "label": "Stored Agent Profile (`F04.create-agent-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-03-stored-agent-profile-f04-create-agent-pr.json", "failed_command": null } ] }, { "feature_id": "F04.create-idempotent", "family": "F04", "result": "pass", "label": "Idempotent create with a client id (`F04.create-idempotent`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-06-idempotent-create-with-a-client-id-f04-c.json", "failed_command": null, "variants": [ { "feature_id": "F04.create-idempotent", "family": "F04", "result": "pass", "label": "Idempotent create with a client id (`F04.create-idempotent`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-06-idempotent-create-with-a-client-id-f04-c.json", "failed_command": null } ] }, { "feature_id": "F04.create-initial-message-run-false", "family": "F04", "result": "xfail", "label": "`initial_message.run: false` (`F04.create-initial-message-run-false`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-07-initial-message-run-false-f04-create-ini.json", "failed_command": "false", "variants": [ { "feature_id": "F04.create-initial-message-run-false", "family": "F04", "result": "xfail", "label": "`initial_message.run: false` (`F04.create-initial-message-run-false`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-07-initial-message-run-false-f04-create-ini.json", "failed_command": "false" } ] }, { "feature_id": "F04.create-inline-agent", "family": "F04", "result": "pass", "label": "Inline agent (`F04.create-inline-agent`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-02-inline-agent-f04-create-inline-agent.json", "failed_command": null, "variants": [ { "feature_id": "F04.create-inline-agent", "family": "F04", "result": "pass", "label": "Inline agent (`F04.create-inline-agent`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-02-inline-agent-f04-create-inline-agent.json", "failed_command": null } ] }, { "feature_id": "F04.create-observability", "family": "F04", "result": "pass", "label": "Observability headers (`F04.create-observability`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-10-observability-headers-f04-create-observa.json", "failed_command": null, "variants": [ { "feature_id": "F04.create-observability", "family": "F04", "result": "pass", "label": "Observability headers (`F04.create-observability`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-10-observability-headers-f04-create-observa.json", "failed_command": null } ] }, { "feature_id": "F04.create-observability-precedence", "family": "F04", "result": "pass", "label": "Body versus headers (`F04.create-observability-precedence`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-11-body-versus-headers-f04-create-observabi.json", "failed_command": null, "variants": [ { "feature_id": "F04.create-observability-precedence", "family": "F04", "result": "pass", "label": "Body versus headers (`F04.create-observability-precedence`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-11-body-versus-headers-f04-create-observabi.json", "failed_command": null } ] }, { "feature_id": "F04.create-parent", "family": "F04", "result": "pass", "label": "Parent and child (`F04.create-parent`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-08-parent-and-child-f04-create-parent.json", "failed_command": null, "variants": [ { "feature_id": "F04.create-parent", "family": "F04", "result": "pass", "label": "Parent and child (`F04.create-parent`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-08-parent-and-child-f04-create-parent.json", "failed_command": null } ] }, { "feature_id": "F04.create-secrets", "family": "F04", "result": "pass", "label": "Start-time secrets (`F04.create-secrets`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-12-start-time-secrets-f04-create-secrets.json", "failed_command": null, "variants": [ { "feature_id": "F04.create-secrets", "family": "F04", "result": "pass", "label": "Start-time secrets (`F04.create-secrets`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-12-start-time-secrets-f04-create-secrets.json", "failed_command": null } ] }, { "feature_id": "F04.create-secrets-get", "family": "F04", "result": "xfail", "label": "Start-time secret visible to GET (`F04.create-secrets-get`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-13-start-time-secret-visible-to-get-f04-cre.json", "failed_command": "control-agent-server api GET \"/$SECID\" --expect 200 --check secret_registry.secret_sources.QA_F04_TOKEN.kind eq StaticSecret --save F04.create-secrets-get/get", "variants": [ { "feature_id": "F04.create-secrets-get", "family": "F04", "result": "xfail", "label": "Start-time secret visible to GET (`F04.create-secrets-get`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-13-start-time-secret-visible-to-get-f04-cre.json", "failed_command": "control-agent-server api GET \"/$SECID\" --expect 200 --check secret_registry.secret_sources.QA_F04_TOKEN.kind eq StaticSecret --save F04.create-secrets-get/get" } ] }, { "feature_id": "F04.create-secrets-plain", "family": "F04", "result": "xfail", "label": "Plain-string secret value (`F04.create-secrets-plain`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-14-plain-string-secret-value-f04-create-sec.json", "failed_command": "control-agent-server api POST --json \"{\\\"workspace\\\": {\\\"working_dir\\\": \\\"$W\\\"}, \\\"agent\\\": $OFFLINE_AGENT, \\\"autotitle\\\": false, \\\"secrets\\\": {\\\"QA_F04_PLAIN\\\": \\\"qa-f04-plain-value\\\"}}\" --expect 201,422 --save F04.create-secrets-plain/start", "variants": [ { "feature_id": "F04.create-secrets-plain", "family": "F04", "result": "xfail", "label": "Plain-string secret value (`F04.create-secrets-plain`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-14-plain-string-secret-value-f04-create-sec.json", "failed_command": "control-agent-server api POST --json \"{\\\"workspace\\\": {\\\"working_dir\\\": \\\"$W\\\"}, \\\"agent\\\": $OFFLINE_AGENT, \\\"autotitle\\\": false, \\\"secrets\\\": {\\\"QA_F04_PLAIN\\\": \\\"qa-f04-plain-value\\\"}}\" --expect 201,422 --save F04.create-secrets-plain/start" } ] }, { "feature_id": "F04.create-settings", "family": "F04", "result": "pass", "label": "Create from the saved settings (`F04.create-settings`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-01-create-from-the-saved-settings-f04-creat.json", "failed_command": null, "variants": [ { "feature_id": "F04.create-settings", "family": "F04", "result": "pass", "label": "Create from the saved settings (`F04.create-settings`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-01-create-from-the-saved-settings-f04-creat.json", "failed_command": null } ] }, { "feature_id": "F04.create-validation", "family": "F04", "result": "pass", "label": "Create validation (`F04.create-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-04-create-validation-f04-create-validation.json", "failed_command": null, "variants": [ { "feature_id": "F04.create-validation", "family": "F04", "result": "pass", "label": "Create validation (`F04.create-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-04-create-validation-f04-create-validation.json", "failed_command": null } ] }, { "feature_id": "F04.create-validation-redaction", "family": "F04", "result": "pass", "label": "422 bodies keep the LLM key out (`F04.create-validation-redaction`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-05-422-bodies-keep-the-llm-key-out-f04-crea.json", "failed_command": null, "variants": [ { "feature_id": "F04.create-validation-redaction", "family": "F04", "result": "pass", "label": "422 bodies keep the LLM key out (`F04.create-validation-redaction`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-05-422-bodies-keep-the-llm-key-out-f04-crea.json", "failed_command": null } ] }, { "feature_id": "F04.create-worktree", "family": "F04", "result": "pass", "label": "Git worktree (`F04.create-worktree`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-09-git-worktree-f04-create-worktree.json", "failed_command": null, "variants": [ { "feature_id": "F04.create-worktree", "family": "F04", "result": "pass", "label": "Git worktree (`F04.create-worktree`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-09-git-worktree-f04-create-worktree.json", "failed_command": null } ] }, { "feature_id": "F04.delete", "family": "F04", "result": "pass", "label": "Delete (`F04.delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-29-delete-f04-delete.json", "failed_command": null, "variants": [ { "feature_id": "F04.delete", "family": "F04", "result": "pass", "label": "Delete (`F04.delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-29-delete-f04-delete.json", "failed_command": null } ] }, { "feature_id": "F04.delete-unknown-id", "family": "F04", "result": "xfail", "label": "DELETE on an unknown id (`F04.delete-unknown-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-30-delete-on-an-unknown-id-f04-delete-unkno.json", "failed_command": "control-agent-server api DELETE \"/$NOPE\" --expect 404 --save F04.delete-unknown-id/unknown", "variants": [ { "feature_id": "F04.delete-unknown-id", "family": "F04", "result": "xfail", "label": "DELETE on an unknown id (`F04.delete-unknown-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-30-delete-on-an-unknown-id-f04-delete-unkno.json", "failed_command": "control-agent-server api DELETE \"/$NOPE\" --expect 404 --save F04.delete-unknown-id/unknown" } ] }, { "feature_id": "F04.final-response", "family": "F04", "result": "pass", "label": "Final response (`F04.final-response`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-16-final-response-f04-final-response.json", "failed_command": null, "variants": [ { "feature_id": "F04.final-response", "family": "F04", "result": "pass", "label": "Final response (`F04.final-response`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-16-final-response-f04-final-response.json", "failed_command": null } ] }, { "feature_id": "F04.get", "family": "F04", "result": "pass", "label": "Read one conversation (`F04.get`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-18-read-one-conversation-f04-get.json", "failed_command": null, "variants": [ { "feature_id": "F04.get", "family": "F04", "result": "pass", "label": "Read one conversation (`F04.get`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-18-read-one-conversation-f04-get.json", "failed_command": null } ] }, { "feature_id": "F04.patch-errors", "family": "F04", "result": "pass", "label": "PATCH errors (`F04.patch-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-26-patch-errors-f04-patch-errors.json", "failed_command": null, "variants": [ { "feature_id": "F04.patch-errors", "family": "F04", "result": "pass", "label": "PATCH errors (`F04.patch-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-26-patch-errors-f04-patch-errors.json", "failed_command": null } ] }, { "feature_id": "F04.patch-tags", "family": "F04", "result": "pass", "label": "Retag (`F04.patch-tags`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-25-retag-f04-patch-tags.json", "failed_command": null, "variants": [ { "feature_id": "F04.patch-tags", "family": "F04", "result": "pass", "label": "Retag (`F04.patch-tags`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-25-retag-f04-patch-tags.json", "failed_command": null } ] }, { "feature_id": "F04.patch-title", "family": "F04", "result": "pass", "label": "Rename (`F04.patch-title`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-24-rename-f04-patch-title.json", "failed_command": null, "variants": [ { "feature_id": "F04.patch-title", "family": "F04", "result": "pass", "label": "Rename (`F04.patch-title`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-24-rename-f04-patch-title.json", "failed_command": null } ] }, { "feature_id": "F04.patch-unknown-id", "family": "F04", "result": "xfail", "label": "PATCH on an unknown id (`F04.patch-unknown-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-28-patch-on-an-unknown-id-f04-patch-unknown.json", "failed_command": "control-agent-server api PATCH \"/$NOPE\" --json '{\"title\": \"QA nobody\"}' --expect 404 --save F04.patch-unknown-id/unknown", "variants": [ { "feature_id": "F04.patch-unknown-id", "family": "F04", "result": "xfail", "label": "PATCH on an unknown id (`F04.patch-unknown-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-28-patch-on-an-unknown-id-f04-patch-unknown.json", "failed_command": "control-agent-server api PATCH \"/$NOPE\" --json '{\"title\": \"QA nobody\"}' --expect 404 --save F04.patch-unknown-id/unknown" } ] }, { "feature_id": "F04.patch-whitespace-title", "family": "F04", "result": "xfail", "label": "Whitespace-only title (`F04.patch-whitespace-title`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-27-whitespace-only-title-f04-patch-whitespa.json", "failed_command": "control-agent-server api PATCH \"/$CHILD\" --json '{\"title\": \" \"}' --expect 422 --save F04.patch-whitespace-title/patch", "variants": [ { "feature_id": "F04.patch-whitespace-title", "family": "F04", "result": "xfail", "label": "Whitespace-only title (`F04.patch-whitespace-title`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-27-whitespace-only-title-f04-patch-whitespa.json", "failed_command": "control-agent-server api PATCH \"/$CHILD\" --json '{\"title\": \" \"}' --expect 422 --save F04.patch-whitespace-title/patch" } ] }, { "feature_id": "F04.read-no-hydrate", "family": "F04", "result": "pass", "label": "Cold reads do not load a conversation (`F04.read-no-hydrate`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-32-cold-reads-do-not-load-a-conversation-f0.json", "failed_command": null, "variants": [ { "feature_id": "F04.read-no-hydrate", "family": "F04", "result": "pass", "label": "Cold reads do not load a conversation (`F04.read-no-hydrate`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-32-cold-reads-do-not-load-a-conversation-f0.json", "failed_command": null } ] }, { "feature_id": "F04.restart-persist", "family": "F04", "result": "pass", "label": "Restart (`F04.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-31-restart-f04-restart-persist.json", "failed_command": null, "variants": [ { "feature_id": "F04.restart-persist", "family": "F04", "result": "pass", "label": "Restart (`F04.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-31-restart-f04-restart-persist.json", "failed_command": null } ] }, { "feature_id": "F04.run-real", "family": "F04", "result": "pass", "label": "A real run (`F04.run-real`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-15-a-real-run-f04-run-real.json", "failed_command": null, "variants": [ { "feature_id": "F04.run-real", "family": "F04", "result": "pass", "label": "A real run (`F04.run-real`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-15-a-real-run-f04-run-real.json", "failed_command": null } ] }, { "feature_id": "F04.search-filter", "family": "F04", "result": "pass", "label": "Status filter (`F04.search-filter`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-22-status-filter-f04-search-filter.json", "failed_command": null, "variants": [ { "feature_id": "F04.search-filter", "family": "F04", "result": "pass", "label": "Status filter (`F04.search-filter`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-22-status-filter-f04-search-filter.json", "failed_command": null } ] }, { "feature_id": "F04.search-page", "family": "F04", "result": "pass", "label": "Search pages (`F04.search-page`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-21-search-pages-f04-search-page.json", "failed_command": null, "variants": [ { "feature_id": "F04.search-page", "family": "F04", "result": "pass", "label": "Search pages (`F04.search-page`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-21-search-pages-f04-search-page.json", "failed_command": null } ] }, { "feature_id": "F04.title-llm-profile", "family": "F04", "result": "pass", "label": "Title from a saved LLM profile (`F04.title-llm-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-17-title-from-a-saved-llm-profile-f04-title.json", "failed_command": null, "variants": [ { "feature_id": "F04.title-llm-profile", "family": "F04", "result": "pass", "label": "Title from a saved LLM profile (`F04.title-llm-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-17-title-from-a-saved-llm-profile-f04-title.json", "failed_command": null } ] }, { "feature_id": "F05.control-errors", "family": "F05", "result": "pass", "label": "Auth and malformed ids (`F05.control-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-13-auth-and-malformed-ids-f05-control-error.json", "failed_command": null, "variants": [ { "feature_id": "F05.control-errors", "family": "F05", "result": "pass", "label": "Auth and malformed ids (`F05.control-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-13-auth-and-malformed-ids-f05-control-error.json", "failed_command": null } ] }, { "feature_id": "F05.interrupt-idle", "family": "F05", "result": "pass", "label": "Interrupt with nothing running (`F05.interrupt-idle`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-07-interrupt-with-nothing-running-f05-inter.json", "failed_command": null, "variants": [ { "feature_id": "F05.interrupt-idle", "family": "F05", "result": "pass", "label": "Interrupt with nothing running (`F05.interrupt-idle`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-07-interrupt-with-nothing-running-f05-inter.json", "failed_command": null } ] }, { "feature_id": "F05.interrupt-running", "family": "F05", "result": "pass", "label": "Interrupt a running conversation (`F05.interrupt-running`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-18-interrupt-a-running-conversation-f05-int.json", "failed_command": null, "variants": [ { "feature_id": "F05.interrupt-running", "family": "F05", "result": "pass", "label": "Interrupt a running conversation (`F05.interrupt-running`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-18-interrupt-a-running-conversation-f05-int.json", "failed_command": null } ] }, { "feature_id": "F05.interrupt-tool-call", "family": "F05", "result": "pass", "label": "Interrupt during a tool call (`F05.interrupt-tool-call`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-21-interrupt-during-a-tool-call-f05-interru.json", "failed_command": null, "variants": [ { "feature_id": "F05.interrupt-tool-call", "family": "F05", "result": "pass", "label": "Interrupt during a tool call (`F05.interrupt-tool-call`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-21-interrupt-during-a-tool-call-f05-interru.json", "failed_command": null } ] }, { "feature_id": "F05.max-iterations-resume", "family": "F05", "result": "pass", "label": "Resume after the cap (`F05.max-iterations-resume`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-27-resume-after-the-cap-f05-max-iterations-.json", "failed_command": null, "variants": [ { "feature_id": "F05.max-iterations-resume", "family": "F05", "result": "pass", "label": "Resume after the cap (`F05.max-iterations-resume`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-27-resume-after-the-cap-f05-max-iterations-.json", "failed_command": null } ] }, { "feature_id": "F05.max-iterations-stop", "family": "F05", "result": "pass", "label": "Iteration cap (`F05.max-iterations-stop`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-26-iteration-cap-f05-max-iterations-stop.json", "failed_command": null, "variants": [ { "feature_id": "F05.max-iterations-stop", "family": "F05", "result": "pass", "label": "Iteration cap (`F05.max-iterations-stop`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-26-iteration-cap-f05-max-iterations-stop.json", "failed_command": null } ] }, { "feature_id": "F05.pause-idle", "family": "F05", "result": "pass", "label": "Pause an idle conversation (`F05.pause-idle`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-04-pause-an-idle-conversation-f05-pause-idl.json", "failed_command": null, "variants": [ { "feature_id": "F05.pause-idle", "family": "F05", "result": "pass", "label": "Pause an idle conversation (`F05.pause-idle`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-04-pause-an-idle-conversation-f05-pause-idl.json", "failed_command": null } ] }, { "feature_id": "F05.pause-repeat", "family": "F05", "result": "pass", "label": "Pause twice (`F05.pause-repeat`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-05-pause-twice-f05-pause-repeat.json", "failed_command": null, "variants": [ { "feature_id": "F05.pause-repeat", "family": "F05", "result": "pass", "label": "Pause twice (`F05.pause-repeat`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-05-pause-twice-f05-pause-repeat.json", "failed_command": null } ] }, { "feature_id": "F05.pause-running", "family": "F05", "result": "pass", "label": "Pause during a long model call (`F05.pause-running`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-15-pause-during-a-long-model-call-f05-pause.json", "failed_command": null, "variants": [ { "feature_id": "F05.pause-running", "family": "F05", "result": "pass", "label": "Pause during a long model call (`F05.pause-running`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-15-pause-during-a-long-model-call-f05-pause.json", "failed_command": null } ] }, { "feature_id": "F05.pause-running-crash", "family": "F05", "result": "fail", "label": "Crash after a mid-call pause (`F05.pause-running-crash`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-31-crash-after-a-mid-call-pause-f05-pause-r.json", "failed_command": "control-agent-server api POST \"/$K/run\" --expect 200 --check success eq true", "variants": [ { "feature_id": "F05.pause-running-crash", "family": "F05", "result": "fail", "label": "Crash after a mid-call pause (`F05.pause-running-crash`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-31-crash-after-a-mid-call-pause-f05-pause-r.json", "failed_command": "control-agent-server api POST \"/$K/run\" --expect 200 --check success eq true" } ] }, { "feature_id": "F05.pause-running-get", "family": "F05", "result": "xfail", "label": "GET after a mid-call pause (`F05.pause-running-get`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-16-get-after-a-mid-call-pause-f05-pause-run.json", "failed_command": "control-agent-server api GET \"/$P\" --check execution_status eq paused --quiet --save F05.pause-running-get/get", "variants": [ { "feature_id": "F05.pause-running-get", "family": "F05", "result": "xfail", "label": "GET after a mid-call pause (`F05.pause-running-get`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-16-get-after-a-mid-call-pause-f05-pause-run.json", "failed_command": "control-agent-server api GET \"/$P\" --check execution_status eq paused --quiet --save F05.pause-running-get/get" } ] }, { "feature_id": "F05.pause-running-release", "family": "F05", "result": "pass", "label": "Release a paused step with interrupt (`F05.pause-running-release`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-17-release-a-paused-step-with-interrupt-f05.json", "failed_command": null, "variants": [ { "feature_id": "F05.pause-running-release", "family": "F05", "result": "pass", "label": "Release a paused step with interrupt (`F05.pause-running-release`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-17-release-a-paused-step-with-interrupt-f05.json", "failed_command": null } ] }, { "feature_id": "F05.pause-terminal", "family": "F05", "result": "pass", "label": "Pause and interrupt a terminal conversation (`F05.pause-terminal`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-09-pause-and-interrupt-a-terminal-conversat.json", "failed_command": null, "variants": [ { "feature_id": "F05.pause-terminal", "family": "F05", "result": "pass", "label": "Pause and interrupt a terminal conversation (`F05.pause-terminal`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-09-pause-and-interrupt-a-terminal-conversat.json", "failed_command": null } ] }, { "feature_id": "F05.pause-tool-step", "family": "F05", "result": "pass", "label": "Pause during a tool call (`F05.pause-tool-step`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-20-pause-during-a-tool-call-f05-pause-tool-.json", "failed_command": null, "variants": [ { "feature_id": "F05.pause-tool-step", "family": "F05", "result": "pass", "label": "Pause during a tool call (`F05.pause-tool-step`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-20-pause-during-a-tool-call-f05-pause-tool-.json", "failed_command": null } ] }, { "feature_id": "F05.restart-hard-running", "family": "F05", "result": "fail", "label": "Crash while running (`F05.restart-hard-running`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-30-crash-while-running-f05-restart-hard-run.json", "failed_command": "control-agent-server api GET \"/$K\" --check execution_status eq error --save F05.restart-hard-running/get", "variants": [ { "feature_id": "F05.restart-hard-running", "family": "F05", "result": "fail", "label": "Crash while running (`F05.restart-hard-running`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-30-crash-while-running-f05-restart-hard-run.json", "failed_command": "control-agent-server api GET \"/$K\" --check execution_status eq error --save F05.restart-hard-running/get" } ] }, { "feature_id": "F05.restart-keeps-status", "family": "F05", "result": "pass", "label": "Statuses survive the restart (`F05.restart-keeps-status`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-29-statuses-survive-the-restart-f05-restart.json", "failed_command": null, "variants": [ { "feature_id": "F05.restart-keeps-status", "family": "F05", "result": "pass", "label": "Statuses survive the restart (`F05.restart-keeps-status`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-29-statuses-survive-the-restart-f05-restart.json", "failed_command": null } ] }, { "feature_id": "F05.restart-running-graceful", "family": "F05", "result": "pass", "label": "Graceful restart while running (`F05.restart-running-graceful`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-28-graceful-restart-while-running-f05-resta.json", "failed_command": null, "variants": [ { "feature_id": "F05.restart-running-graceful", "family": "F05", "result": "pass", "label": "Graceful restart while running (`F05.restart-running-graceful`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-28-graceful-restart-while-running-f05-resta.json", "failed_command": null } ] }, { "feature_id": "F05.run-after-interrupt", "family": "F05", "result": "pass", "label": "Resume after an interrupt (`F05.run-after-interrupt`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-19-resume-after-an-interrupt-f05-run-after-.json", "failed_command": null, "variants": [ { "feature_id": "F05.run-after-interrupt", "family": "F05", "result": "pass", "label": "Resume after an interrupt (`F05.run-after-interrupt`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-19-resume-after-an-interrupt-f05-run-after-.json", "failed_command": null } ] }, { "feature_id": "F05.run-capacity", "family": "F05", "result": "pass", "label": "Run capacity (`F05.run-capacity`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-22-run-capacity-f05-run-capacity.json", "failed_command": null, "variants": [ { "feature_id": "F05.run-capacity", "family": "F05", "result": "pass", "label": "Run capacity (`F05.run-capacity`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-22-run-capacity-f05-run-capacity.json", "failed_command": null } ] }, { "feature_id": "F05.run-conflict", "family": "F05", "result": "pass", "label": "Run while running (`F05.run-conflict`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-14-run-while-running-f05-run-conflict.json", "failed_command": null, "variants": [ { "feature_id": "F05.run-conflict", "family": "F05", "result": "pass", "label": "Run while running (`F05.run-conflict`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-14-run-while-running-f05-run-conflict.json", "failed_command": null } ] }, { "feature_id": "F05.run-error", "family": "F05", "result": "pass", "label": "A run that fails (`F05.run-error`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-08-a-run-that-fails-f05-run-error.json", "failed_command": null, "variants": [ { "feature_id": "F05.run-error", "family": "F05", "result": "pass", "label": "A run that fails (`F05.run-error`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-08-a-run-that-fails-f05-run-error.json", "failed_command": null } ] }, { "feature_id": "F05.run-error-responsive", "family": "F05", "result": "xfail", "label": "Responsive while a run fails (`F05.run-error-responsive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-10-responsive-while-a-run-fails-f05-run-err.json", "failed_command": "test \"$SLOWEST\" -le 1500", "variants": [ { "feature_id": "F05.run-error-responsive", "family": "F05", "result": "xfail", "label": "Responsive while a run fails (`F05.run-error-responsive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-10-responsive-while-a-run-fails-f05-run-err.json", "failed_command": "test \"$SLOWEST\" -le 1500" } ] }, { "feature_id": "F05.run-finished-noop", "family": "F05", "result": "pass", "label": "Run with nothing to do (`F05.run-finished-noop`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-02-run-with-nothing-to-do-f05-run-finished-.json", "failed_command": null, "variants": [ { "feature_id": "F05.run-finished-noop", "family": "F05", "result": "pass", "label": "Run with nothing to do (`F05.run-finished-noop`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-02-run-with-nothing-to-do-f05-run-finished-.json", "failed_command": null } ] }, { "feature_id": "F05.run-new-input", "family": "F05", "result": "pass", "label": "New input after finishing (`F05.run-new-input`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-03-new-input-after-finishing-f05-run-new-in.json", "failed_command": null, "variants": [ { "feature_id": "F05.run-new-input", "family": "F05", "result": "pass", "label": "New input after finishing (`F05.run-new-input`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-03-new-input-after-finishing-f05-run-new-in.json", "failed_command": null } ] }, { "feature_id": "F05.run-queued", "family": "F05", "result": "pass", "label": "Run queued input on a real model (`F05.run-queued`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-01-run-queued-input-on-a-real-model-f05-run.json", "failed_command": null, "variants": [ { "feature_id": "F05.run-queued", "family": "F05", "result": "pass", "label": "Run queued input on a real model (`F05.run-queued`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-01-run-queued-input-on-a-real-model-f05-run.json", "failed_command": null } ] }, { "feature_id": "F05.run-resume-paused", "family": "F05", "result": "pass", "label": "Resume a paused conversation (`F05.run-resume-paused`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-06-resume-a-paused-conversation-f05-run-res.json", "failed_command": null, "variants": [ { "feature_id": "F05.run-resume-paused", "family": "F05", "result": "pass", "label": "Resume a paused conversation (`F05.run-resume-paused`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-06-resume-a-paused-conversation-f05-run-res.json", "failed_command": null } ] }, { "feature_id": "F05.stuck-detected", "family": "F05", "result": "pass", "label": "A run that loops ends `stuck` (`F05.stuck-detected`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-23-a-run-that-loops-ends-stuck-f05-stuck-de.json", "failed_command": null, "variants": [ { "feature_id": "F05.stuck-detected", "family": "F05", "result": "pass", "label": "A run that loops ends `stuck` (`F05.stuck-detected`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-23-a-run-that-loops-ends-stuck-f05-stuck-de.json", "failed_command": null } ] }, { "feature_id": "F05.stuck-new-input", "family": "F05", "result": "pass", "label": "New input un-sticks it (`F05.stuck-new-input`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-25-new-input-un-sticks-it-f05-stuck-new-inp.json", "failed_command": null, "variants": [ { "feature_id": "F05.stuck-new-input", "family": "F05", "result": "pass", "label": "New input un-sticks it (`F05.stuck-new-input`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-25-new-input-un-sticks-it-f05-stuck-new-inp.json", "failed_command": null } ] }, { "feature_id": "F05.stuck-rerun", "family": "F05", "result": "pass", "label": "Run a stuck conversation again (`F05.stuck-rerun`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-24-run-a-stuck-conversation-again-f05-stuck.json", "failed_command": null, "variants": [ { "feature_id": "F05.stuck-rerun", "family": "F05", "result": "pass", "label": "Run a stuck conversation again (`F05.stuck-rerun`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-24-run-a-stuck-conversation-again-f05-stuck.json", "failed_command": null } ] }, { "feature_id": "F05.unknown-id-pause", "family": "F05", "result": "xfail", "label": "Pause or interrupt an unknown id (`F05.unknown-id-pause`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-12-pause-or-interrupt-an-unknown-id-f05-unk.json", "failed_command": "control-agent-server api POST \"/$NOPE/pause\" --expect 404 --save F05.unknown-id-pause/pause", "variants": [ { "feature_id": "F05.unknown-id-pause", "family": "F05", "result": "xfail", "label": "Pause or interrupt an unknown id (`F05.unknown-id-pause`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-12-pause-or-interrupt-an-unknown-id-f05-unk.json", "failed_command": "control-agent-server api POST \"/$NOPE/pause\" --expect 404 --save F05.unknown-id-pause/pause" } ] }, { "feature_id": "F05.unknown-id-run", "family": "F05", "result": "pass", "label": "Run an unknown id (`F05.unknown-id-run`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-11-run-an-unknown-id-f05-unknown-id-run.json", "failed_command": null, "variants": [ { "feature_id": "F05.unknown-id-run", "family": "F05", "result": "pass", "label": "Run an unknown id (`F05.unknown-id-run`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-11-run-an-unknown-id-f05-unknown-id-run.json", "failed_command": null } ] }, { "feature_id": "F06.batch-get", "family": "F06", "result": "pass", "label": "Batch get (`F06.batch-get`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-10-batch-get-f06-batch-get.json", "failed_command": null, "variants": [ { "feature_id": "F06.batch-get", "family": "F06", "result": "pass", "label": "Batch get (`F06.batch-get`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-10-batch-get-f06-batch-get.json", "failed_command": null } ] }, { "feature_id": "F06.count", "family": "F06", "result": "pass", "label": "Count (`F06.count`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-07-count-f06-count.json", "failed_command": null, "variants": [ { "feature_id": "F06.count", "family": "F06", "result": "pass", "label": "Count (`F06.count`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-07-count-f06-count.json", "failed_command": null } ] }, { "feature_id": "F06.get-one", "family": "F06", "result": "pass", "label": "Get one event (`F06.get-one`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-08-get-one-event-f06-get-one.json", "failed_command": null, "variants": [ { "feature_id": "F06.get-one", "family": "F06", "result": "pass", "label": "Get one event (`F06.get-one`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-08-get-one-event-f06-get-one.json", "failed_command": null } ] }, { "feature_id": "F06.not-found-auth", "family": "F06", "result": "pass", "label": "Unknown conversation, bad id, no key (`F06.not-found-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-11-unknown-conversation-bad-id-no-key-f06-n.json", "failed_command": null, "variants": [ { "feature_id": "F06.not-found-auth", "family": "F06", "result": "pass", "label": "Unknown conversation, bad id, no key (`F06.not-found-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-11-unknown-conversation-bad-id-no-key-f06-n.json", "failed_command": null } ] }, { "feature_id": "F06.restart-persist", "family": "F06", "result": "pass", "label": "History survives a restart (`F06.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-25-history-survives-a-restart-f06-restart-p.json", "failed_command": null, "variants": [ { "feature_id": "F06.restart-persist", "family": "F06", "result": "pass", "label": "History survives a restart (`F06.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-25-history-survives-a-restart-f06-restart-p.json", "failed_command": null } ] }, { "feature_id": "F06.search-filters", "family": "F06", "result": "pass", "label": "Filters (`F06.search-filters`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-04-filters-f06-search-filters.json", "failed_command": null, "variants": [ { "feature_id": "F06.search-filters", "family": "F06", "result": "pass", "label": "Filters (`F06.search-filters`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-04-filters-f06-search-filters.json", "failed_command": null } ] }, { "feature_id": "F06.search-kind-short", "family": "F06", "result": "xfail", "label": "Short kind names (`F06.search-kind-short`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-05-short-kind-names-f06-search-kind-short-k.json", "failed_command": "control-agent-server api GET \"/$CID/events/search\" --query kind=MessageEvent --expect 200 --check items len-eq 2 --save F06.search-kind-short/short-name", "variants": [ { "feature_id": "F06.search-kind-short", "family": "F06", "result": "xfail", "label": "Short kind names (`F06.search-kind-short`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-05-short-kind-names-f06-search-kind-short-k.json", "failed_command": "control-agent-server api GET \"/$CID/events/search\" --query kind=MessageEvent --expect 200 --check items len-eq 2 --save F06.search-kind-short/short-name" } ] }, { "feature_id": "F06.search-page", "family": "F06", "result": "pass", "label": "Paging and sort order (`F06.search-page`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-02-paging-and-sort-order-f06-search-page.json", "failed_command": null, "variants": [ { "feature_id": "F06.search-page", "family": "F06", "result": "pass", "label": "Paging and sort order (`F06.search-page`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-02-paging-and-sort-order-f06-search-page.json", "failed_command": null } ] }, { "feature_id": "F06.search-time-window", "family": "F06", "result": "pass", "label": "Time window (`F06.search-time-window`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-06-time-window-f06-search-time-window.json", "failed_command": null, "variants": [ { "feature_id": "F06.search-time-window", "family": "F06", "result": "pass", "label": "Time window (`F06.search-time-window`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-06-time-window-f06-search-time-window.json", "failed_command": null } ] }, { "feature_id": "F06.search-timestamp-order", "family": "F06", "result": "xfail", "label": "Log order is timestamp order (`F06.search-timestamp-order`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-23-log-order-is-timestamp-order-f06-search-.json", "failed_command": "control-agent-server api GET \"/$LIVE/events/count\" --query timestamp__gte=\"$OBS_T\" --check . ge \"$FROM_OBS\" --save F06.search-timestamp-order/count-since-observation", "variants": [ { "feature_id": "F06.search-timestamp-order", "family": "F06", "result": "xfail", "label": "Log order is timestamp order (`F06.search-timestamp-order`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-23-log-order-is-timestamp-order-f06-search-.json", "failed_command": "control-agent-server api GET \"/$LIVE/events/count\" --query timestamp__gte=\"$OBS_T\" --check . ge \"$FROM_OBS\" --save F06.search-timestamp-order/count-since-observation" } ] }, { "feature_id": "F06.search-validation", "family": "F06", "result": "pass", "label": "Bad search parameters (`F06.search-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-03-bad-search-parameters-f06-search-validat.json", "failed_command": null, "variants": [ { "feature_id": "F06.search-validation", "family": "F06", "result": "pass", "label": "Bad search parameters (`F06.search-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-03-bad-search-parameters-f06-search-validat.json", "failed_command": null } ] }, { "feature_id": "F06.send-image", "family": "F06", "result": "pass", "label": "Image content (`F06.send-image`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-13-image-content-f06-send-image.json", "failed_command": null, "variants": [ { "feature_id": "F06.send-image", "family": "F06", "result": "pass", "label": "Image content (`F06.send-image`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-13-image-content-f06-send-image.json", "failed_command": null } ] }, { "feature_id": "F06.send-no-run", "family": "F06", "result": "pass", "label": "Append a message without running (`F06.send-no-run`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-01-append-a-message-without-running-f06-sen.json", "failed_command": null, "variants": [ { "feature_id": "F06.send-no-run", "family": "F06", "result": "pass", "label": "Append a message without running (`F06.send-no-run`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-01-append-a-message-without-running-f06-sen.json", "failed_command": null } ] }, { "feature_id": "F06.send-non-user-role", "family": "F06", "result": "xfail", "label": "Non-user role (`F06.send-non-user-role`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-14-non-user-role-f06-send-non-user-role-kno.json", "failed_command": "control-agent-server api POST \"/$CID/events\" --json '{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"QA_F06_ASSISTANT\"}]}' --expect 4xx --save F06.send-non-user-role/assistant", "variants": [ { "feature_id": "F06.send-non-user-role", "family": "F06", "result": "xfail", "label": "Non-user role (`F06.send-non-user-role`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-14-non-user-role-f06-send-non-user-role-kno.json", "failed_command": "control-agent-server api POST \"/$CID/events\" --json '{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"QA_F06_ASSISTANT\"}]}' --expect 4xx --save F06.send-non-user-role/assistant" } ] }, { "feature_id": "F06.send-resets-finished", "family": "F06", "result": "pass", "label": "A message reopens a finished conversation (`F06.send-resets-finished`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-24-a-message-reopens-a-finished-conversatio.json", "failed_command": null, "variants": [ { "feature_id": "F06.send-resets-finished", "family": "F06", "result": "pass", "label": "A message reopens a finished conversation (`F06.send-resets-finished`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-24-a-message-reopens-a-finished-conversatio.json", "failed_command": null } ] }, { "feature_id": "F06.send-run", "family": "F06", "result": "pass", "label": "Run a turn and watch it live (`F06.send-run`, `F06.ws-live`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-22-run-a-turn-and-watch-it-live-f06-send-ru.json", "failed_command": null, "variants": [ { "feature_id": "F06.send-run", "family": "F06", "result": "pass", "label": "Run a turn and watch it live (`F06.send-run`, `F06.ws-live`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-22-run-a-turn-and-watch-it-live-f06-send-ru.json", "failed_command": null } ] }, { "feature_id": "F06.send-run-limit", "family": "F06", "result": "pass", "label": "Run capacity full (`F06.send-run-limit`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-26-run-capacity-full-f06-send-run-limit.json", "failed_command": null, "variants": [ { "feature_id": "F06.send-run-limit", "family": "F06", "result": "pass", "label": "Run capacity full (`F06.send-run-limit`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-26-run-capacity-full-f06-send-run-limit.json", "failed_command": null } ] }, { "feature_id": "F06.send-validation", "family": "F06", "result": "pass", "label": "Message validation (`F06.send-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-12-message-validation-f06-send-validation.json", "failed_command": null, "variants": [ { "feature_id": "F06.send-validation", "family": "F06", "result": "pass", "label": "Message validation (`F06.send-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-12-message-validation-f06-send-validation.json", "failed_command": null } ] }, { "feature_id": "F06.send-while-running", "family": "F06", "result": "pass", "label": "A message while the agent runs (`F06.send-while-running`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-27-a-message-while-the-agent-runs-f06-send-.json", "failed_command": null, "variants": [ { "feature_id": "F06.send-while-running", "family": "F06", "result": "pass", "label": "A message while the agent runs (`F06.send-while-running`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-27-a-message-while-the-agent-runs-f06-send-.json", "failed_command": null } ] }, { "feature_id": "F06.unknown-event", "family": "F06", "result": "xfail", "label": "Unknown event id (`F06.unknown-event`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-09-unknown-event-id-f06-unknown-event-known.json", "failed_command": "control-agent-server api GET \"/$CID/events/00000000-0000-4000-8000-0000000000e6\" --expect 404 --save F06.unknown-event/get-one", "variants": [ { "feature_id": "F06.unknown-event", "family": "F06", "result": "xfail", "label": "Unknown event id (`F06.unknown-event`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-09-unknown-event-id-f06-unknown-event-known.json", "failed_command": "control-agent-server api GET \"/$CID/events/00000000-0000-4000-8000-0000000000e6\" --expect 404 --save F06.unknown-event/get-one" } ] }, { "feature_id": "F06.ws-bad-frame", "family": "F06", "result": "pass", "label": "Bad frames on the socket (`F06.ws-bad-frame`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-20-bad-frames-on-the-socket-f06-ws-bad-fram.json", "failed_command": null, "variants": [ { "feature_id": "F06.ws-bad-frame", "family": "F06", "result": "pass", "label": "Bad frames on the socket (`F06.ws-bad-frame`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-20-bad-frames-on-the-socket-f06-ws-bad-fram.json", "failed_command": null } ] }, { "feature_id": "F06.ws-delete-close", "family": "F06", "result": "xfail", "label": "Deleting a conversation closes its sockets (`F06.ws-delete-close`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-17-deleting-a-conversation-closes-its-socke.json", "failed_command": "false", "variants": [ { "feature_id": "F06.ws-delete-close", "family": "F06", "result": "xfail", "label": "Deleting a conversation closes its sockets (`F06.ws-delete-close`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-17-deleting-a-conversation-closes-its-socke.json", "failed_command": "false" } ] }, { "feature_id": "F06.ws-handshake", "family": "F06", "result": "pass", "label": "Handshake: auth and close codes (`F06.ws-handshake`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-16-handshake-auth-and-close-codes-f06-ws-ha.json", "failed_command": null, "variants": [ { "feature_id": "F06.ws-handshake", "family": "F06", "result": "pass", "label": "Handshake: auth and close codes (`F06.ws-handshake`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-16-handshake-auth-and-close-codes-f06-ws-ha.json", "failed_command": null } ] }, { "feature_id": "F06.ws-live", "family": "F06", "result": "pass", "label": "Run a turn and watch it live (`F06.send-run`, `F06.ws-live`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-22-run-a-turn-and-watch-it-live-f06-send-ru.json", "failed_command": null, "variants": [ { "feature_id": "F06.ws-live", "family": "F06", "result": "pass", "label": "Run a turn and watch it live (`F06.send-run`, `F06.ws-live`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-22-run-a-turn-and-watch-it-live-f06-send-ru.json", "failed_command": null } ] }, { "feature_id": "F06.ws-replay-all", "family": "F06", "result": "pass", "label": "Replay everything (`F06.ws-replay-all`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-18-replay-everything-f06-ws-replay-all.json", "failed_command": null, "variants": [ { "feature_id": "F06.ws-replay-all", "family": "F06", "result": "pass", "label": "Replay everything (`F06.ws-replay-all`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-18-replay-everything-f06-ws-replay-all.json", "failed_command": null } ] }, { "feature_id": "F06.ws-replay-since", "family": "F06", "result": "pass", "label": "Replay since a timestamp (`F06.ws-replay-since`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-19-replay-since-a-timestamp-f06-ws-replay-s.json", "failed_command": null, "variants": [ { "feature_id": "F06.ws-replay-since", "family": "F06", "result": "pass", "label": "Replay since a timestamp (`F06.ws-replay-since`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-19-replay-since-a-timestamp-f06-ws-replay-s.json", "failed_command": null } ] }, { "feature_id": "F06.ws-send", "family": "F06", "result": "pass", "label": "Send a message over the socket (`F06.ws-send`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-21-send-a-message-over-the-socket-f06-ws-se.json", "failed_command": null, "variants": [ { "feature_id": "F06.ws-send", "family": "F06", "result": "pass", "label": "Send a message over the socket (`F06.ws-send`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-21-send-a-message-over-the-socket-f06-ws-se.json", "failed_command": null } ] }, { "feature_id": "F06.ws-snapshot", "family": "F06", "result": "pass", "label": "First frame is a state snapshot (`F06.ws-snapshot`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-15-first-frame-is-a-state-snapshot-f06-ws-s.json", "failed_command": null, "variants": [ { "feature_id": "F06.ws-snapshot", "family": "F06", "result": "pass", "label": "First frame is a state snapshot (`F06.ws-snapshot`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-15-first-frame-is-a-state-snapshot-f06-ws-s.json", "failed_command": null } ] }, { "feature_id": "F07.auth", "family": "F07", "result": "pass", "label": "Authentication (`F07.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-10-authentication-f07-auth.json", "failed_command": null, "variants": [ { "feature_id": "F07.auth", "family": "F07", "result": "pass", "label": "Authentication (`F07.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-10-authentication-f07-auth.json", "failed_command": null } ] }, { "feature_id": "F07.auth-frame-ignored", "family": "F07", "result": "pass", "label": "Redundant auth frame (`F07.auth-frame-ignored`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-09-redundant-auth-frame-f07-auth-frame-igno.json", "failed_command": null, "variants": [ { "feature_id": "F07.auth-frame-ignored", "family": "F07", "result": "pass", "label": "Redundant auth frame (`F07.auth-frame-ignored`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-09-redundant-auth-frame-f07-auth-frame-igno.json", "failed_command": null } ] }, { "feature_id": "F07.bad-handshake", "family": "F07", "result": "pass", "label": "Malformed handshake (`F07.bad-handshake`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-12-malformed-handshake-f07-bad-handshake.json", "failed_command": null, "variants": [ { "feature_id": "F07.bad-handshake", "family": "F07", "result": "pass", "label": "Malformed handshake (`F07.bad-handshake`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-12-malformed-handshake-f07-bad-handshake.json", "failed_command": null } ] }, { "feature_id": "F07.deleted-close", "family": "F07", "result": "xfail", "label": "Deleting closes the socket (`F07.deleted-close`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-24-deleting-closes-the-socket-f07-deleted-c.json", "failed_command": "false", "variants": [ { "feature_id": "F07.deleted-close", "family": "F07", "result": "xfail", "label": "Deleting closes the socket (`F07.deleted-close`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-24-deleting-closes-the-socket-f07-deleted-c.json", "failed_command": "false" } ] }, { "feature_id": "F07.error-frame", "family": "F07", "result": "pass", "label": "Invalid message (`F07.error-frame`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-06-invalid-message-f07-error-frame.json", "failed_command": null, "variants": [ { "feature_id": "F07.error-frame", "family": "F07", "result": "pass", "label": "Invalid message (`F07.error-frame`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-06-invalid-message-f07-error-frame.json", "failed_command": null } ] }, { "feature_id": "F07.error-non-json", "family": "F07", "result": "pass", "label": "Non-JSON and binary frames (`F07.error-non-json`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-07-non-json-and-binary-frames-f07-error-non.json", "failed_command": null, "variants": [ { "feature_id": "F07.error-non-json", "family": "F07", "result": "pass", "label": "Non-JSON and binary frames (`F07.error-non-json`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-07-non-json-and-binary-frames-f07-error-non.json", "failed_command": null } ] }, { "feature_id": "F07.frame-too-large", "family": "F07", "result": "pass", "label": "Oversized event (`F07.frame-too-large`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-21-oversized-event-f07-frame-too-large.json", "failed_command": null, "variants": [ { "feature_id": "F07.frame-too-large", "family": "F07", "result": "pass", "label": "Oversized event (`F07.frame-too-large`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-21-oversized-event-f07-frame-too-large.json", "failed_command": null } ] }, { "feature_id": "F07.full-replay", "family": "F07", "result": "pass", "label": "Full replay (`F07.full-replay`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-03-full-replay-f07-full-replay.json", "failed_command": null, "variants": [ { "feature_id": "F07.full-replay", "family": "F07", "result": "pass", "label": "Full replay (`F07.full-replay`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-03-full-replay-f07-full-replay.json", "failed_command": null } ] }, { "feature_id": "F07.item-aborted", "family": "F07", "result": "pass", "label": "Interrupted stream (`F07.item-aborted`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-20-interrupted-stream-f07-item-aborted.json", "failed_command": null, "variants": [ { "feature_id": "F07.item-aborted", "family": "F07", "result": "pass", "label": "Interrupted stream (`F07.item-aborted`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-20-interrupted-stream-f07-item-aborted.json", "failed_command": null } ] }, { "feature_id": "F07.live-durable", "family": "F07", "result": "pass", "label": "Live durable frame (`F07.live-durable`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-13-live-durable-frame-f07-live-durable.json", "failed_command": null, "variants": [ { "feature_id": "F07.live-durable", "family": "F07", "result": "pass", "label": "Live durable frame (`F07.live-durable`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-13-live-durable-frame-f07-live-durable.json", "failed_command": null } ] }, { "feature_id": "F07.live-order", "family": "F07", "result": "xfail", "label": "Live frames in seq order (`F07.live-order`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-14-live-frames-in-seq-order-f07-live-order-.json", "failed_command": "control-agent-server state cat \"evidence/F07.live-order/order-$ORD.json\" --check frames.3.frame.seq eq 0 --check frames.4.frame.seq eq 1 --check frames.5.frame.seq eq 2", "variants": [ { "feature_id": "F07.live-order", "family": "F07", "result": "xfail", "label": "Live frames in seq order (`F07.live-order`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-14-live-frames-in-seq-order-f07-live-order-.json", "failed_command": "control-agent-server state cat \"evidence/F07.live-order/order-$ORD.json\" --check frames.3.frame.seq eq 0 --check frames.4.frame.seq eq 1 --check frames.5.frame.seq eq 2" } ] }, { "feature_id": "F07.progress-frames", "family": "F07", "result": "pass", "label": "Streaming progress frames (`F07.progress-frames`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-18-streaming-progress-frames-f07-progress-f.json", "failed_command": null, "variants": [ { "feature_id": "F07.progress-frames", "family": "F07", "result": "pass", "label": "Streaming progress frames (`F07.progress-frames`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-18-streaming-progress-frames-f07-progress-f.json", "failed_command": null } ] }, { "feature_id": "F07.progress-not-replayed", "family": "F07", "result": "pass", "label": "Progress is never replayed (`F07.progress-not-replayed`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-19-progress-is-never-replayed-f07-progress-.json", "failed_command": null, "variants": [ { "feature_id": "F07.progress-not-replayed", "family": "F07", "result": "pass", "label": "Progress is never replayed (`F07.progress-not-replayed`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-19-progress-is-never-replayed-f07-progress-.json", "failed_command": null } ] }, { "feature_id": "F07.reject-non-user", "family": "F07", "result": "pass", "label": "Non-user role (`F07.reject-non-user`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-08-non-user-role-f07-reject-non-user.json", "failed_command": null, "variants": [ { "feature_id": "F07.reject-non-user", "family": "F07", "result": "pass", "label": "Non-user role (`F07.reject-non-user`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-08-non-user-role-f07-reject-non-user.json", "failed_command": null } ] }, { "feature_id": "F07.replay-then-live", "family": "F07", "result": "pass", "label": "Replay and live tail, each event once (`F07.replay-then-live`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-15-replay-and-live-tail-each-event-once-f07.json", "failed_command": null, "variants": [ { "feature_id": "F07.replay-then-live", "family": "F07", "result": "pass", "label": "Replay and live tail, each event once (`F07.replay-then-live`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-15-replay-and-live-tail-each-event-once-f07.json", "failed_command": null } ] }, { "feature_id": "F07.restart-resume", "family": "F07", "result": "pass", "label": "Cursor survives a restart (`F07.restart-resume`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-17-cursor-survives-a-restart-f07-restart-re.json", "failed_command": null, "variants": [ { "feature_id": "F07.restart-resume", "family": "F07", "result": "pass", "label": "Cursor survives a restart (`F07.restart-resume`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-17-cursor-survives-a-restart-f07-restart-re.json", "failed_command": null } ] }, { "feature_id": "F07.resume-cursor", "family": "F07", "result": "pass", "label": "Resume from a cursor (`F07.resume-cursor`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-05-resume-from-a-cursor-f07-resume-cursor.json", "failed_command": null, "variants": [ { "feature_id": "F07.resume-cursor", "family": "F07", "result": "pass", "label": "Resume from a cursor (`F07.resume-cursor`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-05-resume-from-a-cursor-f07-resume-cursor.json", "failed_command": null } ] }, { "feature_id": "F07.send-message", "family": "F07", "result": "pass", "label": "Send a message on the socket (`F07.send-message`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-16-send-a-message-on-the-socket-f07-send-me.json", "failed_command": null, "variants": [ { "feature_id": "F07.send-message", "family": "F07", "result": "pass", "label": "Send a message on the socket (`F07.send-message`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-16-send-a-message-on-the-socket-f07-send-me.json", "failed_command": null } ] }, { "feature_id": "F07.seq-on-disk", "family": "F07", "result": "pass", "label": "Seq is the on-disk index (`F07.seq-on-disk`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-04-seq-is-the-on-disk-index-f07-seq-on-disk.json", "failed_command": null, "variants": [ { "feature_id": "F07.seq-on-disk", "family": "F07", "result": "pass", "label": "Seq is the on-disk index (`F07.seq-on-disk`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-04-seq-is-the-on-disk-index-f07-seq-on-disk.json", "failed_command": null } ] }, { "feature_id": "F07.slow-consumer", "family": "F07", "result": "pass", "label": "Stalled reader (`F07.slow-consumer`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-22-stalled-reader-f07-slow-consumer.json", "failed_command": null, "variants": [ { "feature_id": "F07.slow-consumer", "family": "F07", "result": "pass", "label": "Stalled reader (`F07.slow-consumer`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-22-stalled-reader-f07-slow-consumer.json", "failed_command": null } ] }, { "feature_id": "F07.snapshot-transient", "family": "F07", "result": "pass", "label": "State snapshot (`F07.snapshot-transient`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-02-state-snapshot-f07-snapshot-transient.json", "failed_command": null, "variants": [ { "feature_id": "F07.snapshot-transient", "family": "F07", "result": "pass", "label": "State snapshot (`F07.snapshot-transient`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-02-state-snapshot-f07-snapshot-transient.json", "failed_command": null } ] }, { "feature_id": "F07.subscriber-limit", "family": "F07", "result": "pass", "label": "Subscriber cap (`F07.subscriber-limit`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-23-subscriber-cap-f07-subscriber-limit.json", "failed_command": null, "variants": [ { "feature_id": "F07.subscriber-limit", "family": "F07", "result": "pass", "label": "Subscriber cap (`F07.subscriber-limit`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-23-subscriber-cap-f07-subscriber-limit.json", "failed_command": null } ] }, { "feature_id": "F07.sync-first", "family": "F07", "result": "pass", "label": "Sync comes first (`F07.sync-first`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-01-sync-comes-first-f07-sync-first.json", "failed_command": null, "variants": [ { "feature_id": "F07.sync-first", "family": "F07", "result": "pass", "label": "Sync comes first (`F07.sync-first`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-01-sync-comes-first-f07-sync-first.json", "failed_command": null } ] }, { "feature_id": "F07.unknown-conversation", "family": "F07", "result": "pass", "label": "Unknown conversation (`F07.unknown-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-11-unknown-conversation-f07-unknown-convers.json", "failed_command": null, "variants": [ { "feature_id": "F07.unknown-conversation", "family": "F07", "result": "pass", "label": "Unknown conversation (`F07.unknown-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-11-unknown-conversation-f07-unknown-convers.json", "failed_command": null } ] }, { "feature_id": "F08.always-confirm-wait", "family": "F08", "result": "pass", "label": "Wait for confirmation (`F08.always-confirm-wait`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-03-wait-for-confirmation-f08-always-confirm.json", "failed_command": null, "variants": [ { "feature_id": "F08.always-confirm-wait", "family": "F08", "result": "pass", "label": "Wait for confirmation (`F08.always-confirm-wait`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-03-wait-for-confirmation-f08-always-confirm.json", "failed_command": null } ] }, { "feature_id": "F08.analyzer-errors", "family": "F08", "result": "pass", "label": "Analyzer errors (`F08.analyzer-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-10-analyzer-errors-f08-analyzer-errors.json", "failed_command": null, "variants": [ { "feature_id": "F08.analyzer-errors", "family": "F08", "result": "pass", "label": "Analyzer errors (`F08.analyzer-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-10-analyzer-errors-f08-analyzer-errors.json", "failed_command": null } ] }, { "feature_id": "F08.analyzer-set-clear", "family": "F08", "result": "pass", "label": "Set and clear the analyzer (`F08.analyzer-set-clear`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-09-set-and-clear-the-analyzer-f08-analyzer-.json", "failed_command": null, "variants": [ { "feature_id": "F08.analyzer-set-clear", "family": "F08", "result": "pass", "label": "Set and clear the analyzer (`F08.analyzer-set-clear`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-09-set-and-clear-the-analyzer-f08-analyzer-.json", "failed_command": null } ] }, { "feature_id": "F08.confirm-accept", "family": "F08", "result": "pass", "label": "Accept (`F08.confirm-accept`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-04-accept-f08-confirm-accept.json", "failed_command": null, "variants": [ { "feature_id": "F08.confirm-accept", "family": "F08", "result": "pass", "label": "Accept (`F08.confirm-accept`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-04-accept-f08-confirm-accept.json", "failed_command": null } ] }, { "feature_id": "F08.confirm-capacity", "family": "F08", "result": "pass", "label": "Accept with the run capacity full (`F08.confirm-capacity`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-07-accept-with-the-run-capacity-full-f08-co.json", "failed_command": null, "variants": [ { "feature_id": "F08.confirm-capacity", "family": "F08", "result": "pass", "label": "Accept with the run capacity full (`F08.confirm-capacity`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-07-accept-with-the-run-capacity-full-f08-co.json", "failed_command": null } ] }, { "feature_id": "F08.confirm-errors", "family": "F08", "result": "pass", "label": "Confirmation errors (`F08.confirm-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-06-confirmation-errors-f08-confirm-errors.json", "failed_command": null, "variants": [ { "feature_id": "F08.confirm-errors", "family": "F08", "result": "pass", "label": "Confirmation errors (`F08.confirm-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-06-confirmation-errors-f08-confirm-errors.json", "failed_command": null } ] }, { "feature_id": "F08.confirm-real", "family": "F08", "result": "pass", "label": "A real model asks first (`F08.confirm-real`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-08-a-real-model-asks-first-f08-confirm-real.json", "failed_command": null, "variants": [ { "feature_id": "F08.confirm-real", "family": "F08", "result": "pass", "label": "A real model asks first (`F08.confirm-real`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-08-a-real-model-asks-first-f08-confirm-real.json", "failed_command": null } ] }, { "feature_id": "F08.confirm-reject", "family": "F08", "result": "pass", "label": "Reject (`F08.confirm-reject`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-05-reject-f08-confirm-reject.json", "failed_command": null, "variants": [ { "feature_id": "F08.confirm-reject", "family": "F08", "result": "pass", "label": "Reject (`F08.confirm-reject`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-05-reject-f08-confirm-reject.json", "failed_command": null } ] }, { "feature_id": "F08.confirm-risky", "family": "F08", "result": "pass", "label": "Risk-based confirmation (`F08.confirm-risky`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-11-risk-based-confirmation-f08-confirm-risk.json", "failed_command": null, "variants": [ { "feature_id": "F08.confirm-risky", "family": "F08", "result": "pass", "label": "Risk-based confirmation (`F08.confirm-risky`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-11-risk-based-confirmation-f08-confirm-risk.json", "failed_command": null } ] }, { "feature_id": "F08.policy-errors", "family": "F08", "result": "pass", "label": "Policy errors (`F08.policy-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-02-policy-errors-f08-policy-errors.json", "failed_command": null, "variants": [ { "feature_id": "F08.policy-errors", "family": "F08", "result": "pass", "label": "Policy errors (`F08.policy-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-02-policy-errors-f08-policy-errors.json", "failed_command": null } ] }, { "feature_id": "F08.policy-set", "family": "F08", "result": "pass", "label": "Set the confirmation policy (`F08.policy-set`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-01-set-the-confirmation-policy-f08-policy-s.json", "failed_command": null, "variants": [ { "feature_id": "F08.policy-set", "family": "F08", "result": "pass", "label": "Set the confirmation policy (`F08.policy-set`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-01-set-the-confirmation-policy-f08-policy-s.json", "failed_command": null } ] }, { "feature_id": "F08.secrets-bad-value", "family": "F08", "result": "xfail", "label": "Non-string secret value (`F08.secrets-bad-value`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-15-non-string-secret-value-f08-secrets-bad-.json", "failed_command": "control-agent-server api POST \"/$KID/secrets\" --json '{\"secrets\": {\"QA_F08_NUMBER\": 5}}' --expect 422 --save F08.secrets-bad-value/number", "variants": [ { "feature_id": "F08.secrets-bad-value", "family": "F08", "result": "xfail", "label": "Non-string secret value (`F08.secrets-bad-value`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-15-non-string-secret-value-f08-secrets-bad-.json", "failed_command": "control-agent-server api POST \"/$KID/secrets\" --json '{\"secrets\": {\"QA_F08_NUMBER\": 5}}' --expect 422 --save F08.secrets-bad-value/number" } ] }, { "feature_id": "F08.secrets-errors", "family": "F08", "result": "pass", "label": "Secret errors (`F08.secrets-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-14-secret-errors-f08-secrets-errors.json", "failed_command": null, "variants": [ { "feature_id": "F08.secrets-errors", "family": "F08", "result": "pass", "label": "Secret errors (`F08.secrets-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-14-secret-errors-f08-secrets-errors.json", "failed_command": null } ] }, { "feature_id": "F08.secrets-lookup", "family": "F08", "result": "pass", "label": "Lookup secret (`F08.secrets-lookup`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-18-lookup-secret-f08-secrets-lookup.json", "failed_command": null, "variants": [ { "feature_id": "F08.secrets-lookup", "family": "F08", "result": "pass", "label": "Lookup secret (`F08.secrets-lookup`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-18-lookup-secret-f08-secrets-lookup.json", "failed_command": null } ] }, { "feature_id": "F08.secrets-lookup-self", "family": "F08", "result": "pass", "label": "Lookup of the server's own secret (`F08.secrets-lookup-self`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-19-lookup-of-the-server-s-own-secret-f08-se.json", "failed_command": null, "variants": [ { "feature_id": "F08.secrets-lookup-self", "family": "F08", "result": "pass", "label": "Lookup of the server's own secret (`F08.secrets-lookup-self`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-19-lookup-of-the-server-s-own-secret-f08-se.json", "failed_command": null } ] }, { "feature_id": "F08.secrets-no-cipher", "family": "F08", "result": "pass", "label": "No cipher (`F08.secrets-no-cipher`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-21-no-cipher-f08-secrets-no-cipher.json", "failed_command": null, "variants": [ { "feature_id": "F08.secrets-no-cipher", "family": "F08", "result": "pass", "label": "No cipher (`F08.secrets-no-cipher`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-21-no-cipher-f08-secrets-no-cipher.json", "failed_command": null } ] }, { "feature_id": "F08.secrets-override", "family": "F08", "result": "pass", "label": "Override a start-time secret (`F08.secrets-override`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-17-override-a-start-time-secret-f08-secrets.json", "failed_command": null, "variants": [ { "feature_id": "F08.secrets-override", "family": "F08", "result": "pass", "label": "Override a start-time secret (`F08.secrets-override`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-17-override-a-start-time-secret-f08-secrets.json", "failed_command": null } ] }, { "feature_id": "F08.secrets-profile-scope", "family": "F08", "result": "pass", "label": "Agent Profile secret scope (`F08.secrets-profile-scope`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-20-agent-profile-secret-scope-f08-secrets-p.json", "failed_command": null, "variants": [ { "feature_id": "F08.secrets-profile-scope", "family": "F08", "result": "pass", "label": "Agent Profile secret scope (`F08.secrets-profile-scope`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-20-agent-profile-secret-scope-f08-secrets-p.json", "failed_command": null } ] }, { "feature_id": "F08.secrets-restart", "family": "F08", "result": "fail", "label": "Restart with a saved secret (`F08.secrets-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-22-restart-with-a-saved-secret-f08-secrets-.json", "failed_command": "control-agent-server doctor", "variants": [ { "feature_id": "F08.secrets-restart", "family": "F08", "result": "fail", "label": "Restart with a saved secret (`F08.secrets-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-22-restart-with-a-saved-secret-f08-secrets-.json", "failed_command": "control-agent-server doctor" } ] }, { "feature_id": "F08.secrets-terminal", "family": "F08", "result": "pass", "label": "Secrets in the terminal (`F08.secrets-terminal`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-16-secrets-in-the-terminal-f08-secrets-term.json", "failed_command": null, "variants": [ { "feature_id": "F08.secrets-terminal", "family": "F08", "result": "pass", "label": "Secrets in the terminal (`F08.secrets-terminal`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-16-secrets-in-the-terminal-f08-secrets-term.json", "failed_command": null } ] }, { "feature_id": "F08.secrets-unsaved-restart", "family": "F08", "result": "xfail", "label": "Unsaved secret restart (`F08.secrets-unsaved-restart`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-23-unsaved-secret-restart-f08-secrets-unsav.json", "failed_command": "control-agent-server api GET \"/$UNS\" --check secret_registry.secret_sources.QA_F08_UNSAVED.kind eq StaticSecret --save F08.secrets-unsaved-restart/get", "variants": [ { "feature_id": "F08.secrets-unsaved-restart", "family": "F08", "result": "xfail", "label": "Unsaved secret restart (`F08.secrets-unsaved-restart`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-23-unsaved-secret-restart-f08-secrets-unsav.json", "failed_command": "control-agent-server api GET \"/$UNS\" --check secret_registry.secret_sources.QA_F08_UNSAVED.kind eq StaticSecret --save F08.secrets-unsaved-restart/get" } ] }, { "feature_id": "F08.secrets-update", "family": "F08", "result": "pass", "label": "Update secrets (`F08.secrets-update`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-12-update-secrets-f08-secrets-update.json", "failed_command": null, "variants": [ { "feature_id": "F08.secrets-update", "family": "F08", "result": "pass", "label": "Update secrets (`F08.secrets-update`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-12-update-secrets-f08-secrets-update.json", "failed_command": null } ] }, { "feature_id": "F08.secrets-visible", "family": "F08", "result": "xfail", "label": "Runtime secret visible to GET (`F08.secrets-visible`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-13-runtime-secret-visible-to-get-f08-secret.json", "failed_command": "control-agent-server api GET \"/$KID\" --check secret_registry.secret_sources.QA_F08_FRESH.kind eq StaticSecret --save F08.secrets-visible/get", "variants": [ { "feature_id": "F08.secrets-visible", "family": "F08", "result": "xfail", "label": "Runtime secret visible to GET (`F08.secrets-visible`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-13-runtime-secret-visible-to-get-f08-secret.json", "failed_command": "control-agent-server api GET \"/$KID\" --check secret_registry.secret_sources.QA_F08_FRESH.kind eq StaticSecret --save F08.secrets-visible/get" } ] }, { "feature_id": "F09.acp-model-blank", "family": "F09", "result": "xfail", "label": "Blank ACP model (`F09.acp-model-blank`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-16-blank-acp-model-f09-acp-model-blank-know.json", "failed_command": "control-agent-server api POST \"/$AID2/switch_acp_model\" --json '{\"model\": \" \"}' --expect 400,422 --save F09.acp-model-blank/blank", "variants": [ { "feature_id": "F09.acp-model-blank", "family": "F09", "result": "xfail", "label": "Blank ACP model (`F09.acp-model-blank`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-16-blank-acp-model-f09-acp-model-blank-know.json", "failed_command": "control-agent-server api POST \"/$AID2/switch_acp_model\" --json '{\"model\": \" \"}' --expect 400,422 --save F09.acp-model-blank/blank" } ] }, { "feature_id": "F09.acp-model-deferred", "family": "F09", "result": "pass", "label": "ACP model before the first run (`F09.acp-model-deferred`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-15-acp-model-before-the-first-run-f09-acp-m.json", "failed_command": null, "variants": [ { "feature_id": "F09.acp-model-deferred", "family": "F09", "result": "pass", "label": "ACP model before the first run (`F09.acp-model-deferred`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-15-acp-model-before-the-first-run-f09-acp-m.json", "failed_command": null } ] }, { "feature_id": "F09.acp-model-live", "family": "F09", "result": "pass", "label": "Live ACP switch (`F09.acp-model-live`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-18-live-acp-switch-f09-acp-model-live.json", "failed_command": null, "variants": [ { "feature_id": "F09.acp-model-live", "family": "F09", "result": "pass", "label": "Live ACP switch (`F09.acp-model-live`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-18-live-acp-switch-f09-acp-model-live.json", "failed_command": null } ] }, { "feature_id": "F09.acp-model-live-timeout", "family": "F09", "result": "pass", "label": "Live switch that times out (`F09.acp-model-live-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-20-live-switch-that-times-out-f09-acp-model.json", "failed_command": null, "variants": [ { "feature_id": "F09.acp-model-live-timeout", "family": "F09", "result": "pass", "label": "Live switch that times out (`F09.acp-model-live-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-20-live-switch-that-times-out-f09-acp-model.json", "failed_command": null } ] }, { "feature_id": "F09.acp-model-live-unknown", "family": "F09", "result": "xfail", "label": "Unknown model on a live session (`F09.acp-model-live-unknown`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-19-unknown-model-on-a-live-session-f09-acp-.json", "failed_command": "control-agent-server api POST \"/$LID/switch_acp_model\" --json '{\"model\": \"qa-f09-nope\"}' --expect 400,422 --save F09.acp-model-live-unknown/switch", "variants": [ { "feature_id": "F09.acp-model-live-unknown", "family": "F09", "result": "xfail", "label": "Unknown model on a live session (`F09.acp-model-live-unknown`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-19-unknown-model-on-a-live-session-f09-acp-.json", "failed_command": "control-agent-server api POST \"/$LID/switch_acp_model\" --json '{\"model\": \"qa-f09-nope\"}' --expect 400,422 --save F09.acp-model-live-unknown/switch" } ] }, { "feature_id": "F09.acp-model-native", "family": "F09", "result": "pass", "label": "ACP switch on a regular conversation (`F09.acp-model-native`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-14-acp-switch-on-a-regular-conversation-f09.json", "failed_command": null, "variants": [ { "feature_id": "F09.acp-model-native", "family": "F09", "result": "pass", "label": "ACP switch on a regular conversation (`F09.acp-model-native`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-14-acp-switch-on-a-regular-conversation-f09.json", "failed_command": null } ] }, { "feature_id": "F09.auth-required", "family": "F09", "result": "pass", "label": "Key required (`F09.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-01-key-required-f09-auth-required.json", "failed_command": null, "variants": [ { "feature_id": "F09.auth-required", "family": "F09", "result": "pass", "label": "Key required (`F09.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-01-key-required-f09-auth-required.json", "failed_command": null } ] }, { "feature_id": "F09.load-plugin", "family": "F09", "result": "pass", "label": "Load a plugin (`F09.load-plugin`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-23-load-a-plugin-f09-load-plugin.json", "failed_command": null, "variants": [ { "feature_id": "F09.load-plugin", "family": "F09", "result": "pass", "label": "Load a plugin (`F09.load-plugin`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-23-load-a-plugin-f09-load-plugin.json", "failed_command": null } ] }, { "feature_id": "F09.load-plugin-errors", "family": "F09", "result": "pass", "label": "Plugin load errors (`F09.load-plugin-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-25-plugin-load-errors-f09-load-plugin-error.json", "failed_command": null, "variants": [ { "feature_id": "F09.load-plugin-errors", "family": "F09", "result": "pass", "label": "Plugin load errors (`F09.load-plugin-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-25-plugin-load-errors-f09-load-plugin-error.json", "failed_command": null } ] }, { "feature_id": "F09.load-plugin-hook-runs", "family": "F09", "result": "pass", "label": "The loaded hook runs (`F09.load-plugin-hook-runs`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-24-the-loaded-hook-runs-f09-load-plugin-hoo.json", "failed_command": null, "variants": [ { "feature_id": "F09.load-plugin-hook-runs", "family": "F09", "result": "pass", "label": "The loaded hook runs (`F09.load-plugin-hook-runs`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-24-the-loaded-hook-runs-f09-load-plugin-hoo.json", "failed_command": null } ] }, { "feature_id": "F09.load-plugin-sdk", "family": "F09", "result": "pass", "label": "Load a plugin from the Python SDK (`F09.load-plugin-sdk`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-27-load-a-plugin-from-the-python-sdk-f09-lo.json", "failed_command": null, "variants": [ { "feature_id": "F09.load-plugin-sdk", "family": "F09", "result": "pass", "label": "Load a plugin from the Python SDK (`F09.load-plugin-sdk`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-27-load-a-plugin-from-the-python-sdk-f09-lo.json", "failed_command": null } ] }, { "feature_id": "F09.load-plugin-twice", "family": "F09", "result": "xfail", "label": "Load the same plugin twice (`F09.load-plugin-twice`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-26-load-the-same-plugin-twice-f09-load-plug.json", "failed_command": "control-agent-server api GET \"/$QID\" --quiet --check hook_config.user_prompt_submit len-eq 1 --save F09.load-plugin-twice/get", "variants": [ { "feature_id": "F09.load-plugin-twice", "family": "F09", "result": "xfail", "label": "Load the same plugin twice (`F09.load-plugin-twice`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-26-load-the-same-plugin-twice-f09-load-plug.json", "failed_command": "control-agent-server api GET \"/$QID\" --quiet --check hook_config.user_prompt_submit len-eq 1 --save F09.load-plugin-twice/get" } ] }, { "feature_id": "F09.restart-persist", "family": "F09", "result": "pass", "label": "Restart (`F09.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-28-restart-f09-restart-persist.json", "failed_command": null, "variants": [ { "feature_id": "F09.restart-persist", "family": "F09", "result": "pass", "label": "Restart (`F09.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-28-restart-f09-restart-persist.json", "failed_command": null } ] }, { "feature_id": "F09.switch-llm", "family": "F09", "result": "pass", "label": "Switch to a caller-supplied LLM (`F09.switch-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-02-switch-to-a-caller-supplied-llm-f09-swit.json", "failed_command": null, "variants": [ { "feature_id": "F09.switch-llm", "family": "F09", "result": "pass", "label": "Switch to a caller-supplied LLM (`F09.switch-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-02-switch-to-a-caller-supplied-llm-f09-swit.json", "failed_command": null } ] }, { "feature_id": "F09.switch-llm-acp-refused", "family": "F09", "result": "xfail", "label": "LLM switches on an ACP conversation (`F09.switch-llm-acp-refused`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-17-llm-switches-on-an-acp-conversation-f09-.json", "failed_command": "control-agent-server api POST \"/$AID3/switch_profile\" --json '{\"profile_name\": \"deepseek-pro\"}' --expect 400,409,422 --save F09.switch-llm-acp-refused/switch-profile", "variants": [ { "feature_id": "F09.switch-llm-acp-refused", "family": "F09", "result": "xfail", "label": "LLM switches on an ACP conversation (`F09.switch-llm-acp-refused`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-17-llm-switches-on-an-acp-conversation-f09-.json", "failed_command": "control-agent-server api POST \"/$AID3/switch_profile\" --json '{\"profile_name\": \"deepseek-pro\"}' --expect 400,409,422 --save F09.switch-llm-acp-refused/switch-profile" } ] }, { "feature_id": "F09.switch-llm-before-first-run", "family": "F09", "result": "xfail", "label": "Switch before the first run (`F09.switch-llm-before-first-run`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-11-switch-before-the-first-run-f09-switch-l.json", "failed_command": "control-agent-server api GET \"/$BID\" --until-ok 20 --quiet --check stats.usage_to_metrics.qa-f09-early.accumulated_token_usage.prompt_tokens gt 0 --save F09.switch-llm-before-first-run/stats", "variants": [ { "feature_id": "F09.switch-llm-before-first-run", "family": "F09", "result": "xfail", "label": "Switch before the first run (`F09.switch-llm-before-first-run`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-11-switch-before-the-first-run-f09-switch-l.json", "failed_command": "control-agent-server api GET \"/$BID\" --until-ok 20 --quiet --check stats.usage_to_metrics.qa-f09-early.accumulated_token_usage.prompt_tokens gt 0 --save F09.switch-llm-before-first-run/stats" } ] }, { "feature_id": "F09.switch-llm-encrypted-key", "family": "F09", "result": "pass", "label": "Mid-conversation switch with a cipher token (`F09.switch-llm-encrypted-key`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-05-mid-conversation-switch-with-a-cipher-to.json", "failed_command": null, "variants": [ { "feature_id": "F09.switch-llm-encrypted-key", "family": "F09", "result": "pass", "label": "Mid-conversation switch with a cipher token (`F09.switch-llm-encrypted-key`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-05-mid-conversation-switch-with-a-cipher-to.json", "failed_command": null } ] }, { "feature_id": "F09.switch-llm-errors", "family": "F09", "result": "pass", "label": "Switch errors (`F09.switch-llm-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-04-switch-errors-f09-switch-llm-errors.json", "failed_command": null, "variants": [ { "feature_id": "F09.switch-llm-errors", "family": "F09", "result": "pass", "label": "Switch errors (`F09.switch-llm-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-04-switch-errors-f09-switch-llm-errors.json", "failed_command": null } ] }, { "feature_id": "F09.switch-llm-mid-run", "family": "F09", "result": "fail", "label": "Switch during a running turn (`F09.switch-llm-mid-run`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-12-switch-during-a-running-turn-f09-switch-.json", "failed_command": "control-agent-server ws read qa-f09-midrun --kinds ActionEvent --contains 'sleep 20' --expect-min 1 --wait 120", "variants": [ { "feature_id": "F09.switch-llm-mid-run", "family": "F09", "result": "fail", "label": "Switch during a running turn (`F09.switch-llm-mid-run`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-12-switch-during-a-running-turn-f09-switch-.json", "failed_command": "control-agent-server ws read qa-f09-midrun --kinds ActionEvent --contains 'sleep 20' --expect-min 1 --wait 120" } ] }, { "feature_id": "F09.switch-llm-mid-run-get", "family": "F09", "result": "fail", "label": "GET right after a mid-run switch (`F09.switch-llm-mid-run-get`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-13-get-right-after-a-mid-run-switch-f09-swi.json", "failed_command": "control-agent-server ws read qa-f09-midget --kinds ActionEvent --contains 'sleep 20' --expect-min 1 --wait 120", "variants": [ { "feature_id": "F09.switch-llm-mid-run-get", "family": "F09", "result": "fail", "label": "GET right after a mid-run switch (`F09.switch-llm-mid-run-get`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-13-get-right-after-a-mid-run-switch-f09-swi.json", "failed_command": "control-agent-server ws read qa-f09-midget --kinds ActionEvent --contains 'sleep 20' --expect-min 1 --wait 120" } ] }, { "feature_id": "F09.switch-llm-provider-connection", "family": "F09", "result": "pass", "label": "Switch through a provider connection (`F09.switch-llm-provider-connection`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-06-switch-through-a-provider-connection-f09.json", "failed_command": null, "variants": [ { "feature_id": "F09.switch-llm-provider-connection", "family": "F09", "result": "pass", "label": "Switch through a provider connection (`F09.switch-llm-provider-connection`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-06-switch-through-a-provider-connection-f09.json", "failed_command": null } ] }, { "feature_id": "F09.switch-llm-registered-usage-id", "family": "F09", "result": "xfail", "label": "Switch onto a registered usage id (`F09.switch-llm-registered-usage-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-10-switch-onto-a-registered-usage-id-f09-sw.json", "failed_command": "control-agent-server api GET \"/$KID\" --quiet --check agent.llm.model eq deepseek/deepseek-v4-pro --save F09.switch-llm-registered-usage-id/get", "variants": [ { "feature_id": "F09.switch-llm-registered-usage-id", "family": "F09", "result": "xfail", "label": "Switch onto a registered usage id (`F09.switch-llm-registered-usage-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-10-switch-onto-a-registered-usage-id-f09-sw.json", "failed_command": "control-agent-server api GET \"/$KID\" --quiet --check agent.llm.model eq deepseek/deepseek-v4-pro --save F09.switch-llm-registered-usage-id/get" } ] }, { "feature_id": "F09.switch-llm-ws", "family": "F09", "result": "pass", "label": "Socket view of a switch (`F09.switch-llm-ws`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-03-socket-view-of-a-switch-f09-switch-llm-w.json", "failed_command": null, "variants": [ { "feature_id": "F09.switch-llm-ws", "family": "F09", "result": "pass", "label": "Socket view of a switch (`F09.switch-llm-ws`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-03-socket-view-of-a-switch-f09-switch-llm-w.json", "failed_command": null } ] }, { "feature_id": "F09.switch-profile", "family": "F09", "result": "pass", "label": "Switch to a saved profile (`F09.switch-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-07-switch-to-a-saved-profile-f09-switch-pro.json", "failed_command": null, "variants": [ { "feature_id": "F09.switch-profile", "family": "F09", "result": "pass", "label": "Switch to a saved profile (`F09.switch-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-07-switch-to-a-saved-profile-f09-switch-pro.json", "failed_command": null } ] }, { "feature_id": "F09.switch-profile-edited", "family": "F09", "result": "xfail", "label": "Switch to an edited profile (`F09.switch-profile-edited`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-09-switch-to-an-edited-profile-f09-switch-p.json", "failed_command": "control-agent-server api GET \"/$EID\" --quiet --check agent.llm.model eq openai/qa-f09-edit-2 --save F09.switch-profile-edited/get", "variants": [ { "feature_id": "F09.switch-profile-edited", "family": "F09", "result": "xfail", "label": "Switch to an edited profile (`F09.switch-profile-edited`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-09-switch-to-an-edited-profile-f09-switch-p.json", "failed_command": "control-agent-server api GET \"/$EID\" --quiet --check agent.llm.model eq openai/qa-f09-edit-2 --save F09.switch-profile-edited/get" } ] }, { "feature_id": "F09.switch-profile-errors", "family": "F09", "result": "pass", "label": "Profile switch errors (`F09.switch-profile-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-08-profile-switch-errors-f09-switch-profile.json", "failed_command": null, "variants": [ { "feature_id": "F09.switch-profile-errors", "family": "F09", "result": "pass", "label": "Profile switch errors (`F09.switch-profile-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-08-profile-switch-errors-f09-switch-profile.json", "failed_command": null } ] }, { "feature_id": "F09.ts-client", "family": "F09", "result": "pass", "label": "TypeScript client (`F09.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-21-typescript-client-f09-ts-client.json", "failed_command": null, "variants": [ { "feature_id": "F09.ts-client", "family": "F09", "result": "pass", "label": "TypeScript client (`F09.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-21-typescript-client-f09-ts-client.json", "failed_command": null } ] }, { "feature_id": "F09.ts-client-acp-docs", "family": "F09", "result": "xfail", "label": "TypeScript docstrings for a pre-run ACP switch (`F09.ts-client-acp-docs`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-22-typescript-docstrings-for-a-pre-run-acp-.json", "failed_command": "false", "variants": [ { "feature_id": "F09.ts-client-acp-docs", "family": "F09", "result": "xfail", "label": "TypeScript docstrings for a pre-run ACP switch (`F09.ts-client-acp-docs`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-22-typescript-docstrings-for-a-pre-run-acp-.json", "failed_command": "false" } ] }, { "feature_id": "F10.ask-agent", "family": "F10", "result": "pass", "label": "Ask a side question (`F10.ask-agent`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-20-ask-a-side-question-f10-ask-agent.json", "failed_command": null, "variants": [ { "feature_id": "F10.ask-agent", "family": "F10", "result": "pass", "label": "Ask a side question (`F10.ask-agent`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-20-ask-a-side-question-f10-ask-agent.json", "failed_command": null } ] }, { "feature_id": "F10.ask-agent-errors", "family": "F10", "result": "pass", "label": "Bad input, no key, unreachable model (`F10.ask-agent-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-21-bad-input-no-key-unreachable-model-f10-a.json", "failed_command": null, "variants": [ { "feature_id": "F10.ask-agent-errors", "family": "F10", "result": "pass", "label": "Bad input, no key, unreachable model (`F10.ask-agent-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-21-bad-input-no-key-unreachable-model-f10-a.json", "failed_command": null } ] }, { "feature_id": "F10.ask-agent-running", "family": "F10", "result": "pass", "label": "Side question during a run (`F10.ask-agent-running`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-22-side-question-during-a-run-f10-ask-agent.json", "failed_command": null, "variants": [ { "feature_id": "F10.ask-agent-running", "family": "F10", "result": "pass", "label": "Side question during a run (`F10.ask-agent-running`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-22-side-question-during-a-run-f10-ask-agent.json", "failed_command": null } ] }, { "feature_id": "F10.ask-agent-unknown", "family": "F10", "result": "xfail", "label": "Unknown conversation (`F10.ask-agent-unknown`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-23-unknown-conversation-f10-ask-agent-unkno.json", "failed_command": "control-agent-server api POST \"/$NOPE/ask_agent\" --json '{\"question\": \"hi\"}' --expect 404 --save F10.ask-agent-unknown/unknown", "variants": [ { "feature_id": "F10.ask-agent-unknown", "family": "F10", "result": "xfail", "label": "Unknown conversation (`F10.ask-agent-unknown`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-23-unknown-conversation-f10-ask-agent-unkno.json", "failed_command": "control-agent-server api POST \"/$NOPE/ask_agent\" --json '{\"question\": \"hi\"}' --expect 404 --save F10.ask-agent-unknown/unknown" } ] }, { "feature_id": "F10.condense-errors", "family": "F10", "result": "pass", "label": "Unknown, malformed and unauthenticated (`F10.condense-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-19-unknown-malformed-and-unauthenticated-f1.json", "failed_command": null, "variants": [ { "feature_id": "F10.condense-errors", "family": "F10", "result": "pass", "label": "Unknown, malformed and unauthenticated (`F10.condense-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-19-unknown-malformed-and-unauthenticated-f1.json", "failed_command": null } ] }, { "feature_id": "F10.condense-llm", "family": "F10", "result": "pass", "label": "Condense with DeepSeek (`F10.condense-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-15-condense-with-deepseek-f10-condense-llm.json", "failed_command": null, "variants": [ { "feature_id": "F10.condense-llm", "family": "F10", "result": "pass", "label": "Condense with DeepSeek (`F10.condense-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-15-condense-with-deepseek-f10-condense-llm.json", "failed_command": null } ] }, { "feature_id": "F10.condense-llm-failure", "family": "F10", "result": "pass", "label": "Unreachable condenser model (`F10.condense-llm-failure`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-18-unreachable-condenser-model-f10-condense.json", "failed_command": null, "variants": [ { "feature_id": "F10.condense-llm-failure", "family": "F10", "result": "pass", "label": "Unreachable condenser model (`F10.condense-llm-failure`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-18-unreachable-condenser-model-f10-condense.json", "failed_command": null } ] }, { "feature_id": "F10.condense-no-condenser", "family": "F10", "result": "pass", "label": "No request-handling condenser (`F10.condense-no-condenser`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-16-no-request-handling-condenser-f10-conden.json", "failed_command": null, "variants": [ { "feature_id": "F10.condense-no-condenser", "family": "F10", "result": "pass", "label": "No request-handling condenser (`F10.condense-no-condenser`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-16-no-request-handling-condenser-f10-conden.json", "failed_command": null } ] }, { "feature_id": "F10.condense-status", "family": "F10", "result": "xfail", "label": "Refusal status (`F10.condense-status`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-17-refusal-status-f10-condense-status-known.json", "failed_command": "control-agent-server api POST \"/$NC/condense\" --expect 4xx --save F10.condense-status/disabled", "variants": [ { "feature_id": "F10.condense-status", "family": "F10", "result": "xfail", "label": "Refusal status (`F10.condense-status`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-17-refusal-status-f10-condense-status-known.json", "failed_command": "control-agent-server api POST \"/$NC/condense\" --expect 4xx --save F10.condense-status/disabled" } ] }, { "feature_id": "F10.fork-errors", "family": "F10", "result": "pass", "label": "Unknown, taken and invalid input (`F10.fork-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-06-unknown-taken-and-invalid-input-f10-fork.json", "failed_command": null, "variants": [ { "feature_id": "F10.fork-errors", "family": "F10", "result": "pass", "label": "Unknown, taken and invalid input (`F10.fork-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-06-unknown-taken-and-invalid-input-f10-fork.json", "failed_command": null } ] }, { "feature_id": "F10.fork-from-event", "family": "F10", "result": "pass", "label": "Fork from an event (`F10.fork-from-event`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-03-fork-from-an-event-f10-fork-from-event.json", "failed_command": null, "variants": [ { "feature_id": "F10.fork-from-event", "family": "F10", "result": "pass", "label": "Fork from an event (`F10.fork-from-event`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-03-fork-from-an-event-f10-fork-from-event.json", "failed_command": null } ] }, { "feature_id": "F10.fork-independent", "family": "F10", "result": "pass", "label": "Fork and source diverge (`F10.fork-independent`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-05-fork-and-source-diverge-f10-fork-indepen.json", "failed_command": null, "variants": [ { "feature_id": "F10.fork-independent", "family": "F10", "result": "pass", "label": "Fork and source diverge (`F10.fork-independent`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-05-fork-and-source-diverge-f10-fork-indepen.json", "failed_command": null } ] }, { "feature_id": "F10.fork-keeps-credentials", "family": "F10", "result": "xfail", "label": "The fork keeps the model keys (`F10.fork-keeps-credentials`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-08-the-fork-keeps-the-model-keys-f10-fork-k.json", "failed_command": "control-agent-server state cat base_state.json --conversation \"$KF\" --max-chars 200 --check agent.llm.api_key exists --check agent.condenser.llm.api_key exists", "variants": [ { "feature_id": "F10.fork-keeps-credentials", "family": "F10", "result": "xfail", "label": "The fork keeps the model keys (`F10.fork-keeps-credentials`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-08-the-fork-keeps-the-model-keys-f10-fork-k.json", "failed_command": "control-agent-server state cat base_state.json --conversation \"$KF\" --max-chars 200 --check agent.llm.api_key exists --check agent.condenser.llm.api_key exists" } ] }, { "feature_id": "F10.fork-metrics", "family": "F10", "result": "pass", "label": "Metrics on a fork (`F10.fork-metrics`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-04-metrics-on-a-fork-f10-fork-metrics.json", "failed_command": null, "variants": [ { "feature_id": "F10.fork-metrics", "family": "F10", "result": "pass", "label": "Metrics on a fork (`F10.fork-metrics`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-04-metrics-on-a-fork-f10-fork-metrics.json", "failed_command": null } ] }, { "feature_id": "F10.fork-named", "family": "F10", "result": "pass", "label": "Named fork (`F10.fork-named`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-02-named-fork-f10-fork-named.json", "failed_command": null, "variants": [ { "feature_id": "F10.fork-named", "family": "F10", "result": "pass", "label": "Named fork (`F10.fork-named`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-02-named-fork-f10-fork-named.json", "failed_command": null } ] }, { "feature_id": "F10.fork-run-memory", "family": "F10", "result": "xfail", "label": "A fork runs with the source's memory (`F10.fork-run-memory`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-09-a-fork-runs-with-the-source-s-memory-f10.json", "failed_command": "control-agent-server conversation events \"$AF\" --kinds ConversationErrorEvent --contains LLMAuthenticationError --expect-count 0 --save F10.fork-run-memory/auth-errors", "variants": [ { "feature_id": "F10.fork-run-memory", "family": "F10", "result": "xfail", "label": "A fork runs with the source's memory (`F10.fork-run-memory`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-09-a-fork-runs-with-the-source-s-memory-f10.json", "failed_command": "control-agent-server conversation events \"$AF\" --kinds ConversationErrorEvent --contains LLMAuthenticationError --expect-count 0 --save F10.fork-run-memory/auth-errors" } ] }, { "feature_id": "F10.fork-tags-consistent", "family": "F10", "result": "xfail", "label": "Fork tags agree with the stored metadata (`F10.fork-tags-consistent`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-07-fork-tags-agree-with-the-stored-metadata.json", "failed_command": "control-agent-server state cat meta.json --conversation \"$TF\" --check title eq 'QA F10 tag fork' --check tags.qa eq \"$TF_QA\"", "variants": [ { "feature_id": "F10.fork-tags-consistent", "family": "F10", "result": "xfail", "label": "Fork tags agree with the stored metadata (`F10.fork-tags-consistent`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-07-fork-tags-agree-with-the-stored-metadata.json", "failed_command": "control-agent-server state cat meta.json --conversation \"$TF\" --check title eq 'QA F10 tag fork' --check tags.qa eq \"$TF_QA\"" } ] }, { "feature_id": "F10.fork-whole", "family": "F10", "result": "pass", "label": "Fork the whole log (`F10.fork-whole`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-01-fork-the-whole-log-f10-fork-whole.json", "failed_command": null, "variants": [ { "feature_id": "F10.fork-whole", "family": "F10", "result": "pass", "label": "Fork the whole log (`F10.fork-whole`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-01-fork-the-whole-log-f10-fork-whole.json", "failed_command": null } ] }, { "feature_id": "F10.navigate-branch", "family": "F10", "result": "pass", "label": "The next event branches (`F10.navigate-branch`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-11-the-next-event-branches-f10-navigate-bra.json", "failed_command": null, "variants": [ { "feature_id": "F10.navigate-branch", "family": "F10", "result": "pass", "label": "The next event branches (`F10.navigate-branch`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-11-the-next-event-branches-f10-navigate-bra.json", "failed_command": null } ] }, { "feature_id": "F10.navigate-empty", "family": "F10", "result": "pass", "label": "Navigate to the empty tree (`F10.navigate-empty`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-12-navigate-to-the-empty-tree-f10-navigate-.json", "failed_command": null, "variants": [ { "feature_id": "F10.navigate-empty", "family": "F10", "result": "pass", "label": "Navigate to the empty tree (`F10.navigate-empty`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-12-navigate-to-the-empty-tree-f10-navigate-.json", "failed_command": null } ] }, { "feature_id": "F10.navigate-errors", "family": "F10", "result": "pass", "label": "Unknown, missing and unauthenticated (`F10.navigate-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-13-unknown-missing-and-unauthenticated-f10-.json", "failed_command": null, "variants": [ { "feature_id": "F10.navigate-errors", "family": "F10", "result": "pass", "label": "Unknown, missing and unauthenticated (`F10.navigate-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-13-unknown-missing-and-unauthenticated-f10-.json", "failed_command": null } ] }, { "feature_id": "F10.navigate-event", "family": "F10", "result": "pass", "label": "Navigate to an earlier event (`F10.navigate-event`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-10-navigate-to-an-earlier-event-f10-navigat.json", "failed_command": null, "variants": [ { "feature_id": "F10.navigate-event", "family": "F10", "result": "pass", "label": "Navigate to an earlier event (`F10.navigate-event`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-10-navigate-to-an-earlier-event-f10-navigat.json", "failed_command": null } ] }, { "feature_id": "F10.navigate-final-response", "family": "F10", "result": "xfail", "label": "Final response after navigating back (`F10.navigate-final-response`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-14-final-response-after-navigating-back-f10.json", "failed_command": "control-agent-server api GET \"/$AN/agent_final_response\" --check response eq '' --save F10.navigate-final-response/after-navigate", "variants": [ { "feature_id": "F10.navigate-final-response", "family": "F10", "result": "xfail", "label": "Final response after navigating back (`F10.navigate-final-response`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-14-final-response-after-navigating-back-f10.json", "failed_command": "control-agent-server api GET \"/$AN/agent_final_response\" --check response eq '' --save F10.navigate-final-response/after-navigate" } ] }, { "feature_id": "F10.restart-cold-calls", "family": "F10", "result": "pass", "label": "Condense and ask on a cold conversation (`F10.restart-cold-calls`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-25-condense-and-ask-on-a-cold-conversation-.json", "failed_command": null, "variants": [ { "feature_id": "F10.restart-cold-calls", "family": "F10", "result": "pass", "label": "Condense and ask on a cold conversation (`F10.restart-cold-calls`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-25-condense-and-ask-on-a-cold-conversation-.json", "failed_command": null } ] }, { "feature_id": "F10.restart-persist", "family": "F10", "result": "fail", "label": "State after a restart (`F10.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-24-state-after-a-restart-f10-restart-persis.json", "failed_command": "control-agent-server doctor", "variants": [ { "feature_id": "F10.restart-persist", "family": "F10", "result": "fail", "label": "State after a restart (`F10.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-24-state-after-a-restart-f10-restart-persis.json", "failed_command": "control-agent-server doctor" } ] }, { "feature_id": "F11.goal-auth", "family": "F11", "result": "pass", "label": "Authentication (`F11.goal-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-03-authentication-f11-goal-auth.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-auth", "family": "F11", "result": "pass", "label": "Authentication (`F11.goal-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-03-authentication-f11-goal-auth.json", "failed_command": null } ] }, { "feature_id": "F11.goal-busy", "family": "F11", "result": "pass", "label": "Busy while a round is in flight (`F11.goal-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-12-busy-while-a-round-is-in-flight-f11-goal.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-busy", "family": "F11", "result": "pass", "label": "Busy while a round is in flight (`F11.goal-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-12-busy-while-a-round-is-in-flight-f11-goal.json", "failed_command": null } ] }, { "feature_id": "F11.goal-busy-run", "family": "F11", "result": "pass", "label": "Busy with an ordinary run (`F11.goal-busy-run`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-17-busy-with-an-ordinary-run-f11-goal-busy-.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-busy-run", "family": "F11", "result": "pass", "label": "Busy with an ordinary run (`F11.goal-busy-run`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-17-busy-with-an-ordinary-run-f11-goal-busy-.json", "failed_command": null } ] }, { "feature_id": "F11.goal-cap-overrun", "family": "F11", "result": "xfail", "label": "A resume after a failed last audit overruns the cap (`F11.goal-cap-overrun`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-24-a-resume-after-a-failed-last-audit-overr.json", "failed_command": "jq -e '[.events[].value | select(.iteration > .max_iterations)] | length == 0'", "variants": [ { "feature_id": "F11.goal-cap-overrun", "family": "F11", "result": "xfail", "label": "A resume after a failed last audit overruns the cap (`F11.goal-cap-overrun`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-24-a-resume-after-a-failed-last-audit-overr.json", "failed_command": "jq -e '[.events[].value | select(.iteration > .max_iterations)] | length == 0'" } ] }, { "feature_id": "F11.goal-capacity", "family": "F11", "result": "pass", "label": "Capacity (`F11.goal-capacity`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-22-capacity-f11-goal-capacity.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-capacity", "family": "F11", "result": "pass", "label": "Capacity (`F11.goal-capacity`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-22-capacity-f11-goal-capacity.json", "failed_command": null } ] }, { "feature_id": "F11.goal-capped", "family": "F11", "result": "pass", "label": "Round cap (`F11.goal-capped`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-10-round-cap-f11-goal-capped.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-capped", "family": "F11", "result": "pass", "label": "Round cap (`F11.goal-capped`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-10-round-cap-f11-goal-capped.json", "failed_command": null } ] }, { "feature_id": "F11.goal-complete", "family": "F11", "result": "pass", "label": "A goal on a real model (`F11.goal-complete`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-06-a-goal-on-a-real-model-f11-goal-complete.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-complete", "family": "F11", "result": "pass", "label": "A goal on a real model (`F11.goal-complete`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-06-a-goal-on-a-real-model-f11-goal-complete.json", "failed_command": null } ] }, { "feature_id": "F11.goal-delete-active", "family": "F11", "result": "pass", "label": "Delete a conversation with an active loop (`F11.goal-delete-active`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-23-delete-a-conversation-with-an-active-loo.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-delete-active", "family": "F11", "result": "pass", "label": "Delete a conversation with an active loop (`F11.goal-delete-active`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-23-delete-a-conversation-with-an-active-loo.json", "failed_command": null } ] }, { "feature_id": "F11.goal-followup", "family": "F11", "result": "pass", "label": "Incomplete verdict re-prompts (`F11.goal-followup`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-09-incomplete-verdict-re-prompts-f11-goal-f.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-followup", "family": "F11", "result": "pass", "label": "Incomplete verdict re-prompts (`F11.goal-followup`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-09-incomplete-verdict-re-prompts-f11-goal-f.json", "failed_command": null } ] }, { "feature_id": "F11.goal-in-place", "family": "F11", "result": "pass", "label": "Same conversation, no new one (`F11.goal-in-place`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-08-same-conversation-no-new-one-f11-goal-in.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-in-place", "family": "F11", "result": "pass", "label": "Same conversation, no new one (`F11.goal-in-place`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-08-same-conversation-no-new-one-f11-goal-in.json", "failed_command": null } ] }, { "feature_id": "F11.goal-interrupt-run", "family": "F11", "result": "pass", "label": "Interrupt the round (`F11.goal-interrupt-run`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-16-interrupt-the-round-f11-goal-interrupt-r.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-interrupt-run", "family": "F11", "result": "pass", "label": "Interrupt the round (`F11.goal-interrupt-run`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-16-interrupt-the-round-f11-goal-interrupt-r.json", "failed_command": null } ] }, { "feature_id": "F11.goal-judge-error", "family": "F11", "result": "pass", "label": "The judge fails (`F11.goal-judge-error`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-21-the-judge-fails-f11-goal-judge-error.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-judge-error", "family": "F11", "result": "pass", "label": "The judge fails (`F11.goal-judge-error`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-21-the-judge-fails-f11-goal-judge-error.json", "failed_command": null } ] }, { "feature_id": "F11.goal-restart-active", "family": "F11", "result": "xfail", "label": "A loop active at a restart stays \"running\" (`F11.goal-restart-active`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-25-a-loop-active-at-a-restart-stays-running.json", "failed_command": "jq -e '.events[-1].value.active == false and .events[-1].value.status == \"interrupted\"'", "variants": [ { "feature_id": "F11.goal-restart-active", "family": "F11", "result": "xfail", "label": "A loop active at a restart stays \"running\" (`F11.goal-restart-active`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-25-a-loop-active-at-a-restart-stays-running.json", "failed_command": "jq -e '.events[-1].value.active == false and .events[-1].value.status == \"interrupted\"'" } ] }, { "feature_id": "F11.goal-restart-resume", "family": "F11", "result": "pass", "label": "Resume after a restart (`F11.goal-restart-resume`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-19-resume-after-a-restart-f11-goal-restart-.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-restart-resume", "family": "F11", "result": "pass", "label": "Resume after a restart (`F11.goal-restart-resume`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-19-resume-after-a-restart-f11-goal-restart-.json", "failed_command": null } ] }, { "feature_id": "F11.goal-resume", "family": "F11", "result": "pass", "label": "Resume a stopped loop (`F11.goal-resume`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-18-resume-a-stopped-loop-f11-goal-resume.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-resume", "family": "F11", "result": "pass", "label": "Resume a stopped loop (`F11.goal-resume`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-18-resume-a-stopped-loop-f11-goal-resume.json", "failed_command": null } ] }, { "feature_id": "F11.goal-resume-finished", "family": "F11", "result": "pass", "label": "Nothing to resume after the end (`F11.goal-resume-finished`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-11-nothing-to-resume-after-the-end-f11-goal.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-resume-finished", "family": "F11", "result": "pass", "label": "Nothing to resume after the end (`F11.goal-resume-finished`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-11-nothing-to-resume-after-the-end-f11-goal.json", "failed_command": null } ] }, { "feature_id": "F11.goal-resume-none", "family": "F11", "result": "pass", "label": "Resume with no goal (`F11.goal-resume-none`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-05-resume-with-no-goal-f11-goal-resume-none.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-resume-none", "family": "F11", "result": "pass", "label": "Resume with no goal (`F11.goal-resume-none`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-05-resume-with-no-goal-f11-goal-resume-none.json", "failed_command": null } ] }, { "feature_id": "F11.goal-run-error", "family": "F11", "result": "fail", "label": "The round's provider fails (`F11.goal-run-error`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-20-the-round-s-provider-fails-f11-goal-run-.json", "failed_command": "control-agent-server ws read f11-e --contains '\"key\": \"goal\", \"value\": {\"active\": false, \"status\": \"interrupted\", \"iteration\": 0' --wait 60", "variants": [ { "feature_id": "F11.goal-run-error", "family": "F11", "result": "fail", "label": "The round's provider fails (`F11.goal-run-error`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-20-the-round-s-provider-fails-f11-goal-run-.json", "failed_command": "control-agent-server ws read f11-e --contains '\"key\": \"goal\", \"value\": {\"active\": false, \"status\": \"interrupted\", \"iteration\": 0' --wait 60" } ] }, { "feature_id": "F11.goal-stop", "family": "F11", "result": "pass", "label": "Stop an active loop (`F11.goal-stop`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-13-stop-an-active-loop-f11-goal-stop.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-stop", "family": "F11", "result": "pass", "label": "Stop an active loop (`F11.goal-stop`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-13-stop-an-active-loop-f11-goal-stop.json", "failed_command": null } ] }, { "feature_id": "F11.goal-stop-noop", "family": "F11", "result": "pass", "label": "Stop with nothing to stop (`F11.goal-stop-noop`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-04-stop-with-nothing-to-stop-f11-goal-stop-.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-stop-noop", "family": "F11", "result": "pass", "label": "Stop with nothing to stop (`F11.goal-stop-noop`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-04-stop-with-nothing-to-stop-f11-goal-stop-.json", "failed_command": null } ] }, { "feature_id": "F11.goal-superseded", "family": "F11", "result": "pass", "label": "A user message supersedes the loop (`F11.goal-superseded`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-14-a-user-message-supersedes-the-loop-f11-g.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-superseded", "family": "F11", "result": "pass", "label": "A user message supersedes the loop (`F11.goal-superseded`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-14-a-user-message-supersedes-the-loop-f11-g.json", "failed_command": null } ] }, { "feature_id": "F11.goal-superseded-socket", "family": "F11", "result": "pass", "label": "A socket message supersedes the loop (`F11.goal-superseded-socket`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-15-a-socket-message-supersedes-the-loop-f11.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-superseded-socket", "family": "F11", "result": "pass", "label": "A socket message supersedes the loop (`F11.goal-superseded-socket`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-15-a-socket-message-supersedes-the-loop-f11.json", "failed_command": null } ] }, { "feature_id": "F11.goal-unknown-id", "family": "F11", "result": "pass", "label": "Unknown and malformed ids (`F11.goal-unknown-id`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-02-unknown-and-malformed-ids-f11-goal-unkno.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-unknown-id", "family": "F11", "result": "pass", "label": "Unknown and malformed ids (`F11.goal-unknown-id`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-02-unknown-and-malformed-ids-f11-goal-unkno.json", "failed_command": null } ] }, { "feature_id": "F11.goal-validation", "family": "F11", "result": "pass", "label": "Invalid goal requests (`F11.goal-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-01-invalid-goal-requests-f11-goal-validatio.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-validation", "family": "F11", "result": "pass", "label": "Invalid goal requests (`F11.goal-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-01-invalid-goal-requests-f11-goal-validatio.json", "failed_command": null } ] }, { "feature_id": "F11.goal-ws", "family": "F11", "result": "pass", "label": "Goal status on the socket (`F11.goal-ws`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-07-goal-status-on-the-socket-f11-goal-ws.json", "failed_command": null, "variants": [ { "feature_id": "F11.goal-ws", "family": "F11", "result": "pass", "label": "Goal status on the socket (`F11.goal-ws`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-07-goal-status-on-the-socket-f11-goal-ws.json", "failed_command": null } ] }, { "feature_id": "F12.binding-live-acp", "family": "F12", "result": "not-run", "label": "binding a loaded ACP Codex conversation that has not run yet answers 204, records `required_runtime_credential_bindings: [\"CODEX_AUTH_JSON\"]` in `meta.json` and removes the stored `CODEX_AUTH_JSON` secret.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.binding-live-acp", "family": "F12", "result": "not-run", "label": "binding a loaded ACP Codex conversation that has not run yet answers 204, records `required_runtime_credential_bindings: [\"CODEX_AUTH_JSON\"]` in `meta.json` and removes the stored `CODEX_AUTH_JSON` secret.", "run": null, "transcript": null } ] }, { "feature_id": "F12.binding-native-cold", "family": "F12", "result": "not-run", "label": "a binding sent while a regular (non-ACP) conversation is not loaded never makes that conversation demand a credential binding after the next pause.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.binding-native-cold", "family": "F12", "result": "not-run", "label": "a binding sent while a regular (non-ACP) conversation is not loaded never makes that conversation demand a credential binding after the next pause.", "run": null, "transcript": null } ] }, { "feature_id": "F12.binding-native-too-late", "family": "F12", "result": "not-run", "label": "binding a loaded regular (non-ACP) conversation is refused with 409 `Credential binding activation arrived after ACP initialization` and records nothing.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.binding-native-too-late", "family": "F12", "result": "not-run", "label": "binding a loaded regular (non-ACP) conversation is refused with 409 `Credential binding activation arrived after ACP initialization` and records nothing.", "run": null, "transcript": null } ] }, { "feature_id": "F12.binding-pending", "family": "F12", "result": "not-run", "label": "a binding for a conversation id the server does not know answers 204 after exactly one probe `GET` that carries the given headers; the id stays unknown (404).", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.binding-pending", "family": "F12", "result": "not-run", "label": "a binding for a conversation id the server does not know answers 204 after exactly one probe `GET` that carries the given headers; the id stays unknown (404).", "run": null, "transcript": null } ] }, { "feature_id": "F12.binding-pending-cleared", "family": "F12", "result": "not-run", "label": "a pause drops pending bindings, so a conversation created afterwards with that id keeps its own `CODEX_AUTH_JSON` secret and requires no binding.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.binding-pending-cleared", "family": "F12", "result": "not-run", "label": "a pause drops pending bindings, so a conversation created afterwards with that id keeps its own `CODEX_AUTH_JSON` secret and requires no binding.", "run": null, "transcript": null } ] }, { "feature_id": "F12.binding-pending-on-create", "family": "F12", "result": "not-run", "label": "a pending binding is consumed when a conversation is created with that `conversation_id`, which then requires the binding from the start and stores no `CODEX_AUTH_JSON` secret.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.binding-pending-on-create", "family": "F12", "result": "not-run", "label": "a pending binding is consumed when a conversation is created with that `conversation_id`, which then requires the binding from the start and stores no `CODEX_AUTH_JSON` secret.", "run": null, "transcript": null } ] }, { "feature_id": "F12.binding-profile-scope", "family": "F12", "result": "not-run", "label": "a conversation launched from an agent profile whose `secret_refs` exclude `CODEX_AUTH_JSON` refuses the binding with 403 `Credential binding authorization was rejected` and records nothing.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.binding-profile-scope", "family": "F12", "result": "not-run", "label": "a conversation launched from an agent profile whose `secret_refs` exclude `CODEX_AUTH_JSON` refuses the binding with 403 `Credential binding authorization was rejected` and records nothing.", "run": null, "transcript": null } ] }, { "feature_id": "F12.binding-rebind", "family": "F12", "result": "not-run", "label": "a second activation with the same URL and new headers is accepted (the probe carries the new headers, which are never written to disk); a different URL is refused with 409.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.binding-rebind", "family": "F12", "result": "not-run", "label": "a second activation with the same URL and new headers is accepted (the probe carries the new headers, which are never written to disk); a different URL is refused with 409.", "run": null, "transcript": null } ] }, { "feature_id": "F12.binding-required-after-pause", "family": "F12", "result": "not-run", "label": "after a pause, a conversation that requires a binding is still listed and reports runtime `available`, but event access and `/run` answer 409 `{\"detail\": \"credential_binding_activation_required\", \"retryable\": true}` and both conversation sockets close with 1013, until a new activation, after which events load again.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.binding-required-after-pause", "family": "F12", "result": "not-run", "label": "after a pause, a conversation that requires a binding is still listed and reports runtime `available`, but event access and `/run` answer 409 `{\"detail\": \"credential_binding_activation_required\", \"retryable\": true}` and both conversation sockets close with 1013, until a new activation, after which events load again.", "run": null, "transcript": null } ] }, { "feature_id": "F12.binding-required-after-restart", "family": "F12", "result": "not-run", "label": "the requirement survives a restart (409 until a new activation, then 200), and deleting such a conversation needs no activation.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.binding-required-after-restart", "family": "F12", "result": "not-run", "label": "the requirement survives a restart (409 until a new activation, then 200), and deleting such a conversation needs no activation.", "run": null, "transcript": null } ] }, { "feature_id": "F12.binding-source-statuses", "family": "F12", "result": "not-run", "label": "the source's answer maps to the activation status: 401/403 to 403, 400/404/422 to 422, 409 to 409, 501 to 501, a 200 that is not `{\"value\": str, \"version\": non-empty str}` to 422, and a URL that is not http(s) to 422, each after a single probe (none for the bad URL).", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.binding-source-statuses", "family": "F12", "result": "not-run", "label": "the source's answer maps to the activation status: 401/403 to 403, 400/404/422 to 422, 409 to 409, 501 to 501, a 200 that is not `{\"value\": str, \"version\": non-empty str}` to 422, and a URL that is not http(s) to 422, each after a single probe (none for the bad URL).", "run": null, "transcript": null } ] }, { "feature_id": "F12.binding-source-unavailable", "family": "F12", "result": "not-run", "label": "a source answering 5xx is probed three times with backoff (at least 0.75 s), then the activation fails with 502; an unreachable source also gives 502.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.binding-source-unavailable", "family": "F12", "result": "not-run", "label": "a source answering 5xx is probed three times with backoff (at least 0.75 s), then the activation fails with 502; an unreachable source also gives 502.", "run": null, "transcript": null } ] }, { "feature_id": "F12.binding-unsupported-secret", "family": "F12", "result": "not-run", "label": "only file credentials can be bound: any other secret name gets 404 without the source being contacted.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.binding-unsupported-secret", "family": "F12", "result": "not-run", "label": "only file credentials can be bound: any other secret name gets 404 without the source being contacted.", "run": null, "transcript": null } ] }, { "feature_id": "F12.binding-validation", "family": "F12", "result": "not-run", "label": "a body with extra keys, no `url`, an empty or over-long (4097+ characters) `url`, an empty or over-long (8193+ characters) header value or more than 16 headers, or a non-UUID id, gets 422 without the source being contacted, also for an unsupported secret name (the body is validated first); no key gets 401.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.binding-validation", "family": "F12", "result": "not-run", "label": "a body with extra keys, no `url`, an empty or over-long (4097+ characters) `url`, an empty or over-long (8193+ characters) header value or more than 16 headers, or a non-UUID id, gets 422 without the source being contacted, also for an unsupported secret name (the body is validated first); no key gets 401.", "run": null, "transcript": null } ] }, { "feature_id": "F12.discovery", "family": "F12", "result": "not-run", "label": "the OpenAPI document lists the two runtime routes but not the pause and credential-binding routes, and `/server_info` advertises `credential_binding_v1`, `credential_binding_readiness_probe_v1` and `credential_binding_activation_guard_v1`.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.discovery", "family": "F12", "result": "not-run", "label": "the OpenAPI document lists the two runtime routes but not the pause and credential-binding routes, and `/server_info` advertises `credential_binding_v1`, `credential_binding_readiness_probe_v1` and `credential_binding_activation_guard_v1`.", "run": null, "transcript": null } ] }, { "feature_id": "F12.docker-runtime", "family": "F12", "result": "not-run", "label": "with a Docker daemon, a create starts a container with the configured `--memory`, `--cpus` and `--pids-limit`, the launched profile in `OH_RUNTIME_LAUNCHED_PROFILE` and a startup bound of `conversation_container_startup_timeout`; `POST .../runtime/credentials` returns the container's session key, `DELETE .../runtime` stops it (204) while the conversation stays readable, reprovision starts it again, a request carrying `X-Expose-Secrets` is refused with 422 and an archived runtime answers 410 (blocked here: no Docker daemon).", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.docker-runtime", "family": "F12", "result": "not-run", "label": "with a Docker daemon, a create starts a container with the configured `--memory`, `--cpus` and `--pids-limit`, the launched profile in `OH_RUNTIME_LAUNCHED_PROFILE` and a startup bound of `conversation_container_startup_timeout`; `POST .../runtime/credentials` returns the container's session key, `DELETE .../runtime` stops it (204) while the conversation stays readable, reprovision starts it again, a request carrying `X-Expose-Secrets` is refused with 422 and an archived runtime answers 410 (blocked here: no Docker daemon).", "run": null, "transcript": null } ] }, { "feature_id": "F12.docker-runtime-contract", "family": "F12", "result": "not-run", "label": "on a server with `OH_CONVERSATION_RUNTIME=docker` and no Docker daemon, fork, `switch_llm`, `switch_profile` and credential bindings answer 501 before any lookup (the binding source is never contacted); the runtime routes, `DELETE .../runtime`, `POST .../runtime/credentials` and `/run` answer 404 `Conversation not found` for an unknown id, and the two Docker-only routes 401 without a key; both conversation sockets close with 1008 for an unknown id; create refuses a missing body (400), a non-object (422), a non-local workspace (422) and a bad `conversation_id` (400) without writing a runtime manifest; a well-formed create answers 502, leaving a manifest whose runtime info is 404, whose reprovision is 502 and whose socket closes with 1011, until `DELETE /{id}` removes it.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.docker-runtime-contract", "family": "F12", "result": "not-run", "label": "on a server with `OH_CONVERSATION_RUNTIME=docker` and no Docker daemon, fork, `switch_llm`, `switch_profile` and credential bindings answer 501 before any lookup (the binding source is never contacted); the runtime routes, `DELETE .../runtime`, `POST .../runtime/credentials` and `/run` answer 404 `Conversation not found` for an unknown id, and the two Docker-only routes 401 without a key; both conversation sockets close with 1008 for an unknown id; create refuses a missing body (400), a non-object (422), a non-local workspace (422) and a bad `conversation_id` (400) without writing a runtime manifest; a well-formed create answers 502, leaving a manifest whose runtime info is 404, whose reprovision is 502 and whose socket closes with 1011, until `DELETE /{id}` removes it.", "run": null, "transcript": null } ] }, { "feature_id": "F12.managed-key-refresh", "family": "F12", "result": "not-run", "label": "with `OH_LLM_API_KEY_REFRESH_URL`, `OH_LLM_API_KEY_REFRESH_BASE_URLS` naming the LLM's `base_url` and `OH_LLM_API_KEY_REFRESH_HEADERS` (values expanded from the server environment), a model call rejected with 401 makes exactly one `GET` of the refresh URL with the expanded headers and one retry, and the run finishes; the fetched key is not written to the server's state.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.managed-key-refresh", "family": "F12", "result": "not-run", "label": "with `OH_LLM_API_KEY_REFRESH_URL`, `OH_LLM_API_KEY_REFRESH_BASE_URLS` naming the LLM's `base_url` and `OH_LLM_API_KEY_REFRESH_HEADERS` (values expanded from the server environment), a model call rejected with 401 makes exactly one `GET` of the refresh URL with the expanded headers and one retry, and the run finishes; the fetched key is not written to the server's state.", "run": null, "transcript": null } ] }, { "feature_id": "F12.managed-key-scope", "family": "F12", "result": "not-run", "label": "an LLM whose `base_url` is outside the allow-list, or any LLM once the allow-list is empty, surfaces the 401 (`LLMAuthenticationError`, status `error`) after one model call and no refresh request; the empty allow-list is logged as \"stays off\".", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.managed-key-scope", "family": "F12", "result": "not-run", "label": "an LLM whose `base_url` is outside the allow-list, or any LLM once the allow-list is empty, surfaces the 401 (`LLMAuthenticationError`, status `error`) after one model call and no refresh request; the empty allow-list is logged as \"stays off\".", "run": null, "transcript": null } ] }, { "feature_id": "F12.managed-key-switch", "family": "F12", "result": "not-run", "label": "an in-scope LLM installed with `switch_llm` (or `switch_profile`) recovers from a 401 through the refresh URL like the LLM the conversation was loaded with.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.managed-key-switch", "family": "F12", "result": "not-run", "label": "an in-scope LLM installed with `switch_llm` (or `switch_profile`) recovers from a 401 through the refresh URL like the LLM the conversation was loaded with.", "run": null, "transcript": null } ] }, { "feature_id": "F12.runtime-cold", "family": "F12", "result": "not-run", "label": "runtime info answers for a conversation that is not loaded, before and after a restart, without loading it (no lease file appears).", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.runtime-cold", "family": "F12", "result": "not-run", "label": "runtime info answers for a conversation that is not loaded, before and after a restart, without loading it (no lease file appears).", "run": null, "transcript": null } ] }, { "feature_id": "F12.runtime-errors", "family": "F12", "result": "not-run", "label": "both runtime routes answer 404 for an unknown id, 422 for a non-UUID id and 401 without a session key.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.runtime-errors", "family": "F12", "result": "not-run", "label": "both runtime routes answer 404 for an unknown id, 422 for a non-UUID id and 401 without a session key.", "run": null, "transcript": null } ] }, { "feature_id": "F12.runtime-info", "family": "F12", "result": "not-run", "label": "`GET .../runtime` returns 200 `{\"runtime_status\": \"available\", \"can_resume\": true, \"runtime_error\": null}` for a local conversation, and the conversation's GET, search and batch-get rows embed the same object as `runtime_info`.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.runtime-info", "family": "F12", "result": "not-run", "label": "`GET .../runtime` returns 200 `{\"runtime_status\": \"available\", \"can_resume\": true, \"runtime_error\": null}` for a local conversation, and the conversation's GET, search and batch-get rows embed the same object as `runtime_info`.", "run": null, "transcript": null } ] }, { "feature_id": "F12.runtime-launched-profile", "family": "F12", "result": "not-run", "label": "on a local server started with `OH_RUNTIME_LAUNCHED_PROFILE` (the launched profile a Docker parent passes into each conversation container), a conversation created without an agent profile records that profile as `launched_agent_profile` and its `secret_refs` scope the credential binding (403), while a conversation started from its own agent profile keeps its own profile (204).", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.runtime-launched-profile", "family": "F12", "result": "not-run", "label": "on a local server started with `OH_RUNTIME_LAUNCHED_PROFILE` (the launched profile a Docker parent passes into each conversation container), a conversation created without an agent profile records that profile as `launched_agent_profile` and its `secret_refs` scope the credential binding (403), while a conversation started from its own agent profile keeps its own profile (204).", "run": null, "transcript": null } ] }, { "feature_id": "F12.runtime-reprovision", "family": "F12", "result": "not-run", "label": "`POST .../runtime/reprovision` returns the same runtime info, ignores a body, and changes nothing (status, event count and lease generation stay the same).", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.runtime-reprovision", "family": "F12", "result": "not-run", "label": "`POST .../runtime/reprovision` returns the same runtime info, ignores a body, and changes nothing (status, event count and lease generation stay the same).", "run": null, "transcript": null } ] }, { "feature_id": "F12.sandbox-pause-rehydrate", "family": "F12", "result": "not-run", "label": "after a pause (no restart in between), the next event access loads the conversation again (a new lease, the same events), and a new message runs to `finished` on a real model, which acts again.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.sandbox-pause-rehydrate", "family": "F12", "result": "not-run", "label": "after a pause (no restart in between), the next event access loads the conversation again (a new lease, the same events), and a new message runs to `finished` on a real model, which acts again.", "run": null, "transcript": null } ] }, { "feature_id": "F12.sandbox-pause-releases", "family": "F12", "result": "not-run", "label": "`prepare-for-sandbox-pause` answers 204 with an empty body, deletes every `owner_lease.json`, keeps every conversation listed and readable with its status, is idempotent, and is refused with 401 (leaving the leases) without a key.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.sandbox-pause-releases", "family": "F12", "result": "not-run", "label": "`prepare-for-sandbox-pause` answers 204 with an empty body, deletes every `owner_lease.json`, keeps every conversation listed and readable with its status, is idempotent, and is refused with 401 (leaving the leases) without a key.", "run": null, "transcript": null } ] }, { "feature_id": "F12.sandbox-pause-running", "family": "F12", "result": "not-run", "label": "a conversation whose model call is in flight is cancelled to `paused` within the server's 10 s bound, with a `PauseEvent` and an `InterruptEvent`, and the status change is pushed on its events socket.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.sandbox-pause-running", "family": "F12", "result": "not-run", "label": "a conversation whose model call is in flight is cancelled to `paused` within the server's 10 s bound, with a `PauseEvent` and an `InterruptEvent`, and the status change is pushed on its events socket.", "run": null, "transcript": null } ] }, { "feature_id": "F12.sandbox-pause-websocket", "family": "F12", "result": "not-run", "label": "an events socket or a session socket that was open before the pause is either closed by the server or keeps receiving the conversation's new events; it is never left open and silent.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.sandbox-pause-websocket", "family": "F12", "result": "not-run", "label": "an events socket or a session socket that was open before the pause is either closed by the server or keeps receiving the conversation's new events; it is never left open and silent.", "run": null, "transcript": null } ] }, { "feature_id": "F12.ts-client", "family": "F12", "result": "not-run", "label": "the built TypeScript client's `ConversationClient.getRuntime` and `reprovisionRuntime` return the local runtime info, `getConversation()` embeds it as `runtime_info`, and both methods reject an unknown id with a 404 `HttpError` and a client without a key with 401.", "run": null, "transcript": null, "variants": [ { "feature_id": "F12.ts-client", "family": "F12", "result": "not-run", "label": "the built TypeScript client's `ConversationClient.getRuntime` and `reprovisionRuntime` return the local runtime info, `getConversation()` embeds it as `runtime_info`, and both methods reject an unknown id with a 404 `HttpError` and a client without a key with 401.", "run": null, "transcript": null } ] }, { "feature_id": "F13.auth", "family": "F13", "result": "pass", "label": "Auth (`F13.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-01-auth-f13-auth.json", "failed_command": null, "variants": [ { "feature_id": "F13.auth", "family": "F13", "result": "pass", "label": "Auth (`F13.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-01-auth-f13-auth.json", "failed_command": null } ] }, { "feature_id": "F13.clear", "family": "F13", "result": "pass", "label": "Clear (`F13.clear`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-25-clear-f13-clear.json", "failed_command": null, "variants": [ { "feature_id": "F13.clear", "family": "F13", "result": "pass", "label": "Clear (`F13.clear`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-25-clear-f13-clear.json", "failed_command": null } ] }, { "feature_id": "F13.env-config-credentials", "family": "F13", "result": "pass", "label": "Other credential variables (`F13.env-config-credentials`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-05-other-credential-variables-f13-env-confi.json", "failed_command": null, "variants": [ { "feature_id": "F13.env-config-credentials", "family": "F13", "result": "pass", "label": "Other credential variables (`F13.env-config-credentials`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-05-other-credential-variables-f13-env-confi.json", "failed_command": null } ] }, { "feature_id": "F13.env-sanitized", "family": "F13", "result": "pass", "label": "Sanitized environment (`F13.env-sanitized`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-04-sanitized-environment-f13-env-sanitized.json", "failed_command": null, "variants": [ { "feature_id": "F13.env-sanitized", "family": "F13", "result": "pass", "label": "Sanitized environment (`F13.env-sanitized`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-04-sanitized-environment-f13-env-sanitized.json", "failed_command": null } ] }, { "feature_id": "F13.execute-large-output", "family": "F13", "result": "xfail", "label": "Large output through execute (`F13.execute-large-output`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-16-large-output-through-execute-f13-execute.json", "failed_command": "control-agent-server api POST /api/bash/execute_bash_command --json '{\"command\": \"yes a | head -c 1100000; echo QA_F13_TAIL\", \"timeout\": 60}' --expect 200 --check exit_code eq 0 --check stdout len-eq 1100012 --quiet --save F13.execute-large-output/execute", "variants": [ { "feature_id": "F13.execute-large-output", "family": "F13", "result": "xfail", "label": "Large output through execute (`F13.execute-large-output`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-16-large-output-through-execute-f13-execute.json", "failed_command": "control-agent-server api POST /api/bash/execute_bash_command --json '{\"command\": \"yes a | head -c 1100000; echo QA_F13_TAIL\", \"timeout\": 60}' --expect 200 --check exit_code eq 0 --check stdout len-eq 1100012 --quiet --save F13.execute-large-output/execute" } ] }, { "feature_id": "F13.execute-sync", "family": "F13", "result": "pass", "label": "Synchronous execute (`F13.execute-sync`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-02-synchronous-execute-f13-execute-sync.json", "failed_command": null, "variants": [ { "feature_id": "F13.execute-sync", "family": "F13", "result": "pass", "label": "Synchronous execute (`F13.execute-sync`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-02-synchronous-execute-f13-execute-sync.json", "failed_command": null } ] }, { "feature_id": "F13.get-events", "family": "F13", "result": "pass", "label": "Single and batch reads (`F13.get-events`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-11-single-and-batch-reads-f13-get-events.json", "failed_command": null, "variants": [ { "feature_id": "F13.get-events", "family": "F13", "result": "pass", "label": "Single and batch reads (`F13.get-events`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-11-single-and-batch-reads-f13-get-events.json", "failed_command": null } ] }, { "feature_id": "F13.global-cwd", "family": "F13", "result": "pass", "label": "Global cwd (`F13.global-cwd`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-03-global-cwd-f13-global-cwd.json", "failed_command": null, "variants": [ { "feature_id": "F13.global-cwd", "family": "F13", "result": "pass", "label": "Global cwd (`F13.global-cwd`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-03-global-cwd-f13-global-cwd.json", "failed_command": null } ] }, { "feature_id": "F13.order-gt-paging", "family": "F13", "result": "xfail", "label": "Order filter and paging (`F13.order-gt-paging`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-15-order-filter-and-paging-f13-order-gt-pag.json", "failed_command": "control-agent-server api GET /api/bash/bash_events/search --query command_id__eq=\"$BIG_CMD\" --query order__gt=0 --query limit=1 --check items len-eq 1 --check items.0.order eq 1 --quiet --save F13.order-gt-paging/limit-1", "variants": [ { "feature_id": "F13.order-gt-paging", "family": "F13", "result": "xfail", "label": "Order filter and paging (`F13.order-gt-paging`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-15-order-filter-and-paging-f13-order-gt-pag.json", "failed_command": "control-agent-server api GET /api/bash/bash_events/search --query command_id__eq=\"$BIG_CMD\" --query order__gt=0 --query limit=1 --check items len-eq 1 --check items.0.order eq 1 --quiet --save F13.order-gt-paging/limit-1" } ] }, { "feature_id": "F13.output-chunks", "family": "F13", "result": "pass", "label": "Output chunks (`F13.output-chunks`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-14-output-chunks-f13-output-chunks.json", "failed_command": null, "variants": [ { "feature_id": "F13.output-chunks", "family": "F13", "result": "pass", "label": "Output chunks (`F13.output-chunks`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-14-output-chunks-f13-output-chunks.json", "failed_command": null } ] }, { "feature_id": "F13.persist-restart", "family": "F13", "result": "pass", "label": "Restart (`F13.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-24-restart-f13-persist-restart.json", "failed_command": null, "variants": [ { "feature_id": "F13.persist-restart", "family": "F13", "result": "pass", "label": "Restart (`F13.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-24-restart-f13-persist-restart.json", "failed_command": null } ] }, { "feature_id": "F13.retention", "family": "F13", "result": "pass", "label": "Retention (`F13.retention`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-26-retention-f13-retention.json", "failed_command": null, "variants": [ { "feature_id": "F13.retention", "family": "F13", "result": "pass", "label": "Retention (`F13.retention`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-26-retention-f13-retention.json", "failed_command": null } ] }, { "feature_id": "F13.retention-rolling", "family": "F13", "result": "pass", "label": "Rolling retention (`F13.retention-rolling`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-27-rolling-retention-f13-retention-rolling.json", "failed_command": null, "variants": [ { "feature_id": "F13.retention-rolling", "family": "F13", "result": "pass", "label": "Rolling retention (`F13.retention-rolling`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-27-rolling-retention-f13-retention-rolling.json", "failed_command": null } ] }, { "feature_id": "F13.scoped-bad-id", "family": "F13", "result": "xfail", "label": "Malformed conversation id (`F13.scoped-bad-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-28-malformed-conversation-id-f13-scoped-bad.json", "failed_command": "control-agent-server api GET /not-a-uuid/bash/bash_events/search --expect 422 --save F13.scoped-bad-id/search", "variants": [ { "feature_id": "F13.scoped-bad-id", "family": "F13", "result": "xfail", "label": "Malformed conversation id (`F13.scoped-bad-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-28-malformed-conversation-id-f13-scoped-bad.json", "failed_command": "control-agent-server api GET /not-a-uuid/bash/bash_events/search --expect 422 --save F13.scoped-bad-id/search" } ] }, { "feature_id": "F13.scoped-cwd", "family": "F13", "result": "pass", "label": "Scoped cwd (`F13.scoped-cwd`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-20-scoped-cwd-f13-scoped-cwd.json", "failed_command": null, "variants": [ { "feature_id": "F13.scoped-cwd", "family": "F13", "result": "pass", "label": "Scoped cwd (`F13.scoped-cwd`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-20-scoped-cwd-f13-scoped-cwd.json", "failed_command": null } ] }, { "feature_id": "F13.scoped-idle-eviction", "family": "F13", "result": "xfail", "label": "Idle eviction kills a running command (`F13.scoped-idle-eviction`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-29-idle-eviction-kills-a-running-command-f1.json", "failed_command": "pgrep -f \"$EV_MARK\"", "variants": [ { "feature_id": "F13.scoped-idle-eviction", "family": "F13", "result": "xfail", "label": "Idle eviction kills a running command (`F13.scoped-idle-eviction`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-29-idle-eviction-kills-a-running-command-f1.json", "failed_command": "pgrep -f \"$EV_MARK\"" } ] }, { "feature_id": "F13.scoped-isolation", "family": "F13", "result": "pass", "label": "Scope isolation (`F13.scoped-isolation`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-22-scope-isolation-f13-scoped-isolation.json", "failed_command": null, "variants": [ { "feature_id": "F13.scoped-isolation", "family": "F13", "result": "pass", "label": "Scope isolation (`F13.scoped-isolation`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-22-scope-isolation-f13-scoped-isolation.json", "failed_command": null } ] }, { "feature_id": "F13.scoped-routes", "family": "F13", "result": "pass", "label": "Scoped routes (`F13.scoped-routes`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-21-scoped-routes-f13-scoped-routes.json", "failed_command": null, "variants": [ { "feature_id": "F13.scoped-routes", "family": "F13", "result": "pass", "label": "Scoped routes (`F13.scoped-routes`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-21-scoped-routes-f13-scoped-routes.json", "failed_command": null } ] }, { "feature_id": "F13.scoped-unknown", "family": "F13", "result": "pass", "label": "Unknown conversation (`F13.scoped-unknown`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-23-unknown-conversation-f13-scoped-unknown.json", "failed_command": null, "variants": [ { "feature_id": "F13.scoped-unknown", "family": "F13", "result": "pass", "label": "Unknown conversation (`F13.scoped-unknown`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-23-unknown-conversation-f13-scoped-unknown.json", "failed_command": null } ] }, { "feature_id": "F13.search", "family": "F13", "result": "pass", "label": "Search (`F13.search`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-12-search-f13-search.json", "failed_command": null, "variants": [ { "feature_id": "F13.search", "family": "F13", "result": "pass", "label": "Search (`F13.search`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-12-search-f13-search.json", "failed_command": null } ] }, { "feature_id": "F13.start-async", "family": "F13", "result": "pass", "label": "Asynchronous start (`F13.start-async`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-06-asynchronous-start-f13-start-async.json", "failed_command": null, "variants": [ { "feature_id": "F13.start-async", "family": "F13", "result": "pass", "label": "Asynchronous start (`F13.start-async`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-06-asynchronous-start-f13-start-async.json", "failed_command": null } ] }, { "feature_id": "F13.stop", "family": "F13", "result": "pass", "label": "Stop one command (`F13.stop`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-18-stop-one-command-f13-stop.json", "failed_command": null, "variants": [ { "feature_id": "F13.stop", "family": "F13", "result": "pass", "label": "Stop one command (`F13.stop`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-18-stop-one-command-f13-stop.json", "failed_command": null } ] }, { "feature_id": "F13.stop-idempotent", "family": "F13", "result": "pass", "label": "Stop is idempotent (`F13.stop-idempotent`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-19-stop-is-idempotent-f13-stop-idempotent.json", "failed_command": null, "variants": [ { "feature_id": "F13.stop-idempotent", "family": "F13", "result": "pass", "label": "Stop is idempotent (`F13.stop-idempotent`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-19-stop-is-idempotent-f13-stop-idempotent.json", "failed_command": null } ] }, { "feature_id": "F13.timeout", "family": "F13", "result": "pass", "label": "Timeout (`F13.timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-17-timeout-f13-timeout.json", "failed_command": null, "variants": [ { "feature_id": "F13.timeout", "family": "F13", "result": "pass", "label": "Timeout (`F13.timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-17-timeout-f13-timeout.json", "failed_command": null } ] }, { "feature_id": "F13.timestamp-offset", "family": "F13", "result": "xfail", "label": "Offset timestamps (`F13.timestamp-offset`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-13-offset-timestamps-f13-timestamp-offset-k.json", "failed_command": "control-agent-server api GET /api/bash/bash_events/search --query kind__eq=BashCommand --query timestamp__gte=\"$GTE_PLUS2\" --check items contains \"$TZ_CMD\" --save F13.timestamp-offset/plus-two", "variants": [ { "feature_id": "F13.timestamp-offset", "family": "F13", "result": "xfail", "label": "Offset timestamps (`F13.timestamp-offset`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-13-offset-timestamps-f13-timestamp-offset-k.json", "failed_command": "control-agent-server api GET /api/bash/bash_events/search --query kind__eq=BashCommand --query timestamp__gte=\"$GTE_PLUS2\" --check items contains \"$TZ_CMD\" --save F13.timestamp-offset/plus-two" } ] }, { "feature_id": "F13.ws-command", "family": "F13", "result": "pass", "label": "Commands over the socket (`F13.ws-command`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-08-commands-over-the-socket-f13-ws-command.json", "failed_command": null, "variants": [ { "feature_id": "F13.ws-command", "family": "F13", "result": "pass", "label": "Commands over the socket (`F13.ws-command`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-08-commands-over-the-socket-f13-ws-command.json", "failed_command": null } ] }, { "feature_id": "F13.ws-live", "family": "F13", "result": "pass", "label": "Live and replayed socket frames (`F13.ws-live`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-07-live-and-replayed-socket-frames-f13-ws-l.json", "failed_command": null, "variants": [ { "feature_id": "F13.ws-live", "family": "F13", "result": "pass", "label": "Live and replayed socket frames (`F13.ws-live`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-07-live-and-replayed-socket-frames-f13-ws-l.json", "failed_command": null } ] }, { "feature_id": "F13.ws-resend-all", "family": "F13", "result": "pass", "label": "Deprecated replay flag (`F13.ws-resend-all`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-09-deprecated-replay-flag-f13-ws-resend-all.json", "failed_command": null, "variants": [ { "feature_id": "F13.ws-resend-all", "family": "F13", "result": "pass", "label": "Deprecated replay flag (`F13.ws-resend-all`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-09-deprecated-replay-flag-f13-ws-resend-all.json", "failed_command": null } ] }, { "feature_id": "F13.ws-subscriber-cap", "family": "F13", "result": "pass", "label": "Subscriber cap (`F13.ws-subscriber-cap`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-10-subscriber-cap-f13-ws-subscriber-cap.json", "failed_command": null, "variants": [ { "feature_id": "F13.ws-subscriber-cap", "family": "F13", "result": "pass", "label": "Subscriber cap (`F13.ws-subscriber-cap`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-10-subscriber-cap-f13-ws-subscriber-cap.json", "failed_command": null } ] }, { "feature_id": "F14.archive-auto-base", "family": "F14", "result": "not-run", "label": "without `base_ref`, a clone with an `origin` remote is diffed against `origin/` (the patch holds the local commit and the uncommitted edit, applies to a clone of the origin tip, and carries `X-Archive-Base-Ref: auto`, `X-Archive-Base-Commit` = the origin tip and `X-Archive-Repo-Remote`); a detached HEAD is diffed against HEAD itself.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.archive-auto-base", "family": "F14", "result": "not-run", "label": "without `base_ref`, a clone with an `origin` remote is diffed against `origin/` (the patch holds the local commit and the uncommitted edit, applies to a clone of the origin tip, and carries `X-Archive-Base-Ref: auto`, `X-Archive-Base-Commit` = the origin tip and `X-Archive-Repo-Remote`); a detached HEAD is diffed against HEAD itself.", "run": null, "transcript": null } ] }, { "feature_id": "F14.archive-errors", "family": "F14", "result": "not-run", "label": "a relative path is 400, a missing directory 404, a file 400, a `base_ref` starting with `-` 400, an unknown `base_ref` 400, an unknown `format` 422, and `git-delta` of a non-repo directory 400.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.archive-errors", "family": "F14", "result": "not-run", "label": "a relative path is 400, a missing directory 404, a file 400, a `base_ref` starting with `-` 400, an unknown `base_ref` 400, an unknown `format` 422, and `git-delta` of a non-repo directory 400.", "run": null, "transcript": null } ] }, { "feature_id": "F14.archive-excludes", "family": "F14", "result": "not-run", "label": "`use_default_excludes=false` keeps `node_modules` and `.git` internals but never `.git/config` or reflogs; `exclude=` (repeatable) drops matching files and is recorded in the manifest.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.archive-excludes", "family": "F14", "result": "not-run", "label": "`use_default_excludes=false` keeps `node_modules` and `.git` internals but never `.git/config` or reflogs; `exclude=` (repeatable) drops matching files and is recorded in the manifest.", "run": null, "transcript": null } ] }, { "feature_id": "F14.archive-format-doc", "family": "F14", "result": "not-run", "label": "the OpenAPI description of `format` matches what a non-repository directory holding one repository archives; today it promises a tarball of \"the entire directory\" while only the nested repository is packed.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.archive-format-doc", "family": "F14", "result": "not-run", "label": "the OpenAPI description of `format` matches what a non-repository directory holding one repository archives; today it promises a tarball of \"the entire directory\" while only the nested repository is packed.", "run": null, "transcript": null } ] }, { "feature_id": "F14.archive-git-delta", "family": "F14", "result": "not-run", "label": "the default `git-delta` format with `base_ref=HEAD` is a `text/x-patch` of the uncommitted and untracked changes that applies to a clone of HEAD, carries `X-Archive-Base-Commit`, `X-Archive-Base-Ref`, `X-Archive-Branch` and `X-Archive-Head-Commit`, and leaves the repository's index untouched; without `base_ref` on a remote-less repo the patch is against the empty tree and has no base headers; a subdirectory of a repository yields only that subtree's delta.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.archive-git-delta", "family": "F14", "result": "not-run", "label": "the default `git-delta` format with `base_ref=HEAD` is a `text/x-patch` of the uncommitted and untracked changes that applies to a clone of HEAD, carries `X-Archive-Base-Commit`, `X-Archive-Base-Ref`, `X-Archive-Branch` and `X-Archive-Head-Commit`, and leaves the repository's index untouched; without `base_ref` on a remote-less repo the patch is against the empty tree and has no base headers; a subdirectory of a repository yields only that subtree's delta.", "run": null, "transcript": null } ] }, { "feature_id": "F14.archive-nested-repo", "family": "F14", "result": "not-run", "label": "archiving a non-repo directory that holds exactly one git repository archives that repository; with two repositories `git-delta` is 400 and `tar.gz` archives the directory itself.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.archive-nested-repo", "family": "F14", "result": "not-run", "label": "archiving a non-repo directory that holds exactly one git repository archives that repository; with two repositories `git-delta` is 400 and `tar.gz` archives the directory itself.", "run": null, "transcript": null } ] }, { "feature_id": "F14.archive-tar", "family": "F14", "result": "not-run", "label": "`format=tar.gz` returns an `application/gzip` archive named `.tar.gz`, rooted at `/`, with `archive_manifest.json`, the repository identity headers but no `X-Archive-Base-*`, the default excludes applied (`.git/`, `node_modules/`), symlinks skipped, and no temp file left in the parent directory.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.archive-tar", "family": "F14", "result": "not-run", "label": "`format=tar.gz` returns an `application/gzip` archive named `.tar.gz`, rooted at `/`, with `archive_manifest.json`, the repository identity headers but no `X-Archive-Base-*`, the default excludes applied (`.git/`, `node_modules/`), symlinks skipped, and no temp file left in the parent directory.", "run": null, "transcript": null } ] }, { "feature_id": "F14.auth", "family": "F14", "result": "not-run", "label": "every global and conversation-scoped file route answers 401 without a valid session key, and nothing is written.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.auth", "family": "F14", "result": "not-run", "label": "every global and conversation-scoped file route answers 401 without a valid session key, and nothing is written.", "run": null, "transcript": null } ] }, { "feature_id": "F14.download-errors", "family": "F14", "result": "not-run", "label": "downloading a missing file is 404 `File not found`, a directory 400 `Path is not a file`, a relative path 400.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.download-errors", "family": "F14", "result": "not-run", "label": "downloading a missing file is 404 `File not found`, a directory 400 `Path is not a file`, a relative path 400.", "run": null, "transcript": null } ] }, { "feature_id": "F14.home", "family": "F14", "result": "not-run", "label": "`GET /api/file/home` reports the server user's `HOME`, its visible top-level directories as case-insensitively sorted `favorites` (no files, no symlinks), `/` as the only location, and hidden directories only with `include_hidden=true`.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.home", "family": "F14", "result": "not-run", "label": "`GET /api/file/home` reports the server user's `HOME`, its visible top-level directories as case-insensitively sorted `favorites` (no files, no symlinks), `/` as the only location, and hidden directories only with `include_hidden=true`.", "run": null, "transcript": null } ] }, { "feature_id": "F14.mkdir", "family": "F14", "result": "not-run", "label": "`create_directory` creates missing parents, is idempotent and keeps existing contents, the new directory is listed by `search_subdirs`, and a path that is or runs through a file is 400.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.mkdir", "family": "F14", "result": "not-run", "label": "`create_directory` creates missing parents, is idempotent and keeps existing contents, the new directory is listed by `search_subdirs`, and a path that is or runs through a file is 400.", "run": null, "transcript": null } ] }, { "feature_id": "F14.restart", "family": "F14", "result": "not-run", "label": "after a restart, uploaded files, the scoped download and the trajectory are still served.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.restart", "family": "F14", "result": "not-run", "label": "after a restart, uploaded files, the scoped download and the trajectory are still served.", "run": null, "transcript": null } ] }, { "feature_id": "F14.scoped-conversation-errors", "family": "F14", "result": "not-run", "label": "a scoped route for an unknown conversation is 404 and does nothing; a malformed conversation id is 422.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.scoped-conversation-errors", "family": "F14", "result": "not-run", "label": "a scoped route for an unknown conversation is 404 and does nothing; a malformed conversation id is 422.", "run": null, "transcript": null } ] }, { "feature_id": "F14.scoped-path-guard", "family": "F14", "result": "not-run", "label": "scoped routes reject paths outside the workspace (another conversation's workspace, `..` traversal, a symlink escape, a sibling with the same prefix, a relative path) with 422 and write nothing, while the global route serves the same path.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.scoped-path-guard", "family": "F14", "result": "not-run", "label": "scoped routes reject paths outside the workspace (another conversation's workspace, `..` traversal, a symlink escape, a sibling with the same prefix, a relative path) with 422 and write nothing, while the global route serves the same path.", "run": null, "transcript": null } ] }, { "feature_id": "F14.scoped-trajectory-archive", "family": "F14", "result": "not-run", "label": "the scoped trajectory route serves only its own conversation (another or malformed id is 422), and the scoped archive packs the workspace, including the agent's file, but refuses a path outside it with 422.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.scoped-trajectory-archive", "family": "F14", "result": "not-run", "label": "the scoped trajectory route serves only its own conversation (another or malformed id is 422), and the scoped archive packs the workspace, including the agent's file, but refuses a path outside it with 422.", "run": null, "transcript": null } ] }, { "feature_id": "F14.scoped-transfer", "family": "F14", "result": "not-run", "label": "conversation-scoped upload, download and `create_directory` work inside the conversation's workspace, and the scoped download returns the file the agent wrote.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.scoped-transfer", "family": "F14", "result": "not-run", "label": "conversation-scoped upload, download and `create_directory` work inside the conversation's workspace, and the scoped download returns the file the agent wrote.", "run": null, "transcript": null } ] }, { "feature_id": "F14.sdk-workspace", "family": "F14", "result": "not-run", "label": "the SDK's `RemoteWorkspace.file_upload`/`file_download` round-trip bytes through the global routes and, with `runtime_conversation_id`, through the scoped routes; a scoped upload outside the workspace returns `success=False` with the 422 and writes nothing.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.sdk-workspace", "family": "F14", "result": "not-run", "label": "the SDK's `RemoteWorkspace.file_upload`/`file_download` round-trip bytes through the global routes and, with `runtime_conversation_id`, through the scoped routes; a scoped upload outside the workspace returns `success=False` with the 422 and writes nothing.", "run": null, "transcript": null } ] }, { "feature_id": "F14.subdirs", "family": "F14", "result": "not-run", "label": "`search_subdirs` lists only the immediate real subdirectories of an absolute path (no files, no symlinks, hidden only with `include_hidden=true`), sorted case-insensitively, with absolute paths.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.subdirs", "family": "F14", "result": "not-run", "label": "`search_subdirs` lists only the immediate real subdirectories of an absolute path (no files, no symlinks, hidden only with `include_hidden=true`), sorted case-insensitively, with absolute paths.", "run": null, "transcript": null } ] }, { "feature_id": "F14.subdirs-case-pagination", "family": "F14", "result": "not-run", "label": "following `next_page_id` across two directories that differ only by case (`Foo`, `foo`) reaches the second one instead of repeating the first page forever.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.subdirs-case-pagination", "family": "F14", "result": "not-run", "label": "following `next_page_id` across two directories that differ only by case (`Foo`, `foo`) reaches the second one instead of repeating the first page forever.", "run": null, "transcript": null } ] }, { "feature_id": "F14.subdirs-errors", "family": "F14", "result": "not-run", "label": "`search_subdirs` on a missing directory is 404, on a file 400, on a relative path 400.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.subdirs-errors", "family": "F14", "result": "not-run", "label": "`search_subdirs` on a missing directory is 404, on a file 400, on a relative path 400.", "run": null, "transcript": null } ] }, { "feature_id": "F14.subdirs-pagination", "family": "F14", "result": "not-run", "label": "`limit` and the `next_page_id` cursor page through the entries; `limit` outside 1..100 is 422.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.subdirs-pagination", "family": "F14", "result": "not-run", "label": "`limit` and the `next_page_id` cursor page through the entries; `limit` outside 1..100 is 422.", "run": null, "transcript": null } ] }, { "feature_id": "F14.trajectory", "family": "F14", "result": "not-run", "label": "`GET /api/file/download-trajectory/{id}` returns `.zip` with `meta.json`, `base_state.json` and one file per event, the LLM `api_key` redacted to `` and no Fernet ciphertext, for dashed and hex ids, and leaves no temp zip behind.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.trajectory", "family": "F14", "result": "not-run", "label": "`GET /api/file/download-trajectory/{id}` returns `.zip` with `meta.json`, `base_state.json` and one file per event, the LLM `api_key` redacted to `` and no Fernet ciphertext, for dashed and hex ids, and leaves no temp zip behind.", "run": null, "transcript": null } ] }, { "feature_id": "F14.trajectory-concurrent", "family": "F14", "result": "not-run", "label": "concurrent downloads of the same trajectory each return 200 and a valid zip (five rounds of six parallel downloads, 30 in all).", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.trajectory-concurrent", "family": "F14", "result": "not-run", "label": "concurrent downloads of the same trajectory each return 200 and a valid zip (five rounds of six parallel downloads, 30 in all).", "run": null, "transcript": null } ] }, { "feature_id": "F14.trajectory-errors", "family": "F14", "result": "not-run", "label": "an unknown conversation id is 404 `Conversation not found`, a malformed id 422.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.trajectory-errors", "family": "F14", "result": "not-run", "label": "an unknown conversation id is 404 `Conversation not found`, a malformed id 422.", "run": null, "transcript": null } ] }, { "feature_id": "F14.trajectory-init-path", "family": "F14", "result": "not-run", "label": "after `POST /api/init` sets `conversations_path` on a deferred-init server, trajectories stored there are downloadable.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.trajectory-init-path", "family": "F14", "result": "not-run", "label": "after `POST /api/init` sets `conversations_path` on a deferred-init server, trajectories stored there are downloadable.", "run": null, "transcript": null } ] }, { "feature_id": "F14.upload-atomic", "family": "F14", "result": "not-run", "label": "an overwrite that fails mid-write leaves the previous file intact; today the target is truncated before streaming and keeps a partial file.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.upload-atomic", "family": "F14", "result": "not-run", "label": "an overwrite that fails mid-write leaves the previous file intact; today the target is truncated before streaming and keeps a partial file.", "run": null, "transcript": null } ] }, { "feature_id": "F14.upload-download", "family": "F14", "result": "not-run", "label": "a multipart upload creates missing parent directories, a download returns byte-identical bytes as an `application/octet-stream` attachment named after the basename, and a second upload to the same path replaces the content.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.upload-download", "family": "F14", "result": "not-run", "label": "a multipart upload creates missing parent directories, a download returns byte-identical bytes as an `application/octet-stream` attachment named after the basename, and a second upload to the same path replaces the content.", "run": null, "transcript": null } ] }, { "feature_id": "F14.upload-errors", "family": "F14", "result": "not-run", "label": "an upload without the `file` part or without `path` is 422, and a relative `path` is 400 `Path must be absolute`.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.upload-errors", "family": "F14", "result": "not-run", "label": "an upload without the `file` part or without `path` is 422, and a relative `path` is 400 `Path must be absolute`.", "run": null, "transcript": null } ] }, { "feature_id": "F14.upload-onto-directory", "family": "F14", "result": "not-run", "label": "uploading onto an existing directory, or to a path whose parent is a file, is a client error (4xx); today it is a masked 500.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.upload-onto-directory", "family": "F14", "result": "not-run", "label": "uploading onto an existing directory, or to a path whose parent is a file, is a client error (4xx); today it is a masked 500.", "run": null, "transcript": null } ] }, { "feature_id": "F14.upload-size", "family": "F14", "result": "not-run", "label": "the SDK's `RemoteWorkspace.file_upload` and the TypeScript `RemoteWorkspace.fileUpload` report the uploaded `file_size`, as `LocalWorkspace.file_upload` does; today the server sends only `{\"success\": true}` and both report no size.", "run": null, "transcript": null, "variants": [ { "feature_id": "F14.upload-size", "family": "F14", "result": "not-run", "label": "the SDK's `RemoteWorkspace.file_upload` and the TypeScript `RemoteWorkspace.fileUpload` report the uploaded `file_size`, as `LocalWorkspace.file_upload` does; today the server sends only `{\"success\": true}` and both report no size.", "run": null, "transcript": null } ] }, { "feature_id": "F15.agent-changes", "family": "F15", "result": "pass", "label": "What the agent changed (`F15.agent-changes`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-23-what-the-agent-changed-f15-agent-changes.json", "failed_command": null, "variants": [ { "feature_id": "F15.agent-changes", "family": "F15", "result": "pass", "label": "What the agent changed (`F15.agent-changes`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-23-what-the-agent-changed-f15-agent-changes.json", "failed_command": null } ] }, { "feature_id": "F15.auth", "family": "F15", "result": "pass", "label": "Session key required (`F15.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-01-session-key-required-f15-auth.json", "failed_command": null, "variants": [ { "feature_id": "F15.auth", "family": "F15", "result": "pass", "label": "Session key required (`F15.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-01-session-key-required-f15-auth.json", "failed_command": null } ] }, { "feature_id": "F15.changes", "family": "F15", "result": "pass", "label": "Working-tree changes (`F15.changes`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-02-working-tree-changes-f15-changes.json", "failed_command": null, "variants": [ { "feature_id": "F15.changes", "family": "F15", "result": "pass", "label": "Working-tree changes (`F15.changes`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-02-working-tree-changes-f15-changes.json", "failed_command": null } ] }, { "feature_id": "F15.changes-base", "family": "F15", "result": "pass", "label": "Default display base (`F15.changes-base`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-04-default-display-base-f15-changes-base.json", "failed_command": null, "variants": [ { "feature_id": "F15.changes-base", "family": "F15", "result": "pass", "label": "Default display base (`F15.changes-base`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-04-default-display-base-f15-changes-base.json", "failed_command": null } ] }, { "feature_id": "F15.changes-nested-repo", "family": "F15", "result": "pass", "label": "Nested repository (`F15.changes-nested-repo`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-06-nested-repository-f15-changes-nested-rep.json", "failed_command": null, "variants": [ { "feature_id": "F15.changes-nested-repo", "family": "F15", "result": "pass", "label": "Nested repository (`F15.changes-nested-repo`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-06-nested-repository-f15-changes-nested-rep.json", "failed_command": null } ] }, { "feature_id": "F15.changes-ref", "family": "F15", "result": "pass", "label": "Explicit base and its errors (`F15.changes-ref`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-05-explicit-base-and-its-errors-f15-changes.json", "failed_command": null, "variants": [ { "feature_id": "F15.changes-ref", "family": "F15", "result": "pass", "label": "Explicit base and its errors (`F15.changes-ref`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-05-explicit-base-and-its-errors-f15-changes.json", "failed_command": null } ] }, { "feature_id": "F15.changes-repo-below-path", "family": "F15", "result": "xfail", "label": "Repository below the requested directory (`F15.changes-repo-below-path`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-07-repository-below-the-requested-directory.json", "failed_command": "control-agent-server api GET /api/git/changes --query path=\"$F/ws\" --expect 200 --check . len-eq 2 --check 0.path eq proj/README.md --check 1.path eq proj/notes.txt --save F15.changes-repo-below-path/ws", "variants": [ { "feature_id": "F15.changes-repo-below-path", "family": "F15", "result": "xfail", "label": "Repository below the requested directory (`F15.changes-repo-below-path`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-07-repository-below-the-requested-directory.json", "failed_command": "control-agent-server api GET /api/git/changes --query path=\"$F/ws\" --expect 200 --check . len-eq 2 --check 0.path eq proj/README.md --check 1.path eq proj/notes.txt --save F15.changes-repo-below-path/ws" } ] }, { "feature_id": "F15.changes-special-names", "family": "F15", "result": "xfail", "label": "File names with a space or non-ASCII characters (`F15.changes-special-names`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-08-file-names-with-a-space-or-non-ascii-cha.json", "failed_command": "control-agent-server api GET /api/git/changes --query path=\"$F/names\" --expect 200 --check . len-eq 1 --check 0.status eq UPDATED --check 0.path eq 'my notes.md' --save F15.changes-special-names/space", "variants": [ { "feature_id": "F15.changes-special-names", "family": "F15", "result": "xfail", "label": "File names with a space or non-ASCII characters (`F15.changes-special-names`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-08-file-names-with-a-space-or-non-ascii-cha.json", "failed_command": "control-agent-server api GET /api/git/changes --query path=\"$F/names\" --expect 200 --check . len-eq 1 --check 0.status eq UPDATED --check 0.path eq 'my notes.md' --save F15.changes-special-names/space" } ] }, { "feature_id": "F15.changes-statuses", "family": "F15", "result": "pass", "label": "Staged deletions, additions and renames (`F15.changes-statuses`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-03-staged-deletions-additions-and-renames-f.json", "failed_command": null, "variants": [ { "feature_id": "F15.changes-statuses", "family": "F15", "result": "pass", "label": "Staged deletions, additions and renames (`F15.changes-statuses`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-03-staged-deletions-additions-and-renames-f.json", "failed_command": null } ] }, { "feature_id": "F15.changes-subdir-paths", "family": "F15", "result": "xfail", "label": "Subdirectory of a repository (`F15.changes-subdir-paths`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-09-subdirectory-of-a-repository-f15-changes.json", "failed_command": "python3 - \"$F/subdir.json\" \"$F/subdir/src\" \"$F/subdir\" <<'PY'\nimport json, os, sys\nchanges = json.load(open(sys.argv[1]))\npaths = [c[\"path\"] for c in changes if c[\"status\"] != \"DELETED\"]\nbases = [b for b in sys.argv[2:] if all(os.path.exists(os.path.join(b, p)) for p in paths)]\nassert paths and bases, f\"no single base resolves every path: {changes}\"\nPY", "variants": [ { "feature_id": "F15.changes-subdir-paths", "family": "F15", "result": "xfail", "label": "Subdirectory of a repository (`F15.changes-subdir-paths`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-09-subdirectory-of-a-repository-f15-changes.json", "failed_command": "python3 - \"$F/subdir.json\" \"$F/subdir/src\" \"$F/subdir\" <<'PY'\nimport json, os, sys\nchanges = json.load(open(sys.argv[1]))\npaths = [c[\"path\"] for c in changes if c[\"status\"] != \"DELETED\"]\nbases = [b for b in sys.argv[2:] if all(os.path.exists(os.path.join(b, p)) for p in paths)]\nassert paths and bases, f\"no single base resolves every path: {changes}\"\nPY" } ] }, { "feature_id": "F15.commit-changes", "family": "F15", "result": "pass", "label": "Files of one commit (`F15.commit-changes`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-17-files-of-one-commit-f15-commit-changes.json", "failed_command": null, "variants": [ { "feature_id": "F15.commit-changes", "family": "F15", "result": "pass", "label": "Files of one commit (`F15.commit-changes`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-17-files-of-one-commit-f15-commit-changes.json", "failed_command": null } ] }, { "feature_id": "F15.commits", "family": "F15", "result": "fail", "label": "Commit history (`F15.commits`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-16-commit-history-f15-commits.json", "failed_command": "control-agent-server api GET /api/git/commits --query path=\"$REPO\" --expect 200 --check commits len-eq 2 --check has_more eq false --check commits.0.sha eq \"$C2\" --check commits.0.subject eq 'Add util' --check commits.0.short_sha eq \"$(git -C \"$REPO\" log -1 --format=%h)\" --check commits.0.author eq 'QA Verify' --check commits.0.timestamp matches '^\\d{4}-\\d\\d-\\d\\dT\\d\\d:\\d\\d:\\d\\d[+-]\\d\\d:\\d\\d$' --check commits.1.sha eq \"$C1\" --check commits.1.subject eq 'Initial commit' --save F15.commits/all", "variants": [ { "feature_id": "F15.commits", "family": "F15", "result": "fail", "label": "Commit history (`F15.commits`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-16-commit-history-f15-commits.json", "failed_command": "control-agent-server api GET /api/git/commits --query path=\"$REPO\" --expect 200 --check commits len-eq 2 --check has_more eq false --check commits.0.sha eq \"$C2\" --check commits.0.subject eq 'Add util' --check commits.0.short_sha eq \"$(git -C \"$REPO\" log -1 --format=%h)\" --check commits.0.author eq 'QA Verify' --check commits.0.timestamp matches '^\\d{4}-\\d\\d-\\d\\dT\\d\\d:\\d\\d:\\d\\d[+-]\\d\\d:\\d\\d$' --check commits.1.sha eq \"$C1\" --check commits.1.subject eq 'Initial commit' --save F15.commits/all" } ] }, { "feature_id": "F15.diff", "family": "F15", "result": "pass", "label": "Per-file diff (`F15.diff`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-11-per-file-diff-f15-diff.json", "failed_command": null, "variants": [ { "feature_id": "F15.diff", "family": "F15", "result": "pass", "label": "Per-file diff (`F15.diff`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-11-per-file-diff-f15-diff.json", "failed_command": null } ] }, { "feature_id": "F15.diff-commit", "family": "F15", "result": "pass", "label": "Diff of one commit (`F15.diff-commit`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-12-diff-of-one-commit-f15-diff-commit.json", "failed_command": null, "variants": [ { "feature_id": "F15.diff-commit", "family": "F15", "result": "pass", "label": "Diff of one commit (`F15.diff-commit`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-12-diff-of-one-commit-f15-diff-commit.json", "failed_command": null } ] }, { "feature_id": "F15.diff-deleted-file", "family": "F15", "result": "xfail", "label": "Diff of a deleted file (`F15.diff-deleted-file`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-15-diff-of-a-deleted-file-f15-diff-deleted-.json", "failed_command": "control-agent-server api GET /api/git/diff --query path=\"$F/deleted/src/util.py\" --expect 200 --check original eq \"$(git -C \"$REPO\" show HEAD:src/util.py)\" --check modified eq '' --save F15.diff-deleted-file/diff", "variants": [ { "feature_id": "F15.diff-deleted-file", "family": "F15", "result": "xfail", "label": "Diff of a deleted file (`F15.diff-deleted-file`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-15-diff-of-a-deleted-file-f15-diff-deleted-.json", "failed_command": "control-agent-server api GET /api/git/diff --query path=\"$F/deleted/src/util.py\" --expect 200 --check original eq \"$(git -C \"$REPO\" show HEAD:src/util.py)\" --check modified eq '' --save F15.diff-deleted-file/diff" } ] }, { "feature_id": "F15.diff-errors", "family": "F15", "result": "pass", "label": "Diff errors (`F15.diff-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-13-diff-errors-f15-diff-errors.json", "failed_command": null, "variants": [ { "feature_id": "F15.diff-errors", "family": "F15", "result": "pass", "label": "Diff errors (`F15.diff-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-13-diff-errors-f15-diff-errors.json", "failed_command": null } ] }, { "feature_id": "F15.diff-whitespace", "family": "F15", "result": "xfail", "label": "Phantom whitespace diff (`F15.diff-whitespace`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-14-phantom-whitespace-diff-f15-diff-whitesp.json", "failed_command": "python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); assert d[\"original\"] == d[\"modified\"], d' \"$F/ws-$name.json\"", "variants": [ { "feature_id": "F15.diff-whitespace", "family": "F15", "result": "xfail", "label": "Phantom whitespace diff (`F15.diff-whitespace`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-14-phantom-whitespace-diff-f15-diff-whitesp.json", "failed_command": "python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); assert d[\"original\"] == d[\"modified\"], d' \"$F/ws-$name.json\"" } ] }, { "feature_id": "F15.non-repo", "family": "F15", "result": "pass", "label": "Not a repository, or no commits yet (`F15.non-repo`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-10-not-a-repository-or-no-commits-yet-f15-n.json", "failed_command": null, "variants": [ { "feature_id": "F15.non-repo", "family": "F15", "result": "pass", "label": "Not a repository, or no commits yet (`F15.non-repo`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-10-not-a-repository-or-no-commits-yet-f15-n.json", "failed_command": null } ] }, { "feature_id": "F15.repo-search-errors", "family": "F15", "result": "pass", "label": "Provider and parameter validation (`F15.repo-search-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-25-provider-and-parameter-validation-f15-re.json", "failed_command": null, "variants": [ { "feature_id": "F15.repo-search-errors", "family": "F15", "result": "pass", "label": "Provider and parameter validation (`F15.repo-search-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-25-provider-and-parameter-validation-f15-re.json", "failed_command": null } ] }, { "feature_id": "F15.repo-search-github", "family": "F15", "result": "blocked", "label": "Live GitHub listing (`F15.repo-search-github`), blocked.", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": null, "failed_command": null, "variants": [ { "feature_id": "F15.repo-search-github", "family": "F15", "result": "blocked", "label": "Live GitHub listing (`F15.repo-search-github`), blocked.", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": null, "failed_command": null } ] }, { "feature_id": "F15.repo-search-timeout", "family": "F15", "result": "fail", "label": "GitHub timeout (`F15.repo-search-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-27-github-timeout-f15-repo-search-timeout.json", "failed_command": "control-agent-server api GET /api/git/repositories/search --query provider=github --expect 504 --expect-max-ms 30000 --check detail eq 'Internal Server Error' --check exception eq '504: GitHub repository search timed out' --check . not-contains qa-f15-not-a-real-token --save F15.repo-search-timeout/silent-proxy", "variants": [ { "feature_id": "F15.repo-search-timeout", "family": "F15", "result": "fail", "label": "GitHub timeout (`F15.repo-search-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-27-github-timeout-f15-repo-search-timeout.json", "failed_command": "control-agent-server api GET /api/git/repositories/search --query provider=github --expect 504 --expect-max-ms 30000 --check detail eq 'Internal Server Error' --check exception eq '504: GitHub repository search timed out' --check . not-contains qa-f15-not-a-real-token --save F15.repo-search-timeout/silent-proxy" } ] }, { "feature_id": "F15.repo-search-token", "family": "F15", "result": "pass", "label": "Token lookup (`F15.repo-search-token`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-24-token-lookup-f15-repo-search-token.json", "failed_command": null, "variants": [ { "feature_id": "F15.repo-search-token", "family": "F15", "result": "pass", "label": "Token lookup (`F15.repo-search-token`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-24-token-lookup-f15-repo-search-token.json", "failed_command": null } ] }, { "feature_id": "F15.repo-search-unreachable", "family": "F15", "result": "fail", "label": "GitHub unreachable (`F15.repo-search-unreachable`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-26-github-unreachable-f15-repo-search-unrea.json", "failed_command": "control-agent-server doctor", "variants": [ { "feature_id": "F15.repo-search-unreachable", "family": "F15", "result": "fail", "label": "GitHub unreachable (`F15.repo-search-unreachable`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-26-github-unreachable-f15-repo-search-unrea.json", "failed_command": "control-agent-server doctor" } ] }, { "feature_id": "F15.scoped", "family": "F15", "result": "pass", "label": "Conversation-scoped views (`F15.scoped`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-18-conversation-scoped-views-f15-scoped.json", "failed_command": null, "variants": [ { "feature_id": "F15.scoped", "family": "F15", "result": "pass", "label": "Conversation-scoped views (`F15.scoped`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-18-conversation-scoped-views-f15-scoped.json", "failed_command": null } ] }, { "feature_id": "F15.scoped-guard", "family": "F15", "result": "pass", "label": "Workspace guard (`F15.scoped-guard`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-19-workspace-guard-f15-scoped-guard.json", "failed_command": null, "variants": [ { "feature_id": "F15.scoped-guard", "family": "F15", "result": "pass", "label": "Workspace guard (`F15.scoped-guard`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-19-workspace-guard-f15-scoped-guard.json", "failed_command": null } ] }, { "feature_id": "F15.scoped-restart", "family": "F15", "result": "pass", "label": "After a restart (`F15.scoped-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-20-after-a-restart-f15-scoped-restart.json", "failed_command": null, "variants": [ { "feature_id": "F15.scoped-restart", "family": "F15", "result": "pass", "label": "After a restart (`F15.scoped-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-20-after-a-restart-f15-scoped-restart.json", "failed_command": null } ] }, { "feature_id": "F15.sdk-workspace", "family": "F15", "result": "pass", "label": "Python SDK workspace (`F15.sdk-workspace`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-21-python-sdk-workspace-f15-sdk-workspace.json", "failed_command": null, "variants": [ { "feature_id": "F15.sdk-workspace", "family": "F15", "result": "pass", "label": "Python SDK workspace (`F15.sdk-workspace`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-21-python-sdk-workspace-f15-sdk-workspace.json", "failed_command": null } ] }, { "feature_id": "F15.ts-client", "family": "F15", "result": "fail", "label": "TypeScript client (`F15.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-22-typescript-client-f15-ts-client.json", "failed_command": "( cd clients/typescript && npm ci && npm run build )", "variants": [ { "feature_id": "F15.ts-client", "family": "F15", "result": "fail", "label": "TypeScript client (`F15.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-22-typescript-client-f15-ts-client.json", "failed_command": "( cd clients/typescript && npm ci && npm run build )" } ] }, { "feature_id": "F16.agent-artifact", "family": "F16", "result": "pass", "label": "The agent's own artifact (`F16.agent-artifact`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-14-the-agent-s-own-artifact-f16-agent-artif.json", "failed_command": null, "variants": [ { "feature_id": "F16.agent-artifact", "family": "F16", "result": "pass", "label": "The agent's own artifact (`F16.agent-artifact`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-14-the-agent-s-own-artifact-f16-agent-artif.json", "failed_command": null } ] }, { "feature_id": "F16.auth", "family": "F16", "result": "pass", "label": "Header or cookie (`F16.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-13-header-or-cookie-f16-auth.json", "failed_command": null, "variants": [ { "feature_id": "F16.auth", "family": "F16", "result": "pass", "label": "Header or cookie (`F16.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-13-header-or-cookie-f16-auth.json", "failed_command": null } ] }, { "feature_id": "F16.content-types", "family": "F16", "result": "pass", "label": "Content types (`F16.content-types`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-03-content-types-f16-content-types.json", "failed_command": null, "variants": [ { "feature_id": "F16.content-types", "family": "F16", "result": "pass", "label": "Content types (`F16.content-types`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-03-content-types-f16-content-types.json", "failed_command": null } ] }, { "feature_id": "F16.deleted-conversation", "family": "F16", "result": "pass", "label": "Deleted conversation (`F16.deleted-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-11-deleted-conversation-f16-deleted-convers.json", "failed_command": null, "variants": [ { "feature_id": "F16.deleted-conversation", "family": "F16", "result": "pass", "label": "Deleted conversation (`F16.deleted-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-11-deleted-conversation-f16-deleted-convers.json", "failed_command": null } ] }, { "feature_id": "F16.dir-index", "family": "F16", "result": "pass", "label": "Directory index and missing files (`F16.dir-index`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-04-directory-index-and-missing-files-f16-di.json", "failed_command": null, "variants": [ { "feature_id": "F16.dir-index", "family": "F16", "result": "pass", "label": "Directory index and missing files (`F16.dir-index`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-04-directory-index-and-missing-files-f16-di.json", "failed_command": null } ] }, { "feature_id": "F16.docker-root-cookie", "family": "F16", "result": "xfail", "label": "Docker-mode root and the cookie (`F16.docker-root-cookie`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-25-docker-mode-root-and-the-cookie-f16-dock.json", "failed_command": "control-agent-server api GET \"/$Z/workspace\" --run \"$DK\" --auth none --jar qa-f16-docker --expect 404 --save F16.docker-root-cookie/slashless", "variants": [ { "feature_id": "F16.docker-root-cookie", "family": "F16", "result": "xfail", "label": "Docker-mode root and the cookie (`F16.docker-root-cookie`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-25-docker-mode-root-and-the-cookie-f16-dock.json", "failed_command": "control-agent-server api GET \"/$Z/workspace\" --run \"$DK\" --auth none --jar qa-f16-docker --expect 404 --save F16.docker-root-cookie/slashless" } ] }, { "feature_id": "F16.file-bytes", "family": "F16", "result": "pass", "label": "File bytes and headers (`F16.file-bytes`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-02-file-bytes-and-headers-f16-file-bytes.json", "failed_command": null, "variants": [ { "feature_id": "F16.file-bytes", "family": "F16", "result": "pass", "label": "File bytes and headers (`F16.file-bytes`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-02-file-bytes-and-headers-f16-file-bytes.json", "failed_command": null } ] }, { "feature_id": "F16.live-content", "family": "F16", "result": "pass", "label": "Live content (`F16.live-content`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-06-live-content-f16-live-content.json", "failed_command": null, "variants": [ { "feature_id": "F16.live-content", "family": "F16", "result": "pass", "label": "Live content (`F16.live-content`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-06-live-content-f16-live-content.json", "failed_command": null } ] }, { "feature_id": "F16.range", "family": "F16", "result": "pass", "label": "Byte ranges (`F16.range`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-05-byte-ranges-f16-range.json", "failed_command": null, "variants": [ { "feature_id": "F16.range", "family": "F16", "result": "pass", "label": "Byte ranges (`F16.range`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-05-byte-ranges-f16-range.json", "failed_command": null } ] }, { "feature_id": "F16.restart", "family": "F16", "result": "pass", "label": "Across a restart (`F16.restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-15-across-a-restart-f16-restart.json", "failed_command": null, "variants": [ { "feature_id": "F16.restart", "family": "F16", "result": "pass", "label": "Across a restart (`F16.restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-15-across-a-restart-f16-restart.json", "failed_command": null } ] }, { "feature_id": "F16.root-index", "family": "F16", "result": "pass", "label": "Root index (`F16.root-index`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-01-root-index-f16-root-index.json", "failed_command": null, "variants": [ { "feature_id": "F16.root-index", "family": "F16", "result": "pass", "label": "Root index (`F16.root-index`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-01-root-index-f16-root-index.json", "failed_command": null } ] }, { "feature_id": "F16.symlink-escape", "family": "F16", "result": "pass", "label": "Symlinks (`F16.symlink-escape`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-08-symlinks-f16-symlink-escape.json", "failed_command": null, "variants": [ { "feature_id": "F16.symlink-escape", "family": "F16", "result": "pass", "label": "Symlinks (`F16.symlink-escape`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-08-symlinks-f16-symlink-escape.json", "failed_command": null } ] }, { "feature_id": "F16.traversal-rejected", "family": "F16", "result": "pass", "label": "Traversal (`F16.traversal-rejected`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-07-traversal-f16-traversal-rejected.json", "failed_command": null, "variants": [ { "feature_id": "F16.traversal-rejected", "family": "F16", "result": "pass", "label": "Traversal (`F16.traversal-rejected`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-07-traversal-f16-traversal-rejected.json", "failed_command": null } ] }, { "feature_id": "F16.unknown-conversation", "family": "F16", "result": "pass", "label": "Unknown and malformed ids (`F16.unknown-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-09-unknown-and-malformed-ids-f16-unknown-co.json", "failed_command": null, "variants": [ { "feature_id": "F16.unknown-conversation", "family": "F16", "result": "pass", "label": "Unknown and malformed ids (`F16.unknown-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-09-unknown-and-malformed-ids-f16-unknown-co.json", "failed_command": null } ] }, { "feature_id": "F16.vscode-auth", "family": "F16", "result": "pass", "label": "VS Code routes are header-only (`F16.vscode-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-18-vs-code-routes-are-header-only-f16-vscod.json", "failed_command": null, "variants": [ { "feature_id": "F16.vscode-auth", "family": "F16", "result": "pass", "label": "VS Code routes are header-only (`F16.vscode-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-18-vs-code-routes-are-header-only-f16-vscod.json", "failed_command": null } ] }, { "feature_id": "F16.vscode-disabled", "family": "F16", "result": "pass", "label": "VS Code disabled (`F16.vscode-disabled`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-16-vs-code-disabled-f16-vscode-disabled.json", "failed_command": null, "variants": [ { "feature_id": "F16.vscode-disabled", "family": "F16", "result": "pass", "label": "VS Code disabled (`F16.vscode-disabled`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-16-vs-code-disabled-f16-vscode-disabled.json", "failed_command": null } ] }, { "feature_id": "F16.vscode-enabled-status", "family": "F16", "result": "fail", "label": "Enabled but not running (`F16.vscode-enabled-status`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-19-enabled-but-not-running-f16-vscode-enabl.json", "failed_command": "control-agent-server doctor", "variants": [ { "feature_id": "F16.vscode-enabled-status", "family": "F16", "result": "fail", "label": "Enabled but not running (`F16.vscode-enabled-status`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-19-enabled-but-not-running-f16-vscode-enabl.json", "failed_command": "control-agent-server doctor" } ] }, { "feature_id": "F16.vscode-folder-encoding", "family": "F16", "result": "xfail", "label": "Folder with `&` and `#` (`F16.vscode-folder-encoding`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-22-folder-with-and-f16-vscode-folder-encodi.json", "failed_command": "python3 -c \"$ROUNDTRIP\" \"$OWS\"", "variants": [ { "feature_id": "F16.vscode-folder-encoding", "family": "F16", "result": "xfail", "label": "Folder with `&` and `#` (`F16.vscode-folder-encoding`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-22-folder-with-and-f16-vscode-folder-encodi.json", "failed_command": "python3 -c \"$ROUNDTRIP\" \"$OWS\"" } ] }, { "feature_id": "F16.vscode-running", "family": "F16", "result": "blocked", "label": "VS Code actually running (`F16.vscode-running`), blocked.", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": null, "failed_command": null, "variants": [ { "feature_id": "F16.vscode-running", "family": "F16", "result": "blocked", "label": "VS Code actually running (`F16.vscode-running`), blocked.", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": null, "failed_command": null } ] }, { "feature_id": "F16.vscode-scoped-guards", "family": "F16", "result": "pass", "label": "Scoped guards (`F16.vscode-scoped-guards`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-17-scoped-guards-f16-vscode-scoped-guards.json", "failed_command": null, "variants": [ { "feature_id": "F16.vscode-scoped-guards", "family": "F16", "result": "pass", "label": "Scoped guards (`F16.vscode-scoped-guards`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-17-scoped-guards-f16-vscode-scoped-guards.json", "failed_command": null } ] }, { "feature_id": "F16.vscode-url-after-init", "family": "F16", "result": "fail", "label": "Key delivered by init, no URL (`F16.vscode-url-after-init`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-24-key-delivered-by-init-no-url-f16-vscode-.json", "failed_command": "DI=$(control-agent-server launch --new --no-auth --deferred-init --vscode --name f16-deferred --config-json '{\"vscode_port\": '\"$VPORT\"'}' --print-run)", "variants": [ { "feature_id": "F16.vscode-url-after-init", "family": "F16", "result": "fail", "label": "Key delivered by init, no URL (`F16.vscode-url-after-init`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-24-key-delivered-by-init-no-url-f16-vscode-.json", "failed_command": "DI=$(control-agent-server launch --new --no-auth --deferred-init --vscode --name f16-deferred --config-json '{\"vscode_port\": '\"$VPORT\"'}' --print-run)" } ] }, { "feature_id": "F16.vscode-url-no-key", "family": "F16", "result": "fail", "label": "No key, no URL (`F16.vscode-url-no-key`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-23-no-key-no-url-f16-vscode-url-no-key.json", "failed_command": "NK=$(control-agent-server launch --new --no-auth --vscode --name f16-nokey --config-json '{\"vscode_port\": '\"$VPORT\"'}' --print-run)", "variants": [ { "feature_id": "F16.vscode-url-no-key", "family": "F16", "result": "fail", "label": "No key, no URL (`F16.vscode-url-no-key`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-23-no-key-no-url-f16-vscode-url-no-key.json", "failed_command": "NK=$(control-agent-server launch --new --no-auth --vscode --name f16-nokey --config-json '{\"vscode_port\": '\"$VPORT\"'}' --print-run)" } ] }, { "feature_id": "F16.vscode-url-not-running", "family": "F16", "result": "xfail", "label": "Not running, still advertised (`F16.vscode-url-not-running`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-21-not-running-still-advertised-f16-vscode-.json", "failed_command": "control-agent-server api GET /api/vscode/url --expect 200 --check . eq '{\"url\": null}' --save F16.vscode-url-not-running/global", "variants": [ { "feature_id": "F16.vscode-url-not-running", "family": "F16", "result": "xfail", "label": "Not running, still advertised (`F16.vscode-url-not-running`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-21-not-running-still-advertised-f16-vscode-.json", "failed_command": "control-agent-server api GET /api/vscode/url --expect 200 --check . eq '{\"url\": null}' --save F16.vscode-url-not-running/global" } ] }, { "feature_id": "F16.vscode-url-shape", "family": "F16", "result": "fail", "label": "URL shape (`F16.vscode-url-shape`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-20-url-shape-f16-vscode-url-shape.json", "failed_command": "control-agent-server api GET /api/vscode/url --expect 200 --check url eq \"http://localhost:$VPORT/?tkn=$KEY&folder=workspace\" --save F16.vscode-url-shape/global", "variants": [ { "feature_id": "F16.vscode-url-shape", "family": "F16", "result": "fail", "label": "URL shape (`F16.vscode-url-shape`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-20-url-shape-f16-vscode-url-shape.json", "failed_command": "control-agent-server api GET /api/vscode/url --expect 200 --check url eq \"http://localhost:$VPORT/?tkn=$KEY&folder=workspace\" --save F16.vscode-url-shape/global" } ] }, { "feature_id": "F16.workspace-dir-missing", "family": "F16", "result": "pass", "label": "Workspace directory gone (`F16.workspace-dir-missing`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-10-workspace-directory-gone-f16-workspace-d.json", "failed_command": null, "variants": [ { "feature_id": "F16.workspace-dir-missing", "family": "F16", "result": "pass", "label": "Workspace directory gone (`F16.workspace-dir-missing`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-10-workspace-directory-gone-f16-workspace-d.json", "failed_command": null } ] }, { "feature_id": "F16.worktree-served", "family": "F16", "result": "pass", "label": "Worktree conversation (`F16.worktree-served`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-12-worktree-conversation-f16-worktree-serve.json", "failed_command": null, "variants": [ { "feature_id": "F16.worktree-served", "family": "F16", "result": "pass", "label": "Worktree conversation (`F16.worktree-served`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-12-worktree-conversation-f16-worktree-serve.json", "failed_command": null } ] }, { "feature_id": "F17.add-parents", "family": "F17", "result": "pass", "label": "Register parents (`F17.add-parents`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-11-register-parents-f17-add-parents.json", "failed_command": null, "variants": [ { "feature_id": "F17.add-parents", "family": "F17", "result": "pass", "label": "Register parents (`F17.add-parents`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-11-register-parents-f17-add-parents.json", "failed_command": null } ] }, { "feature_id": "F17.add-workspaces", "family": "F17", "result": "pass", "label": "Register a repository and a plain directory (`F17.add-workspaces`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-03-register-a-repository-and-a-plain-direct.json", "failed_command": null, "variants": [ { "feature_id": "F17.add-workspaces", "family": "F17", "result": "pass", "label": "Register a repository and a plain directory (`F17.add-workspaces`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-03-register-a-repository-and-a-plain-direct.json", "failed_command": null } ] }, { "feature_id": "F17.auth-required", "family": "F17", "result": "pass", "label": "Session key required (`F17.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-16-session-key-required-f17-auth-required.json", "failed_command": null, "variants": [ { "feature_id": "F17.auth-required", "family": "F17", "result": "pass", "label": "Session key required (`F17.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-16-session-key-required-f17-auth-required.json", "failed_command": null } ] }, { "feature_id": "F17.concurrent-adds", "family": "F17", "result": "pass", "label": "Concurrent writes (`F17.concurrent-adds`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-17-concurrent-writes-f17-concurrent-adds.json", "failed_command": null, "variants": [ { "feature_id": "F17.concurrent-adds", "family": "F17", "result": "pass", "label": "Concurrent writes (`F17.concurrent-adds`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-17-concurrent-writes-f17-concurrent-adds.json", "failed_command": null } ] }, { "feature_id": "F17.corrupted-file", "family": "F17", "result": "pass", "label": "Corrupted file (`F17.corrupted-file`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-20-corrupted-file-f17-corrupted-file.json", "failed_command": null, "variants": [ { "feature_id": "F17.corrupted-file", "family": "F17", "result": "pass", "label": "Corrupted file (`F17.corrupted-file`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-20-corrupted-file-f17-corrupted-file.json", "failed_command": null } ] }, { "feature_id": "F17.delete-not-found", "family": "F17", "result": "pass", "label": "Unknown paths (`F17.delete-not-found`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-07-unknown-paths-f17-delete-not-found.json", "failed_command": null, "variants": [ { "feature_id": "F17.delete-not-found", "family": "F17", "result": "pass", "label": "Unknown paths (`F17.delete-not-found`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-07-unknown-paths-f17-delete-not-found.json", "failed_command": null } ] }, { "feature_id": "F17.delete-parent", "family": "F17", "result": "pass", "label": "Delete a parent (`F17.delete-parent`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-13-delete-a-parent-f17-delete-parent.json", "failed_command": null, "variants": [ { "feature_id": "F17.delete-parent", "family": "F17", "result": "pass", "label": "Delete a parent (`F17.delete-parent`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-13-delete-a-parent-f17-delete-parent.json", "failed_command": null } ] }, { "feature_id": "F17.delete-workspace", "family": "F17", "result": "pass", "label": "Delete a workspace (`F17.delete-workspace`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-06-delete-a-workspace-f17-delete-workspace.json", "failed_command": null, "variants": [ { "feature_id": "F17.delete-workspace", "family": "F17", "result": "pass", "label": "Delete a workspace (`F17.delete-workspace`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-06-delete-a-workspace-f17-delete-workspace.json", "failed_command": null } ] }, { "feature_id": "F17.future-schema", "family": "F17", "result": "pass", "label": "Future or invalid schema (`F17.future-schema`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-21-future-or-invalid-schema-f17-future-sche.json", "failed_command": null, "variants": [ { "feature_id": "F17.future-schema", "family": "F17", "result": "pass", "label": "Future or invalid schema (`F17.future-schema`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-21-future-or-invalid-schema-f17-future-sche.json", "failed_command": null } ] }, { "feature_id": "F17.idempotent-add", "family": "F17", "result": "pass", "label": "Idempotent re-add (`F17.idempotent-add`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-05-idempotent-re-add-f17-idempotent-add.json", "failed_command": null, "variants": [ { "feature_id": "F17.idempotent-add", "family": "F17", "result": "pass", "label": "Idempotent re-add (`F17.idempotent-add`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-05-idempotent-re-add-f17-idempotent-add.json", "failed_command": null } ] }, { "feature_id": "F17.io-error", "family": "F17", "result": "pass", "label": "Unreadable file (`F17.io-error`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-22-unreadable-file-f17-io-error.json", "failed_command": null, "variants": [ { "feature_id": "F17.io-error", "family": "F17", "result": "pass", "label": "Unreadable file (`F17.io-error`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-22-unreadable-file-f17-io-error.json", "failed_command": null } ] }, { "feature_id": "F17.list-empty", "family": "F17", "result": "pass", "label": "Empty registry (`F17.list-empty`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-01-empty-registry-f17-list-empty.json", "failed_command": null, "variants": [ { "feature_id": "F17.list-empty", "family": "F17", "result": "pass", "label": "Empty registry (`F17.list-empty`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-01-empty-registry-f17-list-empty.json", "failed_command": null } ] }, { "feature_id": "F17.lists-independent", "family": "F17", "result": "pass", "label": "Two independent lists (`F17.lists-independent`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-14-two-independent-lists-f17-lists-independ.json", "failed_command": null, "variants": [ { "feature_id": "F17.lists-independent", "family": "F17", "result": "pass", "label": "Two independent lists (`F17.lists-independent`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-14-two-independent-lists-f17-lists-independ.json", "failed_command": null } ] }, { "feature_id": "F17.noop-delete-no-file", "family": "F17", "result": "xfail", "label": "A refused delete on a fresh directory (`F17.noop-delete-no-file`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-02-a-refused-delete-on-a-fresh-directory-f1.json", "failed_command": "control-agent-server state ls home/.openhands/workspaces.json --expect-count 0", "variants": [ { "feature_id": "F17.noop-delete-no-file", "family": "F17", "result": "xfail", "label": "A refused delete on a fresh directory (`F17.noop-delete-no-file`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-02-a-refused-delete-on-a-fresh-directory-f1.json", "failed_command": "control-agent-server state ls home/.openhands/workspaces.json --expect-count 0" } ] }, { "feature_id": "F17.noop-write-readonly", "family": "F17", "result": "fail", "label": "No-op writes on read-only storage (`F17.noop-write-readonly`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-23-no-op-writes-on-read-only-storage-f17-no.json", "failed_command": "chattr +i \"$PDIR\"", "variants": [ { "feature_id": "F17.noop-write-readonly", "family": "F17", "result": "fail", "label": "No-op writes on read-only storage (`F17.noop-write-readonly`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-23-no-op-writes-on-read-only-storage-f17-no.json", "failed_command": "chattr +i \"$PDIR\"" } ] }, { "feature_id": "F17.noop-write-untouched", "family": "F17", "result": "xfail", "label": "No-op writes leave the file alone (`F17.noop-write-untouched`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-08-no-op-writes-leave-the-file-alone-f17-no.json", "failed_command": "test \"$(stat -c '%i %y' \"$WS\")\" = \"$STAMP\"", "variants": [ { "feature_id": "F17.noop-write-untouched", "family": "F17", "result": "xfail", "label": "No-op writes leave the file alone (`F17.noop-write-untouched`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-08-no-op-writes-leave-the-file-alone-f17-no.json", "failed_command": "test \"$(stat -c '%i %y' \"$WS\")\" = \"$STAMP\"" } ] }, { "feature_id": "F17.parent-children", "family": "F17", "result": "pass", "label": "Children of a parent (`F17.parent-children`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-12-children-of-a-parent-f17-parent-children.json", "failed_command": null, "variants": [ { "feature_id": "F17.parent-children", "family": "F17", "result": "pass", "label": "Children of a parent (`F17.parent-children`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-12-children-of-a-parent-f17-parent-children.json", "failed_command": null } ] }, { "feature_id": "F17.paths-opaque", "family": "F17", "result": "pass", "label": "Paths are opaque strings (`F17.paths-opaque`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-09-paths-are-opaque-strings-f17-paths-opaqu.json", "failed_command": null, "variants": [ { "feature_id": "F17.paths-opaque", "family": "F17", "result": "pass", "label": "Paths are opaque strings (`F17.paths-opaque`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-09-paths-are-opaque-strings-f17-paths-opaqu.json", "failed_command": null } ] }, { "feature_id": "F17.persist-restart", "family": "F17", "result": "pass", "label": "Persistence across a restart (`F17.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-15-persistence-across-a-restart-f17-persist.json", "failed_command": null, "variants": [ { "feature_id": "F17.persist-restart", "family": "F17", "result": "pass", "label": "Persistence across a restart (`F17.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-15-persistence-across-a-restart-f17-persist.json", "failed_command": null } ] }, { "feature_id": "F17.persistence-location", "family": "F17", "result": "pass", "label": "Location without `OH_PERSISTENCE_DIR` (`F17.persistence-location`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-24-location-without-oh-persistence-dir-f17-.json", "failed_command": null, "variants": [ { "feature_id": "F17.persistence-location", "family": "F17", "result": "pass", "label": "Location without `OH_PERSISTENCE_DIR` (`F17.persistence-location`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-24-location-without-oh-persistence-dir-f17-.json", "failed_command": null } ] }, { "feature_id": "F17.shared-dir", "family": "F17", "result": "pass", "label": "Two servers, one file (`F17.shared-dir`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-18-two-servers-one-file-f17-shared-dir.json", "failed_command": null, "variants": [ { "feature_id": "F17.shared-dir", "family": "F17", "result": "pass", "label": "Two servers, one file (`F17.shared-dir`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-18-two-servers-one-file-f17-shared-dir.json", "failed_command": null } ] }, { "feature_id": "F17.ts-client", "family": "F17", "result": "pass", "label": "TypeScript client (`F17.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-19-typescript-client-f17-ts-client.json", "failed_command": null, "variants": [ { "feature_id": "F17.ts-client", "family": "F17", "result": "pass", "label": "TypeScript client (`F17.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-19-typescript-client-f17-ts-client.json", "failed_command": null } ] }, { "feature_id": "F17.validation", "family": "F17", "result": "pass", "label": "Validation (`F17.validation`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-10-validation-f17-validation.json", "failed_command": null, "variants": [ { "feature_id": "F17.validation", "family": "F17", "result": "pass", "label": "Validation (`F17.validation`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-10-validation-f17-validation.json", "failed_command": null } ] }, { "feature_id": "F17.wire-format", "family": "F17", "result": "pass", "label": "Wire format (`F17.wire-format`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-04-wire-format-f17-wire-format.json", "failed_command": null, "variants": [ { "feature_id": "F17.wire-format", "family": "F17", "result": "pass", "label": "Wire format (`F17.wire-format`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-04-wire-format-f17-wire-format.json", "failed_command": null } ] }, { "feature_id": "F18.active-profile-pointer", "family": "F18", "result": "pass", "label": "Active LLM profile pointer (`F18.active-profile-pointer`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-12-active-llm-profile-pointer-f18-active-pr.json", "failed_command": null, "variants": [ { "feature_id": "F18.active-profile-pointer", "family": "F18", "result": "pass", "label": "Active LLM profile pointer (`F18.active-profile-pointer`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-12-active-llm-profile-pointer-f18-active-pr.json", "failed_command": null } ] }, { "feature_id": "F18.agent-diff-merge", "family": "F18", "result": "pass", "label": "Sparse agent diff and unset (`F18.agent-diff-merge`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-08-sparse-agent-diff-and-unset-f18-agent-di.json", "failed_command": null, "variants": [ { "feature_id": "F18.agent-diff-merge", "family": "F18", "result": "pass", "label": "Sparse agent diff and unset (`F18.agent-diff-merge`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-08-sparse-agent-diff-and-unset-f18-agent-di.json", "failed_command": null } ] }, { "feature_id": "F18.agent-kind-switch", "family": "F18", "result": "pass", "label": "Agent kind switch (`F18.agent-kind-switch`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-13-agent-kind-switch-f18-agent-kind-switch.json", "failed_command": null, "variants": [ { "feature_id": "F18.agent-kind-switch", "family": "F18", "result": "pass", "label": "Agent kind switch (`F18.agent-kind-switch`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-13-agent-kind-switch-f18-agent-kind-switch.json", "failed_command": null } ] }, { "feature_id": "F18.agent-schema", "family": "F18", "result": "pass", "label": "Agent settings schema (`F18.agent-schema`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-02-agent-settings-schema-f18-agent-schema.json", "failed_command": null, "variants": [ { "feature_id": "F18.agent-schema", "family": "F18", "result": "pass", "label": "Agent settings schema (`F18.agent-schema`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-02-agent-settings-schema-f18-agent-schema.json", "failed_command": null } ] }, { "feature_id": "F18.auth-required", "family": "F18", "result": "pass", "label": "Key required (`F18.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-04-key-required-f18-auth-required.json", "failed_command": null, "variants": [ { "feature_id": "F18.auth-required", "family": "F18", "result": "pass", "label": "Key required (`F18.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-04-key-required-f18-auth-required.json", "failed_command": null } ] }, { "feature_id": "F18.conversation-diff", "family": "F18", "result": "pass", "label": "Conversation settings (`F18.conversation-diff`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-09-conversation-settings-f18-conversation-d.json", "failed_command": null, "variants": [ { "feature_id": "F18.conversation-diff", "family": "F18", "result": "pass", "label": "Conversation settings (`F18.conversation-diff`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-09-conversation-settings-f18-conversation-d.json", "failed_command": null } ] }, { "feature_id": "F18.conversation-schema", "family": "F18", "result": "pass", "label": "Conversation settings schema (`F18.conversation-schema`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-03-conversation-settings-schema-f18-convers.json", "failed_command": null, "variants": [ { "feature_id": "F18.conversation-schema", "family": "F18", "result": "pass", "label": "Conversation settings schema (`F18.conversation-schema`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-03-conversation-settings-schema-f18-convers.json", "failed_command": null } ] }, { "feature_id": "F18.corrupted-files", "family": "F18", "result": "pass", "label": "Corrupted files (`F18.corrupted-files`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-23-corrupted-files-f18-corrupted-files.json", "failed_command": null, "variants": [ { "feature_id": "F18.corrupted-files", "family": "F18", "result": "pass", "label": "Corrupted files (`F18.corrupted-files`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-23-corrupted-files-f18-corrupted-files.json", "failed_command": null } ] }, { "feature_id": "F18.dangling-pointers", "family": "F18", "result": "xfail", "label": "Dangling agent-profile pointer (`F18.dangling-pointers`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-27-dangling-agent-profile-pointer-f18-dangl.json", "failed_command": "control-agent-server api PATCH /api/settings --json '{\"active_agent_profile_id\": \"00000000-0000-4000-8000-0000000000f1\"}' --expect 404,422 --save F18.dangling-pointers/agent-profile-id", "variants": [ { "feature_id": "F18.dangling-pointers", "family": "F18", "result": "xfail", "label": "Dangling agent-profile pointer (`F18.dangling-pointers`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-27-dangling-agent-profile-pointer-f18-dangl.json", "failed_command": "control-agent-server api PATCH /api/settings --json '{\"active_agent_profile_id\": \"00000000-0000-4000-8000-0000000000f1\"}' --expect 404,422 --save F18.dangling-pointers/agent-profile-id" }, { "feature_id": "F18.dangling-pointers", "family": "F18", "result": "xfail", "label": "Dangling meta-profile pointer (`F18.dangling-pointers`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-28-dangling-meta-profile-pointer-f18-dangli.json", "failed_command": "control-agent-server api PATCH /api/settings --json '{\"active_meta_profile\": \"qa-f18-no-router\"}' --expect 404,422 --save F18.dangling-pointers/meta-profile" } ] }, { "feature_id": "F18.encrypted-at-rest", "family": "F18", "result": "pass", "label": "Encrypted at rest (`F18.encrypted-at-rest`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-17-encrypted-at-rest-f18-encrypted-at-rest.json", "failed_command": null, "variants": [ { "feature_id": "F18.encrypted-at-rest", "family": "F18", "result": "pass", "label": "Encrypted at rest (`F18.encrypted-at-rest`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-17-encrypted-at-rest-f18-encrypted-at-rest.json", "failed_command": null } ] }, { "feature_id": "F18.expose-modes", "family": "F18", "result": "pass", "label": "Exposure modes (`F18.expose-modes`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-06-exposure-modes-f18-expose-modes.json", "failed_command": null, "variants": [ { "feature_id": "F18.expose-modes", "family": "F18", "result": "pass", "label": "Exposure modes (`F18.expose-modes`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-06-exposure-modes-f18-expose-modes.json", "failed_command": null } ] }, { "feature_id": "F18.get-defaults", "family": "F18", "result": "pass", "label": "Fresh defaults (`F18.get-defaults`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-01-fresh-defaults-f18-get-defaults.json", "failed_command": null, "variants": [ { "feature_id": "F18.get-defaults", "family": "F18", "result": "pass", "label": "Fresh defaults (`F18.get-defaults`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-01-fresh-defaults-f18-get-defaults.json", "failed_command": null } ] }, { "feature_id": "F18.misc-settings-merge", "family": "F18", "result": "pass", "label": "Opaque misc settings (`F18.misc-settings-merge`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-10-opaque-misc-settings-f18-misc-settings-m.json", "failed_command": null, "variants": [ { "feature_id": "F18.misc-settings-merge", "family": "F18", "result": "pass", "label": "Opaque misc settings (`F18.misc-settings-merge`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-10-opaque-misc-settings-f18-misc-settings-m.json", "failed_command": null } ] }, { "feature_id": "F18.no-cipher", "family": "F18", "result": "pass", "label": "No cipher (`F18.no-cipher`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-24-no-cipher-f18-no-cipher.json", "failed_command": null, "variants": [ { "feature_id": "F18.no-cipher", "family": "F18", "result": "pass", "label": "No cipher (`F18.no-cipher`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-24-no-cipher-f18-no-cipher.json", "failed_command": null } ] }, { "feature_id": "F18.patch-llm", "family": "F18", "result": "pass", "label": "Set the LLM (`F18.patch-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-05-set-the-llm-f18-patch-llm.json", "failed_command": null, "variants": [ { "feature_id": "F18.patch-llm", "family": "F18", "result": "pass", "label": "Set the LLM (`F18.patch-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-05-set-the-llm-f18-patch-llm.json", "failed_command": null } ] }, { "feature_id": "F18.patch-validation", "family": "F18", "result": "pass", "label": "Rejected PATCH bodies (`F18.patch-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-11-rejected-patch-bodies-f18-patch-validati.json", "failed_command": null, "variants": [ { "feature_id": "F18.patch-validation", "family": "F18", "result": "pass", "label": "Rejected PATCH bodies (`F18.patch-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-11-rejected-patch-bodies-f18-patch-validati.json", "failed_command": null } ] }, { "feature_id": "F18.persist-restart", "family": "F18", "result": "pass", "label": "Persistence across a restart (`F18.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-22-persistence-across-a-restart-f18-persist.json", "failed_command": null, "variants": [ { "feature_id": "F18.persist-restart", "family": "F18", "result": "pass", "label": "Persistence across a restart (`F18.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-22-persistence-across-a-restart-f18-persist.json", "failed_command": null } ] }, { "feature_id": "F18.roundtrip", "family": "F18", "result": "pass", "label": "Post back what GET returned (`F18.roundtrip`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-07-post-back-what-get-returned-f18-roundtri.json", "failed_command": null, "variants": [ { "feature_id": "F18.roundtrip", "family": "F18", "result": "pass", "label": "Post back what GET returned (`F18.roundtrip`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-07-post-back-what-get-returned-f18-roundtri.json", "failed_command": null } ] }, { "feature_id": "F18.sdk-clone-repos", "family": "F18", "result": "pass", "label": "SDK clones with the stored token (`F18.sdk-clone-repos`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-20-sdk-clones-with-the-stored-token-f18-sdk.json", "failed_command": null, "variants": [ { "feature_id": "F18.sdk-clone-repos", "family": "F18", "result": "pass", "label": "SDK clones with the stored token (`F18.sdk-clone-repos`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-20-sdk-clones-with-the-stored-token-f18-sdk.json", "failed_command": null } ] }, { "feature_id": "F18.sdk-remote-workspace", "family": "F18", "result": "pass", "label": "SDK consumer (`F18.sdk-remote-workspace`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-19-sdk-consumer-f18-sdk-remote-workspace.json", "failed_command": null, "variants": [ { "feature_id": "F18.sdk-remote-workspace", "family": "F18", "result": "pass", "label": "SDK consumer (`F18.sdk-remote-workspace`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-19-sdk-consumer-f18-sdk-remote-workspace.json", "failed_command": null } ] }, { "feature_id": "F18.secret-delete", "family": "F18", "result": "pass", "label": "Delete a secret (`F18.secret-delete`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-15-delete-a-secret-f18-secret-delete.json", "failed_command": null, "variants": [ { "feature_id": "F18.secret-delete", "family": "F18", "result": "pass", "label": "Delete a secret (`F18.secret-delete`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-15-delete-a-secret-f18-secret-delete.json", "failed_command": null } ] }, { "feature_id": "F18.secret-empty-value", "family": "F18", "result": "pass", "label": "Placeholder secret value (`F18.secret-empty-value`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-25-placeholder-secret-value-f18-secret-empt.json", "failed_command": null, "variants": [ { "feature_id": "F18.secret-empty-value", "family": "F18", "result": "pass", "label": "Placeholder secret value (`F18.secret-empty-value`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-25-placeholder-secret-value-f18-secret-empt.json", "failed_command": null }, { "feature_id": "F18.secret-empty-value", "family": "F18", "result": "pass", "label": "Empty secret value (`F18.secret-empty-value`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-26-empty-secret-value-f18-secret-empty-valu.json", "failed_command": null } ] }, { "feature_id": "F18.secret-key-rotation", "family": "F18", "result": "xfail", "label": "Cipher key rotation (`F18.secret-key-rotation`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-32-cipher-key-rotation-f18-secret-key-rotat.json", "failed_command": "control-agent-server state cat home/.openhands/secrets.json --check custom_secrets.QA_F18_TOKEN.secret matches '^gAAAAA'", "variants": [ { "feature_id": "F18.secret-key-rotation", "family": "F18", "result": "xfail", "label": "Cipher key rotation (`F18.secret-key-rotation`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-32-cipher-key-rotation-f18-secret-key-rotat.json", "failed_command": "control-agent-server state cat home/.openhands/secrets.json --check custom_secrets.QA_F18_TOKEN.secret matches '^gAAAAA'" } ] }, { "feature_id": "F18.secret-name-contract", "family": "F18", "result": "xfail", "label": "Documented status for an invalid name (`F18.secret-name-contract`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-31-documented-status-for-an-invalid-name-f1.json", "failed_command": "control-agent-server api GET /api/settings/secrets/1bad --expect \"${CLAIMED:-422}\" --save F18.secret-name-contract/invalid-name", "variants": [ { "feature_id": "F18.secret-name-contract", "family": "F18", "result": "xfail", "label": "Documented status for an invalid name (`F18.secret-name-contract`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-31-documented-status-for-an-invalid-name-f1.json", "failed_command": "control-agent-server api GET /api/settings/secrets/1bad --expect \"${CLAIMED:-422}\" --save F18.secret-name-contract/invalid-name" } ] }, { "feature_id": "F18.secret-name-validation", "family": "F18", "result": "pass", "label": "Secret names (`F18.secret-name-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-16-secret-names-f18-secret-name-validation.json", "failed_command": null, "variants": [ { "feature_id": "F18.secret-name-validation", "family": "F18", "result": "pass", "label": "Secret names (`F18.secret-name-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-16-secret-names-f18-secret-name-validation.json", "failed_command": null } ] }, { "feature_id": "F18.secret-put-list-get", "family": "F18", "result": "pass", "label": "Upsert, list and read a secret (`F18.secret-put-list-get`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-14-upsert-list-and-read-a-secret-f18-secret.json", "failed_command": null, "variants": [ { "feature_id": "F18.secret-put-list-get", "family": "F18", "result": "pass", "label": "Upsert, list and read a secret (`F18.secret-put-list-get`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-14-upsert-list-and-read-a-secret-f18-secret.json", "failed_command": null } ] }, { "feature_id": "F18.secret-scope-agent-profile", "family": "F18", "result": "pass", "label": "Secrets scoped by an agent profile (`F18.secret-scope-agent-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-18-secrets-scoped-by-an-agent-profile-f18-s.json", "failed_command": null, "variants": [ { "feature_id": "F18.secret-scope-agent-profile", "family": "F18", "result": "pass", "label": "Secrets scoped by an agent profile (`F18.secret-scope-agent-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-18-secrets-scoped-by-an-agent-profile-f18-s.json", "failed_command": null } ] }, { "feature_id": "F18.secret-scope-profile-errors", "family": "F18", "result": "xfail", "label": "Agent profile that no longer loads (`F18.secret-scope-profile-errors`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-29-agent-profile-that-no-longer-loads-f18-s.json", "failed_command": "control-agent-server api GET /api/settings/secrets --query \"agent_profile_id=$BROKEN\" --expect 4xx --save F18.secret-scope-profile-errors/unloadable", "variants": [ { "feature_id": "F18.secret-scope-profile-errors", "family": "F18", "result": "xfail", "label": "Agent profile that no longer loads (`F18.secret-scope-profile-errors`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-29-agent-profile-that-no-longer-loads-f18-s.json", "failed_command": "control-agent-server api GET /api/settings/secrets --query \"agent_profile_id=$BROKEN\" --expect 4xx --save F18.secret-scope-profile-errors/unloadable" }, { "feature_id": "F18.secret-scope-profile-errors", "family": "F18", "result": "xfail", "label": "Agent-profile store busy (`F18.secret-scope-profile-errors`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-30-agent-profile-store-busy-f18-secret-scop.json", "failed_command": "control-agent-server api GET /api/settings/secrets --query agent_profile_id=00000000-0000-4000-8000-0000000000f1 --timeout 60 --expect 503 --save F18.secret-scope-profile-errors/secrets-busy" } ] }, { "feature_id": "F18.ts-client", "family": "F18", "result": "pass", "label": "TypeScript client (`F18.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-21-typescript-client-f18-ts-client.json", "failed_command": null, "variants": [ { "feature_id": "F18.ts-client", "family": "F18", "result": "pass", "label": "TypeScript client (`F18.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-21-typescript-client-f18-ts-client.json", "failed_command": null } ] }, { "feature_id": "F19.auth", "family": "F19", "result": "pass", "label": "Auth (`F19.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-01-auth-f19-auth.json", "failed_command": null, "variants": [ { "feature_id": "F19.auth", "family": "F19", "result": "pass", "label": "Auth (`F19.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-01-auth-f19-auth.json", "failed_command": null } ] }, { "feature_id": "F19.conversation-secrets-at-rest", "family": "F19", "result": "pass", "label": "Credentials at rest (`F19.conversation-secrets-at-rest`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-27-credentials-at-rest-f19-conversation-sec.json", "failed_command": null, "variants": [ { "feature_id": "F19.conversation-secrets-at-rest", "family": "F19", "result": "pass", "label": "Credentials at rest (`F19.conversation-secrets-at-rest`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-27-credentials-at-rest-f19-conversation-sec.json", "failed_command": null } ] }, { "feature_id": "F19.conversation-tool", "family": "F19", "result": "pass", "label": "The agent uses the MCP tool (`F19.conversation-tool`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-25-the-agent-uses-the-mcp-tool-f19-conversa.json", "failed_command": null, "variants": [ { "feature_id": "F19.conversation-tool", "family": "F19", "result": "pass", "label": "The agent uses the MCP tool (`F19.conversation-tool`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-25-the-agent-uses-the-mcp-tool-f19-conversa.json", "failed_command": null } ] }, { "feature_id": "F19.disabled-server", "family": "F19", "result": "pass", "label": "Disabled servers stay off (`F19.disabled-server`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-24-disabled-servers-stay-off-f19-disabled-s.json", "failed_command": null, "variants": [ { "feature_id": "F19.disabled-server", "family": "F19", "result": "pass", "label": "Disabled servers stay off (`F19.disabled-server`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-24-disabled-servers-stay-off-f19-disabled-s.json", "failed_command": null } ] }, { "feature_id": "F19.oauth-callback-bad-port", "family": "F19", "result": "xfail", "label": "Port out of range (`F19.oauth-callback-bad-port`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-18-port-out-of-range-f19-oauth-callback-bad.json", "failed_command": "control-agent-server api POST \"/api/mcp/oauth/callback/$JB\" --json '{\"callback_url\":\"http://localhost:99999/callback?code=x\"}' --expect 400 --save F19.oauth-callback-bad-port/port-99999", "variants": [ { "feature_id": "F19.oauth-callback-bad-port", "family": "F19", "result": "xfail", "label": "Port out of range (`F19.oauth-callback-bad-port`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-18-port-out-of-range-f19-oauth-callback-bad.json", "failed_command": "control-agent-server api POST \"/api/mcp/oauth/callback/$JB\" --json '{\"callback_url\":\"http://localhost:99999/callback?code=x\"}' --expect 400 --save F19.oauth-callback-bad-port/port-99999" } ] }, { "feature_id": "F19.oauth-callback-replay", "family": "F19", "result": "xfail", "label": "Replayed callback (`F19.oauth-callback-replay`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-17-replayed-callback-f19-oauth-callback-rep.json", "failed_command": "control-agent-server api POST \"/api/mcp/oauth/callback/$J\" --json \"$RBODY\" --expect 200,4xx --save F19.oauth-callback-replay/replay", "variants": [ { "feature_id": "F19.oauth-callback-replay", "family": "F19", "result": "xfail", "label": "Replayed callback (`F19.oauth-callback-replay`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-17-replayed-callback-f19-oauth-callback-rep.json", "failed_command": "control-agent-server api POST \"/api/mcp/oauth/callback/$J\" --json \"$RBODY\" --expect 200,4xx --save F19.oauth-callback-replay/replay" } ] }, { "feature_id": "F19.oauth-callback-validation", "family": "F19", "result": "pass", "label": "Callback URL checks on a live job (`F19.oauth-callback-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-19-callback-url-checks-on-a-live-job-f19-oa.json", "failed_command": null, "variants": [ { "feature_id": "F19.oauth-callback-validation", "family": "F19", "result": "pass", "label": "Callback URL checks on a live job (`F19.oauth-callback-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-19-callback-url-checks-on-a-live-job-f19-oa.json", "failed_command": null } ] }, { "feature_id": "F19.oauth-consent-denied", "family": "F19", "result": "pass", "label": "Denied consent (`F19.oauth-consent-denied`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-20-denied-consent-f19-oauth-consent-denied.json", "failed_command": null, "variants": [ { "feature_id": "F19.oauth-consent-denied", "family": "F19", "result": "pass", "label": "Denied consent (`F19.oauth-consent-denied`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-20-denied-consent-f19-oauth-consent-denied.json", "failed_command": null } ] }, { "feature_id": "F19.oauth-consent-timeout", "family": "F19", "result": "pass", "label": "Consent timeout (`F19.oauth-consent-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-21-consent-timeout-f19-oauth-consent-timeou.json", "failed_command": null, "variants": [ { "feature_id": "F19.oauth-consent-timeout", "family": "F19", "result": "pass", "label": "Consent timeout (`F19.oauth-consent-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-21-consent-timeout-f19-oauth-consent-timeou.json", "failed_command": null } ] }, { "feature_id": "F19.oauth-flow", "family": "F19", "result": "pass", "label": "Full OAuth consent (`F19.oauth-flow`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-16-full-oauth-consent-f19-oauth-flow.json", "failed_command": null, "variants": [ { "feature_id": "F19.oauth-flow", "family": "F19", "result": "pass", "label": "Full OAuth consent (`F19.oauth-flow`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-16-full-oauth-consent-f19-oauth-flow.json", "failed_command": null } ] }, { "feature_id": "F19.oauth-inline-writeback", "family": "F19", "result": "pass", "label": "Inline OAuth server refresh (`F19.oauth-inline-writeback`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-22-inline-oauth-server-refresh-f19-oauth-in.json", "failed_command": null, "variants": [ { "feature_id": "F19.oauth-inline-writeback", "family": "F19", "result": "pass", "label": "Inline OAuth server refresh (`F19.oauth-inline-writeback`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-22-inline-oauth-server-refresh-f19-oauth-in.json", "failed_command": null } ] }, { "feature_id": "F19.oauth-refresh-writeback", "family": "F19", "result": "pass", "label": "Stored OAuth token refresh (`F19.oauth-refresh-writeback`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-26-stored-oauth-token-refresh-f19-oauth-ref.json", "failed_command": null, "variants": [ { "feature_id": "F19.oauth-refresh-writeback", "family": "F19", "result": "pass", "label": "Stored OAuth token refresh (`F19.oauth-refresh-writeback`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-26-stored-oauth-token-refresh-f19-oauth-ref.json", "failed_command": null } ] }, { "feature_id": "F19.oauth-start-errors", "family": "F19", "result": "pass", "label": "OAuth start refusals and early outcomes (`F19.oauth-start-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-14-oauth-start-refusals-and-early-outcomes-.json", "failed_command": null, "variants": [ { "feature_id": "F19.oauth-start-errors", "family": "F19", "result": "pass", "label": "OAuth start refusals and early outcomes (`F19.oauth-start-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-14-oauth-start-refusals-and-early-outcomes-.json", "failed_command": null } ] }, { "feature_id": "F19.oauth-stored-state", "family": "F19", "result": "pass", "label": "Stored OAuth state (`F19.oauth-stored-state`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-23-stored-oauth-state-f19-oauth-stored-stat.json", "failed_command": null, "variants": [ { "feature_id": "F19.oauth-stored-state", "family": "F19", "result": "pass", "label": "Stored OAuth state (`F19.oauth-stored-state`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-23-stored-oauth-state-f19-oauth-stored-stat.json", "failed_command": null } ] }, { "feature_id": "F19.oauth-unknown-job", "family": "F19", "result": "pass", "label": "Unknown jobs (`F19.oauth-unknown-job`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-15-unknown-jobs-f19-oauth-unknown-job.json", "failed_command": null, "variants": [ { "feature_id": "F19.oauth-unknown-job", "family": "F19", "result": "pass", "label": "Unknown jobs (`F19.oauth-unknown-job`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-15-unknown-jobs-f19-oauth-unknown-job.json", "failed_command": null } ] }, { "feature_id": "F19.restart", "family": "F19", "result": "pass", "label": "Restart (`F19.restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-28-restart-f19-restart.json", "failed_command": null, "variants": [ { "feature_id": "F19.restart", "family": "F19", "result": "pass", "label": "Restart (`F19.restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-28-restart-f19-restart.json", "failed_command": null } ] }, { "feature_id": "F19.settings-create", "family": "F19", "result": "pass", "label": "Create one server (`F19.settings-create`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-02-create-one-server-f19-settings-create.json", "failed_command": null, "variants": [ { "feature_id": "F19.settings-create", "family": "F19", "result": "pass", "label": "Create one server (`F19.settings-create`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-02-create-one-server-f19-settings-create.json", "failed_command": null } ] }, { "feature_id": "F19.settings-delete", "family": "F19", "result": "pass", "label": "Delete one server (`F19.settings-delete`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-06-delete-one-server-f19-settings-delete.json", "failed_command": null, "variants": [ { "feature_id": "F19.settings-delete", "family": "F19", "result": "pass", "label": "Delete one server (`F19.settings-delete`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-06-delete-one-server-f19-settings-delete.json", "failed_command": null } ] }, { "feature_id": "F19.settings-empty-server", "family": "F19", "result": "xfail", "label": "Server with neither command nor url (`F19.settings-empty-server`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-04-server-with-neither-command-nor-url-f19-.json", "failed_command": "false", "variants": [ { "feature_id": "F19.settings-empty-server", "family": "F19", "result": "xfail", "label": "Server with neither command nor url (`F19.settings-empty-server`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-04-server-with-neither-command-nor-url-f19-.json", "failed_command": "false" } ] }, { "feature_id": "F19.settings-patch", "family": "F19", "result": "pass", "label": "Sparse patch (`F19.settings-patch`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-05-sparse-patch-f19-settings-patch.json", "failed_command": null, "variants": [ { "feature_id": "F19.settings-patch", "family": "F19", "result": "pass", "label": "Sparse patch (`F19.settings-patch`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-05-sparse-patch-f19-settings-patch.json", "failed_command": null } ] }, { "feature_id": "F19.settings-validation", "family": "F19", "result": "pass", "label": "Invalid servers (`F19.settings-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-03-invalid-servers-f19-settings-validation.json", "failed_command": null, "variants": [ { "feature_id": "F19.settings-validation", "family": "F19", "result": "pass", "label": "Invalid servers (`F19.settings-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-03-invalid-servers-f19-settings-validation.json", "failed_command": null } ] }, { "feature_id": "F19.test-connect-failures", "family": "F19", "result": "pass", "label": "Connection failures (`F19.test-connect-failures`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-10-connection-failures-f19-test-connect-fai.json", "failed_command": null, "variants": [ { "feature_id": "F19.test-connect-failures", "family": "F19", "result": "pass", "label": "Connection failures (`F19.test-connect-failures`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-10-connection-failures-f19-test-connect-fai.json", "failed_command": null } ] }, { "feature_id": "F19.test-encrypted-env", "family": "F19", "result": "pass", "label": "Encrypted env from settings (`F19.test-encrypted-env`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-12-encrypted-env-from-settings-f19-test-enc.json", "failed_command": null, "variants": [ { "feature_id": "F19.test-encrypted-env", "family": "F19", "result": "pass", "label": "Encrypted env from settings (`F19.test-encrypted-env`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-12-encrypted-env-from-settings-f19-test-enc.json", "failed_command": null } ] }, { "feature_id": "F19.test-remote-bearer", "family": "F19", "result": "pass", "label": "Remote server behind a bearer token (`F19.test-remote-bearer`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-13-remote-server-behind-a-bearer-token-f19-.json", "failed_command": null, "variants": [ { "feature_id": "F19.test-remote-bearer", "family": "F19", "result": "pass", "label": "Remote server behind a bearer token (`F19.test-remote-bearer`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-13-remote-server-behind-a-bearer-token-f19-.json", "failed_command": null } ] }, { "feature_id": "F19.test-stdio", "family": "F19", "result": "pass", "label": "Test a stdio server (`F19.test-stdio`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-08-test-a-stdio-server-f19-test-stdio.json", "failed_command": null, "variants": [ { "feature_id": "F19.test-stdio", "family": "F19", "result": "pass", "label": "Test a stdio server (`F19.test-stdio`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-08-test-a-stdio-server-f19-test-stdio.json", "failed_command": null } ] }, { "feature_id": "F19.test-timeout", "family": "F19", "result": "pass", "label": "Handshake timeout (`F19.test-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-11-handshake-timeout-f19-test-timeout.json", "failed_command": null, "variants": [ { "feature_id": "F19.test-timeout", "family": "F19", "result": "pass", "label": "Handshake timeout (`F19.test-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-11-handshake-timeout-f19-test-timeout.json", "failed_command": null } ] }, { "feature_id": "F19.test-tool-errors", "family": "F19", "result": "fail", "label": "Tool failures stay in `tool_result` (`F19.test-tool-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-09-tool-failures-stay-in-tool-result-f19-te.json", "failed_command": "control-agent-server api POST /api/mcp/test --timeout 20 --expect-max-ms 15000 --save F19.test-tool-errors/slow-tool --json \"$(jq -nc --argjson s \"$TOOLS_SRV\" '{server: $s, timeout: 3, tool_call: {name: \"qa_sleep\", arguments: {seconds: 30}}}')\" --expect 200 --check ok eq true --check tool_result.is_error eq true --check tool_result.text contains 'timed out after 3.0 seconds'", "variants": [ { "feature_id": "F19.test-tool-errors", "family": "F19", "result": "fail", "label": "Tool failures stay in `tool_result` (`F19.test-tool-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-09-tool-failures-stay-in-tool-result-f19-te.json", "failed_command": "control-agent-server api POST /api/mcp/test --timeout 20 --expect-max-ms 15000 --save F19.test-tool-errors/slow-tool --json \"$(jq -nc --argjson s \"$TOOLS_SRV\" '{server: $s, timeout: 3, tool_call: {name: \"qa_sleep\", arguments: {seconds: 30}}}')\" --expect 200 --check ok eq true --check tool_result.is_error eq true --check tool_result.text contains 'timed out after 3.0 seconds'" } ] }, { "feature_id": "F19.test-validation", "family": "F19", "result": "pass", "label": "Connection test body validation (`F19.test-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-07-connection-test-body-validation-f19-test.json", "failed_command": null, "variants": [ { "feature_id": "F19.test-validation", "family": "F19", "result": "pass", "label": "Connection test body validation (`F19.test-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-07-connection-test-body-validation-f19-test.json", "failed_command": null } ] }, { "feature_id": "F20.activate", "family": "F20", "result": "pass", "label": "Activate (`F20.activate`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-20-activate-f20-activate.json", "failed_command": null, "variants": [ { "feature_id": "F20.activate", "family": "F20", "result": "pass", "label": "Activate (`F20.activate`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-20-activate-f20-activate.json", "failed_command": null } ] }, { "feature_id": "F20.activate-acp", "family": "F20", "result": "pass", "label": "ACP settings (`F20.activate-acp`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-22-acp-settings-f20-activate-acp.json", "failed_command": null, "variants": [ { "feature_id": "F20.activate-acp", "family": "F20", "result": "pass", "label": "ACP settings (`F20.activate-acp`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-22-acp-settings-f20-activate-acp.json", "failed_command": null } ] }, { "feature_id": "F20.activate-conversation", "family": "F20", "result": "pass", "label": "Activation drives conversations (`F20.activate-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-21-activation-drives-conversations-f20-acti.json", "failed_command": null, "variants": [ { "feature_id": "F20.activate-conversation", "family": "F20", "result": "pass", "label": "Activation drives conversations (`F20.activate-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-21-activation-drives-conversations-f20-acti.json", "failed_command": null } ] }, { "feature_id": "F20.activate-dangling-provider", "family": "F20", "result": "pass", "label": "Dangling provider link (`F20.activate-dangling-provider`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-25-dangling-provider-link-f20-activate-dang.json", "failed_command": null, "variants": [ { "feature_id": "F20.activate-dangling-provider", "family": "F20", "result": "pass", "label": "Dangling provider link (`F20.activate-dangling-provider`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-25-dangling-provider-link-f20-activate-dang.json", "failed_command": null } ] }, { "feature_id": "F20.activate-linked-provider", "family": "F20", "result": "pass", "label": "Linked provider (`F20.activate-linked-provider`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-26-linked-provider-f20-activate-linked-prov.json", "failed_command": null, "variants": [ { "feature_id": "F20.activate-linked-provider", "family": "F20", "result": "pass", "label": "Linked provider (`F20.activate-linked-provider`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-26-linked-provider-f20-activate-linked-prov.json", "failed_command": null } ] }, { "feature_id": "F20.auth-required", "family": "F20", "result": "pass", "label": "Key required (`F20.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-01-key-required-f20-auth-required.json", "failed_command": null, "variants": [ { "feature_id": "F20.auth-required", "family": "F20", "result": "pass", "label": "Key required (`F20.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-01-key-required-f20-auth-required.json", "failed_command": null } ] }, { "feature_id": "F20.corrupt-files", "family": "F20", "result": "pass", "label": "Corrupted files (`F20.corrupt-files`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-09-corrupted-files-f20-corrupt-files.json", "failed_command": null, "variants": [ { "feature_id": "F20.corrupt-files", "family": "F20", "result": "pass", "label": "Corrupted files (`F20.corrupt-files`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-09-corrupted-files-f20-corrupt-files.json", "failed_command": null } ] }, { "feature_id": "F20.delete", "family": "F20", "result": "pass", "label": "Delete (`F20.delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-33-delete-f20-delete.json", "failed_command": null, "variants": [ { "feature_id": "F20.delete", "family": "F20", "result": "pass", "label": "Delete (`F20.delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-33-delete-f20-delete.json", "failed_command": null } ] }, { "feature_id": "F20.delete-guarded", "family": "F20", "result": "pass", "label": "Guarded delete (`F20.delete-guarded`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-32-guarded-delete-f20-delete-guarded.json", "failed_command": null, "variants": [ { "feature_id": "F20.delete-guarded", "family": "F20", "result": "pass", "label": "Guarded delete (`F20.delete-guarded`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-32-guarded-delete-f20-delete-guarded.json", "failed_command": null } ] }, { "feature_id": "F20.get-secrets", "family": "F20", "result": "pass", "label": "Reading the key (`F20.get-secrets`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-04-reading-the-key-f20-get-secrets.json", "failed_command": null, "variants": [ { "feature_id": "F20.get-secrets", "family": "F20", "result": "pass", "label": "Reading the key (`F20.get-secrets`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-04-reading-the-key-f20-get-secrets.json", "failed_command": null } ] }, { "feature_id": "F20.json-alias-guard", "family": "F20", "result": "xfail", "label": "Delete through the `.json` alias (`F20.json-alias-guard`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-34-delete-through-the-json-alias-f20-json-a.json", "failed_command": "jq -e '.status == 409' \"$F20/alias-delete.json\"", "variants": [ { "feature_id": "F20.json-alias-guard", "family": "F20", "result": "xfail", "label": "Delete through the `.json` alias (`F20.json-alias-guard`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-34-delete-through-the-json-alias-f20-json-a.json", "failed_command": "jq -e '.status == 409' \"$F20/alias-delete.json\"" } ] }, { "feature_id": "F20.key-encrypted-at-rest", "family": "F20", "result": "pass", "label": "Encrypted at rest (`F20.key-encrypted-at-rest`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-03-encrypted-at-rest-f20-key-encrypted-at-r.json", "failed_command": null, "variants": [ { "feature_id": "F20.key-encrypted-at-rest", "family": "F20", "result": "pass", "label": "Encrypted at rest (`F20.key-encrypted-at-rest`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-03-encrypted-at-rest-f20-key-encrypted-at-r.json", "failed_command": null } ] }, { "feature_id": "F20.name-rules", "family": "F20", "result": "pass", "label": "Name rules (`F20.name-rules`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-08-name-rules-f20-name-rules.json", "failed_command": null, "variants": [ { "feature_id": "F20.name-rules", "family": "F20", "result": "pass", "label": "Name rules (`F20.name-rules`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-08-name-rules-f20-name-rules.json", "failed_command": null } ] }, { "feature_id": "F20.no-cipher", "family": "F20", "result": "pass", "label": "No cipher (`F20.no-cipher`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-28-no-cipher-f20-no-cipher.json", "failed_command": null, "variants": [ { "feature_id": "F20.no-cipher", "family": "F20", "result": "pass", "label": "No cipher (`F20.no-cipher`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-28-no-cipher-f20-no-cipher.json", "failed_command": null } ] }, { "feature_id": "F20.not-found", "family": "F20", "result": "pass", "label": "Unknown names (`F20.not-found`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-07-unknown-names-f20-not-found.json", "failed_command": null, "variants": [ { "feature_id": "F20.not-found", "family": "F20", "result": "pass", "label": "Unknown names (`F20.not-found`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-07-unknown-names-f20-not-found.json", "failed_command": null } ] }, { "feature_id": "F20.persist-restart", "family": "F20", "result": "pass", "label": "Restart (`F20.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-29-restart-f20-persist-restart.json", "failed_command": null, "variants": [ { "feature_id": "F20.persist-restart", "family": "F20", "result": "pass", "label": "Restart (`F20.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-29-restart-f20-persist-restart.json", "failed_command": null } ] }, { "feature_id": "F20.profile-limit", "family": "F20", "result": "pass", "label": "Profile limit (`F20.profile-limit`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-12-profile-limit-f20-profile-limit.json", "failed_command": null, "variants": [ { "feature_id": "F20.profile-limit", "family": "F20", "result": "pass", "label": "Profile limit (`F20.profile-limit`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-12-profile-limit-f20-profile-limit.json", "failed_command": null } ] }, { "feature_id": "F20.rename", "family": "F20", "result": "pass", "label": "Rename (`F20.rename`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-30-rename-f20-rename.json", "failed_command": null, "variants": [ { "feature_id": "F20.rename", "family": "F20", "result": "pass", "label": "Rename (`F20.rename`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-30-rename-f20-rename.json", "failed_command": null } ] }, { "feature_id": "F20.rename-errors", "family": "F20", "result": "pass", "label": "Rename errors (`F20.rename-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-31-rename-errors-f20-rename-errors.json", "failed_command": null, "variants": [ { "feature_id": "F20.rename-errors", "family": "F20", "result": "pass", "label": "Rename errors (`F20.rename-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-31-rename-errors-f20-rename-errors.json", "failed_command": null } ] }, { "feature_id": "F20.save", "family": "F20", "result": "pass", "label": "Save (`F20.save`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-02-save-f20-save.json", "failed_command": null, "variants": [ { "feature_id": "F20.save", "family": "F20", "result": "pass", "label": "Save (`F20.save`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-02-save-f20-save.json", "failed_command": null } ] }, { "feature_id": "F20.save-encrypted-roundtrip", "family": "F20", "result": "pass", "label": "Encrypted round trip (`F20.save-encrypted-roundtrip`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-05-encrypted-round-trip-f20-save-encrypted-.json", "failed_command": null, "variants": [ { "feature_id": "F20.save-encrypted-roundtrip", "family": "F20", "result": "pass", "label": "Encrypted round trip (`F20.save-encrypted-roundtrip`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-05-encrypted-round-trip-f20-save-encrypted-.json", "failed_command": null } ] }, { "feature_id": "F20.save-without-secrets", "family": "F20", "result": "pass", "label": "Save without secrets (`F20.save-without-secrets`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-06-save-without-secrets-f20-save-without-se.json", "failed_command": null, "variants": [ { "feature_id": "F20.save-without-secrets", "family": "F20", "result": "pass", "label": "Save without secrets (`F20.save-without-secrets`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-06-save-without-secrets-f20-save-without-se.json", "failed_command": null } ] }, { "feature_id": "F20.sdk-get-llm", "family": "F20", "result": "pass", "label": "SDK consumer (`F20.sdk-get-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-23-sdk-consumer-f20-sdk-get-llm.json", "failed_command": null, "variants": [ { "feature_id": "F20.sdk-get-llm", "family": "F20", "result": "pass", "label": "SDK consumer (`F20.sdk-get-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-23-sdk-consumer-f20-sdk-get-llm.json", "failed_command": null } ] }, { "feature_id": "F20.sdk-get-llm-linked", "family": "F20", "result": "pass", "label": "SDK on a linked profile (`F20.sdk-get-llm-linked`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-27-sdk-on-a-linked-profile-f20-sdk-get-llm-.json", "failed_command": null, "variants": [ { "feature_id": "F20.sdk-get-llm-linked", "family": "F20", "result": "pass", "label": "SDK on a linked profile (`F20.sdk-get-llm-linked`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-27-sdk-on-a-linked-profile-f20-sdk-get-llm-.json", "failed_command": null } ] }, { "feature_id": "F20.secret-key-rotation", "family": "F20", "result": "xfail", "label": "Secret key rotation (`F20.secret-key-rotation`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-35-secret-key-rotation-f20-secret-key-rotat.json", "failed_command": "control-agent-server api POST /api/profiles/qa-rotate/activate --expect 4xx --save F20.secret-key-rotation/activate", "variants": [ { "feature_id": "F20.secret-key-rotation", "family": "F20", "result": "xfail", "label": "Secret key rotation (`F20.secret-key-rotation`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-35-secret-key-rotation-f20-secret-key-rotat.json", "failed_command": "control-agent-server api POST /api/profiles/qa-rotate/activate --expect 4xx --save F20.secret-key-rotation/activate" } ] }, { "feature_id": "F20.store-busy", "family": "F20", "result": "pass", "label": "Store busy (`F20.store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-10-store-busy-f20-store-busy.json", "failed_command": null, "variants": [ { "feature_id": "F20.store-busy", "family": "F20", "result": "pass", "label": "Store busy (`F20.store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-10-store-busy-f20-store-busy.json", "failed_command": null } ] }, { "feature_id": "F20.store-busy-responsive", "family": "F20", "result": "xfail", "label": "Lock wait blocks the server (`F20.store-busy-responsive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-11-lock-wait-blocks-the-server-f20-store-bu.json", "failed_command": "jq -e '.elapsed_ms <= 3000' \"$F20/busy-alive.json\"", "variants": [ { "feature_id": "F20.store-busy-responsive", "family": "F20", "result": "xfail", "label": "Lock wait blocks the server (`F20.store-busy-responsive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-11-lock-wait-blocks-the-server-f20-store-bu.json", "failed_command": "jq -e '.elapsed_ms <= 3000' \"$F20/busy-alive.json\"" } ] }, { "feature_id": "F20.ts-client", "family": "F20", "result": "pass", "label": "TypeScript client (`F20.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-24-typescript-client-f20-ts-client.json", "failed_command": null, "variants": [ { "feature_id": "F20.ts-client", "family": "F20", "result": "pass", "label": "TypeScript client (`F20.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-24-typescript-client-f20-ts-client.json", "failed_command": null } ] }, { "feature_id": "F20.validate-bad-key-type", "family": "F20", "result": "xfail", "label": "DeepSeek 401 type (`F20.validate-bad-key-type`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-15-deepseek-401-type-f20-validate-bad-key-t.json", "failed_command": "jq -e '.response.body.error.type == \"LLMAuthenticationError\"' \"$F20/bad-key.json\"", "variants": [ { "feature_id": "F20.validate-bad-key-type", "family": "F20", "result": "xfail", "label": "DeepSeek 401 type (`F20.validate-bad-key-type`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-15-deepseek-401-type-f20-validate-bad-key-t.json", "failed_command": "jq -e '.response.body.error.type == \"LLMAuthenticationError\"' \"$F20/bad-key.json\"" } ] }, { "feature_id": "F20.validate-hang-bounded", "family": "F20", "result": "xfail", "label": "Hanging provider is slower (`F20.validate-hang-bounded`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-19-hanging-provider-is-slower-f20-validate-.json", "failed_command": "test \"$RC\" = 0", "variants": [ { "feature_id": "F20.validate-hang-bounded", "family": "F20", "result": "xfail", "label": "Hanging provider is slower (`F20.validate-hang-bounded`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-19-hanging-provider-is-slower-f20-validate-.json", "failed_command": "test \"$RC\" = 0" } ] }, { "feature_id": "F20.validate-live", "family": "F20", "result": "pass", "label": "Validate a working config (`F20.validate-live`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-13-validate-a-working-config-f20-validate-l.json", "failed_command": null, "variants": [ { "feature_id": "F20.validate-live", "family": "F20", "result": "pass", "label": "Validate a working config (`F20.validate-live`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-13-validate-a-working-config-f20-validate-l.json", "failed_command": null } ] }, { "feature_id": "F20.validate-provider-errors", "family": "F20", "result": "pass", "label": "Provider failures (`F20.validate-provider-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-16-provider-failures-f20-validate-provider-.json", "failed_command": null, "variants": [ { "feature_id": "F20.validate-provider-errors", "family": "F20", "result": "pass", "label": "Provider failures (`F20.validate-provider-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-16-provider-failures-f20-validate-provider-.json", "failed_command": null } ] }, { "feature_id": "F20.validate-rejects", "family": "F20", "result": "pass", "label": "Validate rejects (`F20.validate-rejects`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-14-validate-rejects-f20-validate-rejects.json", "failed_command": null, "variants": [ { "feature_id": "F20.validate-rejects", "family": "F20", "result": "pass", "label": "Validate rejects (`F20.validate-rejects`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-14-validate-rejects-f20-validate-rejects.json", "failed_command": null } ] }, { "feature_id": "F20.validate-subscription-login", "family": "F20", "result": "pass", "label": "Subscription without a login (`F20.validate-subscription-login`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-17-subscription-without-a-login-f20-validat.json", "failed_command": null, "variants": [ { "feature_id": "F20.validate-subscription-login", "family": "F20", "result": "pass", "label": "Subscription without a login (`F20.validate-subscription-login`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-17-subscription-without-a-login-f20-validat.json", "failed_command": null } ] }, { "feature_id": "F20.validate-unreachable-bounded", "family": "F20", "result": "xfail", "label": "Unreachable endpoint is slow (`F20.validate-unreachable-bounded`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-18-unreachable-endpoint-is-slow-f20-validat.json", "failed_command": "jq -e '.elapsed_ms <= 30000' \"$F20/unreachable.json\"", "variants": [ { "feature_id": "F20.validate-unreachable-bounded", "family": "F20", "result": "xfail", "label": "Unreachable endpoint is slow (`F20.validate-unreachable-bounded`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-18-unreachable-endpoint-is-slow-f20-validat.json", "failed_command": "jq -e '.elapsed_ms <= 30000' \"$F20/unreachable.json\"" } ] }, { "feature_id": "F21.acp-kind", "family": "F21", "result": "pass", "label": "ACP profile (`F21.acp-kind`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-07-acp-profile-f21-acp-kind.json", "failed_command": null, "variants": [ { "feature_id": "F21.acp-kind", "family": "F21", "result": "pass", "label": "ACP profile (`F21.acp-kind`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-07-acp-profile-f21-acp-kind.json", "failed_command": null } ] }, { "feature_id": "F21.acp-skill-sourcing", "family": "F21", "result": "pass", "label": "ACP skill sourcing (`F21.acp-skill-sourcing`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-26-acp-skill-sourcing-f21-acp-skill-sourcin.json", "failed_command": null, "variants": [ { "feature_id": "F21.acp-skill-sourcing", "family": "F21", "result": "pass", "label": "ACP skill sourcing (`F21.acp-skill-sourcing`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-26-acp-skill-sourcing-f21-acp-skill-sourcin.json", "failed_command": null } ] }, { "feature_id": "F21.activate", "family": "F21", "result": "pass", "label": "Activate by id (`F21.activate`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-11-activate-by-id-f21-activate.json", "failed_command": null, "variants": [ { "feature_id": "F21.activate", "family": "F21", "result": "pass", "label": "Activate by id (`F21.activate`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-11-activate-by-id-f21-activate.json", "failed_command": null } ] }, { "feature_id": "F21.activate-corrupt-settings", "family": "F21", "result": "pass", "label": "Activate over a corrupt settings file (`F21.activate-corrupt-settings`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-12-activate-over-a-corrupt-settings-file-f2.json", "failed_command": null, "variants": [ { "feature_id": "F21.activate-corrupt-settings", "family": "F21", "result": "pass", "label": "Activate over a corrupt settings file (`F21.activate-corrupt-settings`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-12-activate-over-a-corrupt-settings-file-f2.json", "failed_command": null } ] }, { "feature_id": "F21.auth-required", "family": "F21", "result": "pass", "label": "No key, no access (`F21.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-01-no-key-no-access-f21-auth-required.json", "failed_command": null, "variants": [ { "feature_id": "F21.auth-required", "family": "F21", "result": "pass", "label": "No key, no access (`F21.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-01-no-key-no-access-f21-auth-required.json", "failed_command": null } ] }, { "feature_id": "F21.busy-blocks-server", "family": "F21", "result": "xfail", "label": "Lock wait stalls the server (`F21.busy-blocks-server`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-29-lock-wait-stalls-the-server-f21-busy-blo.json", "failed_command": "control-agent-server api GET /alive --auth none --expect 200 --timeout 60 --expect-max-ms 2000 --save F21.busy-blocks-server/alive", "variants": [ { "feature_id": "F21.busy-blocks-server", "family": "F21", "result": "xfail", "label": "Lock wait stalls the server (`F21.busy-blocks-server`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-29-lock-wait-stalls-the-server-f21-busy-blo.json", "failed_command": "control-agent-server api GET /alive --auth none --expect 200 --timeout 60 --expect-max-ms 2000 --save F21.busy-blocks-server/alive" } ] }, { "feature_id": "F21.conversation-start", "family": "F21", "result": "pass", "label": "Start a conversation from a profile (`F21.conversation-start`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-21-start-a-conversation-from-a-profile-f21-.json", "failed_command": null, "variants": [ { "feature_id": "F21.conversation-start", "family": "F21", "result": "pass", "label": "Start a conversation from a profile (`F21.conversation-start`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-21-start-a-conversation-from-a-profile-f21-.json", "failed_command": null } ] }, { "feature_id": "F21.corrupt-file", "family": "F21", "result": "pass", "label": "Corrupt file (`F21.corrupt-file`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-20-corrupt-file-f21-corrupt-file.json", "failed_command": null, "variants": [ { "feature_id": "F21.corrupt-file", "family": "F21", "result": "pass", "label": "Corrupt file (`F21.corrupt-file`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-20-corrupt-file-f21-corrupt-file.json", "failed_command": null } ] }, { "feature_id": "F21.dangling-llm-report", "family": "F21", "result": "xfail", "label": "How a dangling LLM reference is reported (`F21.dangling-llm-report`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-30-how-a-dangling-llm-reference-is-reported.json", "failed_command": "jq -e '.dangling_llm_profile_ref == \"qa-f21-missing\"' \"$RAW\" > /dev/null", "variants": [ { "feature_id": "F21.dangling-llm-report", "family": "F21", "result": "xfail", "label": "How a dangling LLM reference is reported (`F21.dangling-llm-report`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-30-how-a-dangling-llm-reference-is-reported.json", "failed_command": "jq -e '.dangling_llm_profile_ref == \"qa-f21-missing\"' \"$RAW\" > /dev/null" } ] }, { "feature_id": "F21.delete", "family": "F21", "result": "pass", "label": "Delete (`F21.delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-14-delete-f21-delete.json", "failed_command": null, "variants": [ { "feature_id": "F21.delete", "family": "F21", "result": "pass", "label": "Delete (`F21.delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-14-delete-f21-delete.json", "failed_command": null } ] }, { "feature_id": "F21.delete-active-acp-reset", "family": "F21", "result": "pass", "label": "Active ACP reset (`F21.delete-active-acp-reset`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-15-active-acp-reset-f21-delete-active-acp-r.json", "failed_command": null, "variants": [ { "feature_id": "F21.delete-active-acp-reset", "family": "F21", "result": "pass", "label": "Active ACP reset (`F21.delete-active-acp-reset`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-15-active-acp-reset-f21-delete-active-acp-r.json", "failed_command": null } ] }, { "feature_id": "F21.json-suffix-identity", "family": "F21", "result": "xfail", "label": "`.json` names lose their identity (`F21.json-suffix-identity`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-32-json-names-lose-their-identity-f21-json-.json", "failed_command": "control-agent-server api GET /api/agent-profiles/qa-f21-suffix.json --check profile.id eq \"$J1\" --check profile.revision eq 1", "variants": [ { "feature_id": "F21.json-suffix-identity", "family": "F21", "result": "xfail", "label": "`.json` names lose their identity (`F21.json-suffix-identity`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-32-json-names-lose-their-identity-f21-json-.json", "failed_command": "control-agent-server api GET /api/agent-profiles/qa-f21-suffix.json --check profile.id eq \"$J1\" --check profile.revision eq 1" } ] }, { "feature_id": "F21.launch-additions", "family": "F21", "result": "pass", "label": "Launch additions (`F21.launch-additions`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-23-launch-additions-f21-launch-additions.json", "failed_command": null, "variants": [ { "feature_id": "F21.launch-additions", "family": "F21", "result": "pass", "label": "Launch additions (`F21.launch-additions`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-23-launch-additions-f21-launch-additions.json", "failed_command": null } ] }, { "feature_id": "F21.launch-store-busy", "family": "F21", "result": "pass", "label": "Busy store at conversation start (`F21.launch-store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-28-busy-store-at-conversation-start-f21-lau.json", "failed_command": null, "variants": [ { "feature_id": "F21.launch-store-busy", "family": "F21", "result": "pass", "label": "Busy store at conversation start (`F21.launch-store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-28-busy-store-at-conversation-start-f21-lau.json", "failed_command": null } ] }, { "feature_id": "F21.lazy-seed", "family": "F21", "result": "pass", "label": "Lazy seed (`F21.lazy-seed`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-02-lazy-seed-f21-lazy-seed.json", "failed_command": null, "variants": [ { "feature_id": "F21.lazy-seed", "family": "F21", "result": "pass", "label": "Lazy seed (`F21.lazy-seed`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-02-lazy-seed-f21-lazy-seed.json", "failed_command": null } ] }, { "feature_id": "F21.limit", "family": "F21", "result": "pass", "label": "At most 50 (`F21.limit`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-19-at-most-50-f21-limit.json", "failed_command": null, "variants": [ { "feature_id": "F21.limit", "family": "F21", "result": "pass", "label": "At most 50 (`F21.limit`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-19-at-most-50-f21-limit.json", "failed_command": null } ] }, { "feature_id": "F21.materialize-dangling", "family": "F21", "result": "pass", "label": "Dangling references (`F21.materialize-dangling`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-17-dangling-references-f21-materialize-dang.json", "failed_command": null, "variants": [ { "feature_id": "F21.materialize-dangling", "family": "F21", "result": "pass", "label": "Dangling references (`F21.materialize-dangling`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-17-dangling-references-f21-materialize-dang.json", "failed_command": null } ] }, { "feature_id": "F21.materialize-draft", "family": "F21", "result": "pass", "label": "Draft preview (`F21.materialize-draft`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-18-draft-preview-f21-materialize-draft.json", "failed_command": null, "variants": [ { "feature_id": "F21.materialize-draft", "family": "F21", "result": "pass", "label": "Draft preview (`F21.materialize-draft`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-18-draft-preview-f21-materialize-draft.json", "failed_command": null } ] }, { "feature_id": "F21.materialize-valid", "family": "F21", "result": "pass", "label": "Materialize a resolvable profile (`F21.materialize-valid`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-16-materialize-a-resolvable-profile-f21-mat.json", "failed_command": null, "variants": [ { "feature_id": "F21.materialize-valid", "family": "F21", "result": "pass", "label": "Materialize a resolvable profile (`F21.materialize-valid`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-16-materialize-a-resolvable-profile-f21-mat.json", "failed_command": null } ] }, { "feature_id": "F21.name-rules", "family": "F21", "result": "pass", "label": "Name rules (`F21.name-rules`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-09-name-rules-f21-name-rules.json", "failed_command": null, "variants": [ { "feature_id": "F21.name-rules", "family": "F21", "result": "pass", "label": "Name rules (`F21.name-rules`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-09-name-rules-f21-name-rules.json", "failed_command": null } ] }, { "feature_id": "F21.not-found", "family": "F21", "result": "pass", "label": "Unknown names and ids (`F21.not-found`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-10-unknown-names-and-ids-f21-not-found.json", "failed_command": null, "variants": [ { "feature_id": "F21.not-found", "family": "F21", "result": "pass", "label": "Unknown names and ids (`F21.not-found`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-10-unknown-names-and-ids-f21-not-found.json", "failed_command": null } ] }, { "feature_id": "F21.rename", "family": "F21", "result": "pass", "label": "Rename (`F21.rename`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-13-rename-f21-rename.json", "failed_command": null, "variants": [ { "feature_id": "F21.rename", "family": "F21", "result": "pass", "label": "Rename (`F21.rename`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-13-rename-f21-rename.json", "failed_command": null } ] }, { "feature_id": "F21.restart", "family": "F21", "result": "pass", "label": "Restart (`F21.restart`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-25-restart-f21-restart.json", "failed_command": null, "variants": [ { "feature_id": "F21.restart", "family": "F21", "result": "pass", "label": "Restart (`F21.restart`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-25-restart-f21-restart.json", "failed_command": null } ] }, { "feature_id": "F21.save-create", "family": "F21", "result": "pass", "label": "Create (`F21.save-create`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-05-create-f21-save-create.json", "failed_command": null, "variants": [ { "feature_id": "F21.save-create", "family": "F21", "result": "pass", "label": "Create (`F21.save-create`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-05-create-f21-save-create.json", "failed_command": null } ] }, { "feature_id": "F21.save-overwrite", "family": "F21", "result": "pass", "label": "Overwrite (`F21.save-overwrite`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-06-overwrite-f21-save-overwrite.json", "failed_command": null, "variants": [ { "feature_id": "F21.save-overwrite", "family": "F21", "result": "pass", "label": "Overwrite (`F21.save-overwrite`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-06-overwrite-f21-save-overwrite.json", "failed_command": null } ] }, { "feature_id": "F21.save-validation", "family": "F21", "result": "pass", "label": "Save validation (`F21.save-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-08-save-validation-f21-save-validation.json", "failed_command": null, "variants": [ { "feature_id": "F21.save-validation", "family": "F21", "result": "pass", "label": "Save validation (`F21.save-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-08-save-validation-f21-save-validation.json", "failed_command": null } ] }, { "feature_id": "F21.secret-scope", "family": "F21", "result": "pass", "label": "Secret scope (`F21.secret-scope`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-22-secret-scope-f21-secret-scope.json", "failed_command": null, "variants": [ { "feature_id": "F21.secret-scope", "family": "F21", "result": "pass", "label": "Secret scope (`F21.secret-scope`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-22-secret-scope-f21-secret-scope.json", "failed_command": null } ] }, { "feature_id": "F21.seed-acp", "family": "F21", "result": "pass", "label": "ACP seed (`F21.seed-acp`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-04-acp-seed-f21-seed-acp.json", "failed_command": null, "variants": [ { "feature_id": "F21.seed-acp", "family": "F21", "result": "pass", "label": "ACP seed (`F21.seed-acp`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-04-acp-seed-f21-seed-acp.json", "failed_command": null } ] }, { "feature_id": "F21.seed-no-ghost-llm", "family": "F21", "result": "pass", "label": "No ghost LLM profile (`F21.seed-no-ghost-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-03-no-ghost-llm-profile-f21-seed-no-ghost-l.json", "failed_command": null, "variants": [ { "feature_id": "F21.seed-no-ghost-llm", "family": "F21", "result": "pass", "label": "No ghost LLM profile (`F21.seed-no-ghost-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-03-no-ghost-llm-profile-f21-seed-no-ghost-l.json", "failed_command": null } ] }, { "feature_id": "F21.store-busy", "family": "F21", "result": "pass", "label": "Store busy (`F21.store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-27-store-busy-f21-store-busy.json", "failed_command": null, "variants": [ { "feature_id": "F21.store-busy", "family": "F21", "result": "pass", "label": "Store busy (`F21.store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-27-store-busy-f21-store-busy.json", "failed_command": null } ] }, { "feature_id": "F21.ts-client", "family": "F21", "result": "pass", "label": "TypeScript client (`F21.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-24-typescript-client-f21-ts-client.json", "failed_command": null, "variants": [ { "feature_id": "F21.ts-client", "family": "F21", "result": "pass", "label": "TypeScript client (`F21.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-24-typescript-client-f21-ts-client.json", "failed_command": null } ] }, { "feature_id": "F21.ts-payload-types", "family": "F21", "result": "xfail", "label": "TypeScript payload types (`F21.ts-payload-types`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-31-typescript-payload-types-f21-ts-payload-.json", "failed_command": "$TSC \"$D/exported.ts\"", "variants": [ { "feature_id": "F21.ts-payload-types", "family": "F21", "result": "xfail", "label": "TypeScript payload types (`F21.ts-payload-types`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-31-typescript-payload-types-f21-ts-payload-.json", "failed_command": "$TSC \"$D/exported.ts\"" } ] }, { "feature_id": "F22.acp-clears-pointer", "family": "F22", "result": "pass", "label": "ACP clears the pointer (`F22.acp-clears-pointer`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-16-acp-clears-the-pointer-f22-acp-clears-po.json", "failed_command": null, "variants": [ { "feature_id": "F22.acp-clears-pointer", "family": "F22", "result": "pass", "label": "ACP clears the pointer (`F22.acp-clears-pointer`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-16-acp-clears-the-pointer-f22-acp-clears-po.json", "failed_command": null } ] }, { "feature_id": "F22.activate", "family": "F22", "result": "pass", "label": "Activate (`F22.activate`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-09-activate-f22-activate.json", "failed_command": null, "variants": [ { "feature_id": "F22.activate", "family": "F22", "result": "pass", "label": "Activate (`F22.activate`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-09-activate-f22-activate.json", "failed_command": null } ] }, { "feature_id": "F22.activate-acp", "family": "F22", "result": "xfail", "label": "Activation on ACP settings (`F22.activate-acp`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-17-activation-on-acp-settings-f22-activate-.json", "failed_command": "jq -e '.status >= 400 and .status < 500' \"$F22/acp-activate.json\"", "variants": [ { "feature_id": "F22.activate-acp", "family": "F22", "result": "xfail", "label": "Activation on ACP settings (`F22.activate-acp`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-17-activation-on-acp-settings-f22-activate-.json", "failed_command": "jq -e '.status >= 400 and .status < 500' \"$F22/acp-activate.json\"" } ] }, { "feature_id": "F22.auth-required", "family": "F22", "result": "pass", "label": "Key required (`F22.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-01-key-required-f22-auth-required.json", "failed_command": null, "variants": [ { "feature_id": "F22.auth-required", "family": "F22", "result": "pass", "label": "Key required (`F22.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-01-key-required-f22-auth-required.json", "failed_command": null } ] }, { "feature_id": "F22.corrupt-files", "family": "F22", "result": "pass", "label": "Corrupted files (`F22.corrupt-files`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-18-corrupted-files-f22-corrupt-files.json", "failed_command": null, "variants": [ { "feature_id": "F22.corrupt-files", "family": "F22", "result": "pass", "label": "Corrupted files (`F22.corrupt-files`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-18-corrupted-files-f22-corrupt-files.json", "failed_command": null } ] }, { "feature_id": "F22.corrupt-settings", "family": "F22", "result": "xfail", "label": "Unreadable settings (`F22.corrupt-settings`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-19-unreadable-settings-f22-corrupt-settings.json", "failed_command": "jq -e '.status == 409' \"$F22/corrupt-settings-activate.json\"", "variants": [ { "feature_id": "F22.corrupt-settings", "family": "F22", "result": "xfail", "label": "Unreadable settings (`F22.corrupt-settings`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-19-unreadable-settings-f22-corrupt-settings.json", "failed_command": "jq -e '.status == 409' \"$F22/corrupt-settings-activate.json\"" } ] }, { "feature_id": "F22.delete", "family": "F22", "result": "pass", "label": "Delete (`F22.delete`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-14-delete-f22-delete.json", "failed_command": null, "variants": [ { "feature_id": "F22.delete", "family": "F22", "result": "pass", "label": "Delete (`F22.delete`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-14-delete-f22-delete.json", "failed_command": null } ] }, { "feature_id": "F22.delete-active-clears", "family": "F22", "result": "pass", "label": "Delete the active one (`F22.delete-active-clears`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-15-delete-the-active-one-f22-delete-active-.json", "failed_command": null, "variants": [ { "feature_id": "F22.delete-active-clears", "family": "F22", "result": "pass", "label": "Delete the active one (`F22.delete-active-clears`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-15-delete-the-active-one-f22-delete-active-.json", "failed_command": null } ] }, { "feature_id": "F22.empty-router", "family": "F22", "result": "xfail", "label": "Router with no route (`F22.empty-router`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-05-router-with-no-route-f22-empty-router-kn.json", "failed_command": "control-agent-server api POST /api/meta-profiles/qa-empty --json '{\"classifier_model\": \"deepseek-flash\"}' --expect 422 --save F22.empty-router/save", "variants": [ { "feature_id": "F22.empty-router", "family": "F22", "result": "xfail", "label": "Router with no route (`F22.empty-router`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-05-router-with-no-route-f22-empty-router-kn.json", "failed_command": "control-agent-server api POST /api/meta-profiles/qa-empty --json '{\"classifier_model\": \"deepseek-flash\"}' --expect 422 --save F22.empty-router/save" } ] }, { "feature_id": "F22.limit", "family": "F22", "result": "pass", "label": "Limit (`F22.limit`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-22-limit-f22-limit.json", "failed_command": null, "variants": [ { "feature_id": "F22.limit", "family": "F22", "result": "pass", "label": "Limit (`F22.limit`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-22-limit-f22-limit.json", "failed_command": null } ] }, { "feature_id": "F22.name-rules", "family": "F22", "result": "pass", "label": "Name rules (`F22.name-rules`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-07-name-rules-f22-name-rules.json", "failed_command": null, "variants": [ { "feature_id": "F22.name-rules", "family": "F22", "result": "pass", "label": "Name rules (`F22.name-rules`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-07-name-rules-f22-name-rules.json", "failed_command": null } ] }, { "feature_id": "F22.not-found", "family": "F22", "result": "pass", "label": "Unknown names (`F22.not-found`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-08-unknown-names-f22-not-found.json", "failed_command": null, "variants": [ { "feature_id": "F22.not-found", "family": "F22", "result": "pass", "label": "Unknown names (`F22.not-found`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-08-unknown-names-f22-not-found.json", "failed_command": null } ] }, { "feature_id": "F22.overwrite", "family": "F22", "result": "pass", "label": "Overwrite (`F22.overwrite`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-06-overwrite-f22-overwrite.json", "failed_command": null, "variants": [ { "feature_id": "F22.overwrite", "family": "F22", "result": "pass", "label": "Overwrite (`F22.overwrite`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-06-overwrite-f22-overwrite.json", "failed_command": null } ] }, { "feature_id": "F22.persist-restart", "family": "F22", "result": "pass", "label": "Restart (`F22.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-13-restart-f22-persist-restart.json", "failed_command": null, "variants": [ { "feature_id": "F22.persist-restart", "family": "F22", "result": "pass", "label": "Restart (`F22.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-13-restart-f22-persist-restart.json", "failed_command": null } ] }, { "feature_id": "F22.persistence-dir", "family": "F22", "result": "xfail", "label": "Persistence directory with `~` (`F22.persistence-dir`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-24-persistence-directory-with-f22-persisten.json", "failed_command": "control-agent-server state ls 'home/qa-f22-tilde/meta-profiles/qa-tilde.json' --run \"$T\" --expect-count 1", "variants": [ { "feature_id": "F22.persistence-dir", "family": "F22", "result": "xfail", "label": "Persistence directory with `~` (`F22.persistence-dir`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-24-persistence-directory-with-f22-persisten.json", "failed_command": "control-agent-server state ls 'home/qa-f22-tilde/meta-profiles/qa-tilde.json' --run \"$T\" --expect-count 1" } ] }, { "feature_id": "F22.route-classes", "family": "F22", "result": "pass", "label": "Classes-mode routing (`F22.route-classes`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-10-classes-mode-routing-f22-route-classes.json", "failed_command": null, "variants": [ { "feature_id": "F22.route-classes", "family": "F22", "result": "pass", "label": "Classes-mode routing (`F22.route-classes`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-10-classes-mode-routing-f22-route-classes.json", "failed_command": null } ] }, { "feature_id": "F22.route-direct", "family": "F22", "result": "pass", "label": "Direct routing (`F22.route-direct`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-11-direct-routing-f22-route-direct.json", "failed_command": null, "variants": [ { "feature_id": "F22.route-direct", "family": "F22", "result": "pass", "label": "Direct routing (`F22.route-direct`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-11-direct-routing-f22-route-direct.json", "failed_command": null } ] }, { "feature_id": "F22.route-missing-profile", "family": "F22", "result": "pass", "label": "Missing LLM profile (`F22.route-missing-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-12-missing-llm-profile-f22-route-missing-pr.json", "failed_command": null, "variants": [ { "feature_id": "F22.route-missing-profile", "family": "F22", "result": "pass", "label": "Missing LLM profile (`F22.route-missing-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-12-missing-llm-profile-f22-route-missing-pr.json", "failed_command": null } ] }, { "feature_id": "F22.save-classes", "family": "F22", "result": "pass", "label": "Save in classes mode (`F22.save-classes`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-02-save-in-classes-mode-f22-save-classes.json", "failed_command": null, "variants": [ { "feature_id": "F22.save-classes", "family": "F22", "result": "pass", "label": "Save in classes mode (`F22.save-classes`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-02-save-in-classes-mode-f22-save-classes.json", "failed_command": null } ] }, { "feature_id": "F22.save-direct", "family": "F22", "result": "pass", "label": "Save in direct mode (`F22.save-direct`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-03-save-in-direct-mode-f22-save-direct.json", "failed_command": null, "variants": [ { "feature_id": "F22.save-direct", "family": "F22", "result": "pass", "label": "Save in direct mode (`F22.save-direct`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-03-save-in-direct-mode-f22-save-direct.json", "failed_command": null } ] }, { "feature_id": "F22.store-busy", "family": "F22", "result": "fail", "label": "Store busy (`F22.store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-20-store-busy-f22-store-busy.json", "failed_command": "test $((SECONDS - T0)) -le 33", "variants": [ { "feature_id": "F22.store-busy", "family": "F22", "result": "fail", "label": "Store busy (`F22.store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-20-store-busy-f22-store-busy.json", "failed_command": "test $((SECONDS - T0)) -le 33" } ] }, { "feature_id": "F22.store-busy-responsive", "family": "F22", "result": "xfail", "label": "Lock wait blocks the server (`F22.store-busy-responsive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-21-lock-wait-blocks-the-server-f22-store-bu.json", "failed_command": "test \"$SLOW\" = 0", "variants": [ { "feature_id": "F22.store-busy-responsive", "family": "F22", "result": "xfail", "label": "Lock wait blocks the server (`F22.store-busy-responsive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-21-lock-wait-blocks-the-server-f22-store-bu.json", "failed_command": "test \"$SLOW\" = 0" } ] }, { "feature_id": "F22.ts-client", "family": "F22", "result": "pass", "label": "TypeScript client (`F22.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-23-typescript-client-f22-ts-client.json", "failed_command": null, "variants": [ { "feature_id": "F22.ts-client", "family": "F22", "result": "pass", "label": "TypeScript client (`F22.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-23-typescript-client-f22-ts-client.json", "failed_command": null } ] }, { "feature_id": "F22.validation", "family": "F22", "result": "pass", "label": "Validation (`F22.validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-04-validation-f22-validation.json", "failed_command": null, "variants": [ { "feature_id": "F22.validation", "family": "F22", "result": "pass", "label": "Validation (`F22.validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-04-validation-f22-validation.json", "failed_command": null } ] }, { "feature_id": "F23.acp-discovery-errors", "family": "F23", "result": "pass", "label": "ACP discovery errors (`F23.acp-discovery-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-27-acp-discovery-errors-f23-acp-discovery-e.json", "failed_command": null, "variants": [ { "feature_id": "F23.acp-discovery-errors", "family": "F23", "result": "pass", "label": "ACP discovery errors (`F23.acp-discovery-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-27-acp-discovery-errors-f23-acp-discovery-e.json", "failed_command": null } ] }, { "feature_id": "F23.auth-required", "family": "F23", "result": "pass", "label": "Key required (`F23.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-01-key-required-f23-auth-required.json", "failed_command": null, "variants": [ { "feature_id": "F23.auth-required", "family": "F23", "result": "pass", "label": "Key required (`F23.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-01-key-required-f23-auth-required.json", "failed_command": null } ] }, { "feature_id": "F23.catalog-ts-client", "family": "F23", "result": "pass", "label": "TypeScript client (`F23.catalog-ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-07-typescript-client-f23-catalog-ts-client.json", "failed_command": null, "variants": [ { "feature_id": "F23.catalog-ts-client", "family": "F23", "result": "pass", "label": "TypeScript client (`F23.catalog-ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-07-typescript-client-f23-catalog-ts-client.json", "failed_command": null } ] }, { "feature_id": "F23.conn-create", "family": "F23", "result": "pass", "label": "Create (`F23.conn-create`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-08-create-f23-conn-create.json", "failed_command": null, "variants": [ { "feature_id": "F23.conn-create", "family": "F23", "result": "pass", "label": "Create (`F23.conn-create`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-08-create-f23-conn-create.json", "failed_command": null } ] }, { "feature_id": "F23.conn-create-blank-key", "family": "F23", "result": "xfail", "label": "Blank keys on create (`F23.conn-create-blank-key`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-12-blank-keys-on-create-f23-conn-create-bla.json", "failed_command": "test \"$S_SPACE $S_MASK\" = '422 422'", "variants": [ { "feature_id": "F23.conn-create-blank-key", "family": "F23", "result": "xfail", "label": "Blank keys on create (`F23.conn-create-blank-key`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-12-blank-keys-on-create-f23-conn-create-bla.json", "failed_command": "test \"$S_SPACE $S_MASK\" = '422 422'" } ] }, { "feature_id": "F23.conn-create-validation", "family": "F23", "result": "pass", "label": "Create validation (`F23.conn-create-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-11-create-validation-f23-conn-create-valida.json", "failed_command": null, "variants": [ { "feature_id": "F23.conn-create-validation", "family": "F23", "result": "pass", "label": "Create validation (`F23.conn-create-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-11-create-validation-f23-conn-create-valida.json", "failed_command": null } ] }, { "feature_id": "F23.conn-delete", "family": "F23", "result": "pass", "label": "Delete (`F23.conn-delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-25-delete-f23-conn-delete.json", "failed_command": null, "variants": [ { "feature_id": "F23.conn-delete", "family": "F23", "result": "pass", "label": "Delete (`F23.conn-delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-25-delete-f23-conn-delete.json", "failed_command": null } ] }, { "feature_id": "F23.conn-key-encrypted", "family": "F23", "result": "pass", "label": "Key hidden and encrypted (`F23.conn-key-encrypted`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-09-key-hidden-and-encrypted-f23-conn-key-en.json", "failed_command": null, "variants": [ { "feature_id": "F23.conn-key-encrypted", "family": "F23", "result": "pass", "label": "Key hidden and encrypted (`F23.conn-key-encrypted`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-09-key-hidden-and-encrypted-f23-conn-key-en.json", "failed_command": null } ] }, { "feature_id": "F23.conn-limit", "family": "F23", "result": "pass", "label": "Limit (`F23.conn-limit`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-26-limit-f23-conn-limit.json", "failed_command": null, "variants": [ { "feature_id": "F23.conn-limit", "family": "F23", "result": "pass", "label": "Limit (`F23.conn-limit`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-26-limit-f23-conn-limit.json", "failed_command": null } ] }, { "feature_id": "F23.conn-patch-blank-key", "family": "F23", "result": "xfail", "label": "Blank keys on update (`F23.conn-patch-blank-key`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-16-blank-keys-on-update-f23-conn-patch-blan.json", "failed_command": "test \"$APPLIED\" = '[sk-qa-f23-default][sk-qa-f23-default][sk-qa-f23-default]'", "variants": [ { "feature_id": "F23.conn-patch-blank-key", "family": "F23", "result": "xfail", "label": "Blank keys on update (`F23.conn-patch-blank-key`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-16-blank-keys-on-update-f23-conn-patch-blan.json", "failed_command": "test \"$APPLIED\" = '[sk-qa-f23-default][sk-qa-f23-default][sk-qa-f23-default]'" } ] }, { "feature_id": "F23.conn-rotate-key", "family": "F23", "result": "pass", "label": "Rotate the key (`F23.conn-rotate-key`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-17-rotate-the-key-f23-conn-rotate-key.json", "failed_command": null, "variants": [ { "feature_id": "F23.conn-rotate-key", "family": "F23", "result": "pass", "label": "Rotate the key (`F23.conn-rotate-key`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-17-rotate-the-key-f23-conn-rotate-key.json", "failed_command": null } ] }, { "feature_id": "F23.conn-update", "family": "F23", "result": "pass", "label": "Update (`F23.conn-update`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-14-update-f23-conn-update.json", "failed_command": null, "variants": [ { "feature_id": "F23.conn-update", "family": "F23", "result": "pass", "label": "Update (`F23.conn-update`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-14-update-f23-conn-update.json", "failed_command": null } ] }, { "feature_id": "F23.conn-update-validation", "family": "F23", "result": "pass", "label": "Update validation (`F23.conn-update-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-15-update-validation-f23-conn-update-valida.json", "failed_command": null, "variants": [ { "feature_id": "F23.conn-update-validation", "family": "F23", "result": "pass", "label": "Update validation (`F23.conn-update-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-15-update-validation-f23-conn-update-valida.json", "failed_command": null } ] }, { "feature_id": "F23.corrupt-file", "family": "F23", "result": "pass", "label": "Corrupted file (`F23.corrupt-file`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-21-corrupted-file-f23-corrupt-file.json", "failed_command": null, "variants": [ { "feature_id": "F23.corrupt-file", "family": "F23", "result": "pass", "label": "Corrupted file (`F23.corrupt-file`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-21-corrupted-file-f23-corrupt-file.json", "failed_command": null } ] }, { "feature_id": "F23.delete-guard", "family": "F23", "result": "pass", "label": "Delete guard (`F23.delete-guard`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-24-delete-guard-f23-delete-guard.json", "failed_command": null, "variants": [ { "feature_id": "F23.delete-guard", "family": "F23", "result": "pass", "label": "Delete guard (`F23.delete-guard`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-24-delete-guard-f23-delete-guard.json", "failed_command": null } ] }, { "feature_id": "F23.linked-run", "family": "F23", "result": "pass", "label": "Linked profile runs on the connection (`F23.linked-run`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-13-linked-profile-runs-on-the-connection-f2.json", "failed_command": null, "variants": [ { "feature_id": "F23.linked-run", "family": "F23", "result": "pass", "label": "Linked profile runs on the connection (`F23.linked-run`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-13-linked-profile-runs-on-the-connection-f2.json", "failed_command": null } ] }, { "feature_id": "F23.models", "family": "F23", "result": "pass", "label": "All models (`F23.models`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-03-all-models-f23-models.json", "failed_command": null, "variants": [ { "feature_id": "F23.models", "family": "F23", "result": "pass", "label": "All models (`F23.models`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-03-all-models-f23-models.json", "failed_command": null } ] }, { "feature_id": "F23.models-filter", "family": "F23", "result": "pass", "label": "Provider filter (`F23.models-filter`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-04-provider-filter-f23-models-filter.json", "failed_command": null, "variants": [ { "feature_id": "F23.models-filter", "family": "F23", "result": "pass", "label": "Provider filter (`F23.models-filter`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-04-provider-filter-f23-models-filter.json", "failed_command": null } ] }, { "feature_id": "F23.models-no-bedrock", "family": "F23", "result": "xfail", "label": "No Bedrock ids under a provider (`F23.models-no-bedrock`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-05-no-bedrock-ids-under-a-provider-f23-mode.json", "failed_command": "jq -e 'map(select(test(\"^deepseek\\\\.\"))) | debug | length == 0'", "variants": [ { "feature_id": "F23.models-no-bedrock", "family": "F23", "result": "xfail", "label": "No Bedrock ids under a provider (`F23.models-no-bedrock`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-05-no-bedrock-ids-under-a-provider-f23-mode.json", "failed_command": "jq -e 'map(select(test(\"^deepseek\\\\.\"))) | debug | length == 0'" } ] }, { "feature_id": "F23.no-cipher", "family": "F23", "result": "pass", "label": "No cipher (`F23.no-cipher`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-10-no-cipher-f23-no-cipher.json", "failed_command": null, "variants": [ { "feature_id": "F23.no-cipher", "family": "F23", "result": "pass", "label": "No cipher (`F23.no-cipher`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-10-no-cipher-f23-no-cipher.json", "failed_command": null } ] }, { "feature_id": "F23.persist-restart", "family": "F23", "result": "pass", "label": "Restart (`F23.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-19-restart-f23-persist-restart.json", "failed_command": null, "variants": [ { "feature_id": "F23.persist-restart", "family": "F23", "result": "pass", "label": "Restart (`F23.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-19-restart-f23-persist-restart.json", "failed_command": null } ] }, { "feature_id": "F23.providers", "family": "F23", "result": "pass", "label": "Providers (`F23.providers`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-02-providers-f23-providers.json", "failed_command": null, "variants": [ { "feature_id": "F23.providers", "family": "F23", "result": "pass", "label": "Providers (`F23.providers`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-02-providers-f23-providers.json", "failed_command": null } ] }, { "feature_id": "F23.store-busy", "family": "F23", "result": "pass", "label": "Store busy (`F23.store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-22-store-busy-f23-store-busy.json", "failed_command": null, "variants": [ { "feature_id": "F23.store-busy", "family": "F23", "result": "pass", "label": "Store busy (`F23.store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-22-store-busy-f23-store-busy.json", "failed_command": null } ] }, { "feature_id": "F23.store-busy-responsive", "family": "F23", "result": "xfail", "label": "Lock wait blocks the server (`F23.store-busy-responsive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-23-lock-wait-blocks-the-server-f23-store-bu.json", "failed_command": "control-agent-server api GET /alive --auth none --expect 200 --expect-max-ms 3000 --timeout 60 --save F23.store-busy-responsive/alive", "variants": [ { "feature_id": "F23.store-busy-responsive", "family": "F23", "result": "xfail", "label": "Lock wait blocks the server (`F23.store-busy-responsive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-23-lock-wait-blocks-the-server-f23-store-bu.json", "failed_command": "control-agent-server api GET /alive --auth none --expect 200 --expect-max-ms 3000 --timeout 60 --save F23.store-busy-responsive/alive" } ] }, { "feature_id": "F23.strict-decrypt", "family": "F23", "result": "pass", "label": "Secret key changed (`F23.strict-decrypt`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-20-secret-key-changed-f23-strict-decrypt.json", "failed_command": null, "variants": [ { "feature_id": "F23.strict-decrypt", "family": "F23", "result": "pass", "label": "Secret key changed (`F23.strict-decrypt`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-20-secret-key-changed-f23-strict-decrypt.json", "failed_command": null } ] }, { "feature_id": "F23.validate-linked-draft", "family": "F23", "result": "xfail", "label": "Validate a linked draft (`F23.validate-linked-draft`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-18-validate-a-linked-draft-f23-validate-lin.json", "failed_command": "control-agent-server api POST /api/profiles/qa-f23-draft/validate --timeout 120 --expect 200 --json \"{\\\"llm\\\": {\\\"model\\\": \\\"deepseek/deepseek-flash\\\", \\\"provider_connection_id\\\": \\\"$CONN\\\", \\\"num_retries\\\": 0}}\" --save F23.validate-linked-draft/linked --check valid eq true --check error eq null", "variants": [ { "feature_id": "F23.validate-linked-draft", "family": "F23", "result": "xfail", "label": "Validate a linked draft (`F23.validate-linked-draft`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-18-validate-a-linked-draft-f23-validate-lin.json", "failed_command": "control-agent-server api POST /api/profiles/qa-f23-draft/validate --timeout 120 --expect 200 --json \"{\\\"llm\\\": {\\\"model\\\": \\\"deepseek/deepseek-flash\\\", \\\"provider_connection_id\\\": \\\"$CONN\\\", \\\"num_retries\\\": 0}}\" --save F23.validate-linked-draft/linked --check valid eq true --check error eq null" } ] }, { "feature_id": "F23.verified-models", "family": "F23", "result": "pass", "label": "Verified models (`F23.verified-models`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-06-verified-models-f23-verified-models.json", "failed_command": null, "variants": [ { "feature_id": "F23.verified-models", "family": "F23", "result": "pass", "label": "Verified models (`F23.verified-models`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-06-verified-models-f23-verified-models.json", "failed_command": null } ] }, { "feature_id": "F24.auth-required", "family": "F24", "result": "pass", "label": "Key required (`F24.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-04-key-required-f24-auth-required.json", "failed_command": null, "variants": [ { "feature_id": "F24.auth-required", "family": "F24", "result": "pass", "label": "Key required (`F24.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-04-key-required-f24-auth-required.json", "failed_command": null } ] }, { "feature_id": "F24.device-poll-invalid", "family": "F24", "result": "pass", "label": "Malformed polls (`F24.device-poll-invalid`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-09-malformed-polls-f24-device-poll-invalid.json", "failed_command": null, "variants": [ { "feature_id": "F24.device-poll-invalid", "family": "F24", "result": "pass", "label": "Malformed polls (`F24.device-poll-invalid`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-09-malformed-polls-f24-device-poll-invalid.json", "failed_command": null } ] }, { "feature_id": "F24.device-poll-outage", "family": "F24", "result": "pass", "label": "Outage during a pending login (`F24.device-poll-outage`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-17-outage-during-a-pending-login-f24-device.json", "failed_command": null, "variants": [ { "feature_id": "F24.device-poll-outage", "family": "F24", "result": "pass", "label": "Outage during a pending login (`F24.device-poll-outage`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-17-outage-during-a-pending-login-f24-device.json", "failed_command": null } ] }, { "feature_id": "F24.device-poll-pending", "family": "F24", "result": "pass", "label": "Pending login (`F24.device-poll-pending`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-12-pending-login-f24-device-poll-pending.json", "failed_command": null, "variants": [ { "feature_id": "F24.device-poll-pending", "family": "F24", "result": "pass", "label": "Pending login (`F24.device-poll-pending`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-12-pending-login-f24-device-poll-pending.json", "failed_command": null } ] }, { "feature_id": "F24.device-poll-unknown", "family": "F24", "result": "pass", "label": "Unknown token (`F24.device-poll-unknown`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-10-unknown-token-f24-device-poll-unknown.json", "failed_command": null, "variants": [ { "feature_id": "F24.device-poll-unknown", "family": "F24", "result": "pass", "label": "Unknown token (`F24.device-poll-unknown`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-10-unknown-token-f24-device-poll-unknown.json", "failed_command": null } ] }, { "feature_id": "F24.device-start", "family": "F24", "result": "pass", "label": "Start a login (`F24.device-start`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-11-start-a-login-f24-device-start.json", "failed_command": null, "variants": [ { "feature_id": "F24.device-start", "family": "F24", "result": "pass", "label": "Start a login (`F24.device-start`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-11-start-a-login-f24-device-start.json", "failed_command": null } ] }, { "feature_id": "F24.device-start-offline", "family": "F24", "result": "xfail", "label": "Start and poll a login offline (`F24.device-start-offline`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-19-start-and-poll-a-login-offline-f24-devic.json", "failed_command": "control-agent-server api POST /api/llm/subscription/openai/device/start --run \"$OUTAGE\" --expect 502,503,504 --save F24.device-start-offline/start", "variants": [ { "feature_id": "F24.device-start-offline", "family": "F24", "result": "xfail", "label": "Start and poll a login offline (`F24.device-start-offline`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-19-start-and-poll-a-login-offline-f24-devic.json", "failed_command": "control-agent-server api POST /api/llm/subscription/openai/device/start --run \"$OUTAGE\" --expect 502,503,504 --save F24.device-start-offline/start" } ] }, { "feature_id": "F24.login-complete", "family": "F24", "result": "blocked", "label": "Complete a login (`F24.login-complete`), blocked.", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": null, "failed_command": null, "variants": [ { "feature_id": "F24.login-complete", "family": "F24", "result": "blocked", "label": "Complete a login (`F24.login-complete`), blocked.", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": null, "failed_command": null } ] }, { "feature_id": "F24.logout", "family": "F24", "result": "pass", "label": "Logout (`F24.logout`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-06-logout-f24-logout.json", "failed_command": null, "variants": [ { "feature_id": "F24.logout", "family": "F24", "result": "pass", "label": "Logout (`F24.logout`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-06-logout-f24-logout.json", "failed_command": null } ] }, { "feature_id": "F24.logout-cancels-pending", "family": "F24", "result": "pass", "label": "Logout cancels pending logins (`F24.logout-cancels-pending`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-13-logout-cancels-pending-logins-f24-logout.json", "failed_command": null, "variants": [ { "feature_id": "F24.logout-cancels-pending", "family": "F24", "result": "pass", "label": "Logout cancels pending logins (`F24.logout-cancels-pending`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-13-logout-cancels-pending-logins-f24-logout.json", "failed_command": null } ] }, { "feature_id": "F24.models", "family": "F24", "result": "pass", "label": "Subscription models (`F24.models`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-01-subscription-models-f24-models.json", "failed_command": null, "variants": [ { "feature_id": "F24.models", "family": "F24", "result": "pass", "label": "Subscription models (`F24.models`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-01-subscription-models-f24-models.json", "failed_command": null } ] }, { "feature_id": "F24.offline-safe", "family": "F24", "result": "pass", "label": "Offline-safe routes (`F24.offline-safe`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-16-offline-safe-routes-f24-offline-safe.json", "failed_command": null, "variants": [ { "feature_id": "F24.offline-safe", "family": "F24", "result": "pass", "label": "Offline-safe routes (`F24.offline-safe`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-16-offline-safe-routes-f24-offline-safe.json", "failed_command": null } ] }, { "feature_id": "F24.pending-lost-on-restart", "family": "F24", "result": "pass", "label": "Pending logins and restarts (`F24.pending-lost-on-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-14-pending-logins-and-restarts-f24-pending-.json", "failed_command": null, "variants": [ { "feature_id": "F24.pending-lost-on-restart", "family": "F24", "result": "pass", "label": "Pending logins and restarts (`F24.pending-lost-on-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-14-pending-logins-and-restarts-f24-pending-.json", "failed_command": null } ] }, { "feature_id": "F24.status-connected", "family": "F24", "result": "pass", "label": "Status with a stored login (`F24.status-connected`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-03-status-with-a-stored-login-f24-status-co.json", "failed_command": null, "variants": [ { "feature_id": "F24.status-connected", "family": "F24", "result": "pass", "label": "Status with a stored login (`F24.status-connected`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-03-status-with-a-stored-login-f24-status-co.json", "failed_command": null } ] }, { "feature_id": "F24.status-disconnected", "family": "F24", "result": "pass", "label": "Status with no login (`F24.status-disconnected`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-02-status-with-no-login-f24-status-disconne.json", "failed_command": null, "variants": [ { "feature_id": "F24.status-disconnected", "family": "F24", "result": "pass", "label": "Status with no login (`F24.status-disconnected`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-02-status-with-no-login-f24-status-disconne.json", "failed_command": null } ] }, { "feature_id": "F24.status-invalid-file", "family": "F24", "result": "pass", "label": "Invalid credentials file (`F24.status-invalid-file`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-08-invalid-credentials-file-f24-status-inva.json", "failed_command": null, "variants": [ { "feature_id": "F24.status-invalid-file", "family": "F24", "result": "pass", "label": "Invalid credentials file (`F24.status-invalid-file`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-08-invalid-credentials-file-f24-status-inva.json", "failed_command": null } ] }, { "feature_id": "F24.status-offline", "family": "F24", "result": "xfail", "label": "Status with an expired login offline (`F24.status-offline`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-18-status-with-an-expired-login-offline-f24.json", "failed_command": "control-agent-server api GET /api/llm/subscription/openai/status --run \"$OFF\" --expect 200 --check connected eq false --save F24.status-offline/status", "variants": [ { "feature_id": "F24.status-offline", "family": "F24", "result": "xfail", "label": "Status with an expired login offline (`F24.status-offline`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-18-status-with-an-expired-login-offline-f24.json", "failed_command": "control-agent-server api GET /api/llm/subscription/openai/status --run \"$OFF\" --expect 200 --check connected eq false --save F24.status-offline/status" } ] }, { "feature_id": "F24.status-refresh-rejected", "family": "F24", "result": "pass", "label": "Rejected refresh (`F24.status-refresh-rejected`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-07-rejected-refresh-f24-status-refresh-reje.json", "failed_command": null, "variants": [ { "feature_id": "F24.status-refresh-rejected", "family": "F24", "result": "pass", "label": "Rejected refresh (`F24.status-refresh-rejected`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-07-rejected-refresh-f24-status-refresh-reje.json", "failed_command": null } ] }, { "feature_id": "F24.status-restart", "family": "F24", "result": "pass", "label": "Login across a restart (`F24.status-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-05-login-across-a-restart-f24-status-restar.json", "failed_command": null, "variants": [ { "feature_id": "F24.status-restart", "family": "F24", "result": "pass", "label": "Login across a restart (`F24.status-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-05-login-across-a-restart-f24-status-restar.json", "failed_command": null } ] }, { "feature_id": "F24.ts-client", "family": "F24", "result": "pass", "label": "TypeScript client (`F24.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-15-typescript-client-f24-ts-client.json", "failed_command": null, "variants": [ { "feature_id": "F24.ts-client", "family": "F24", "result": "pass", "label": "TypeScript client (`F24.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-15-typescript-client-f24-ts-client.json", "failed_command": null } ] }, { "feature_id": "F25.auth", "family": "F25", "result": "pass", "label": "Session key required (`F25.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-01-session-key-required-f25-auth.json", "failed_command": null, "variants": [ { "feature_id": "F25.auth", "family": "F25", "result": "pass", "label": "Session key required (`F25.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-01-session-key-required-f25-auth.json", "failed_command": null } ] }, { "feature_id": "F25.catalog-project", "family": "F25", "result": "pass", "label": "Project skills (`F25.catalog-project`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-04-project-skills-f25-catalog-project.json", "failed_command": null, "variants": [ { "feature_id": "F25.catalog-project", "family": "F25", "result": "pass", "label": "Project skills (`F25.catalog-project`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-04-project-skills-f25-catalog-project.json", "failed_command": null } ] }, { "feature_id": "F25.catalog-registered-marketplace", "family": "F25", "result": "pass", "label": "Registered marketplace skills (`F25.catalog-registered-marketplace`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-25-registered-marketplace-skills-f25-catalo.json", "failed_command": null, "variants": [ { "feature_id": "F25.catalog-registered-marketplace", "family": "F25", "result": "pass", "label": "Registered marketplace skills (`F25.catalog-registered-marketplace`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-25-registered-marketplace-skills-f25-catalo.json", "failed_command": null } ] }, { "feature_id": "F25.catalog-server-marketplaces", "family": "F25", "result": "pass", "label": "Server-default marketplaces (`F25.catalog-server-marketplaces`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-29-server-default-marketplaces-f25-catalog-.json", "failed_command": null, "variants": [ { "feature_id": "F25.catalog-server-marketplaces", "family": "F25", "result": "pass", "label": "Server-default marketplaces (`F25.catalog-server-marketplaces`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-29-server-default-marketplaces-f25-catalog-.json", "failed_command": null } ] }, { "feature_id": "F25.catalog-server-sources", "family": "F25", "result": "pass", "label": "Sandbox and organization sources (`F25.catalog-server-sources`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-06-sandbox-and-organization-sources-f25-cat.json", "failed_command": null, "variants": [ { "feature_id": "F25.catalog-server-sources", "family": "F25", "result": "pass", "label": "Sandbox and organization sources (`F25.catalog-server-sources`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-06-sandbox-and-organization-sources-f25-cat.json", "failed_command": null } ] }, { "feature_id": "F25.catalog-user-precedence", "family": "F25", "result": "pass", "label": "User skills and precedence (`F25.catalog-user-precedence`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-05-user-skills-and-precedence-f25-catalog-u.json", "failed_command": null, "variants": [ { "feature_id": "F25.catalog-user-precedence", "family": "F25", "result": "pass", "label": "User skills and precedence (`F25.catalog-user-precedence`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-05-user-skills-and-precedence-f25-catalog-u.json", "failed_command": null } ] }, { "feature_id": "F25.catalog-validation", "family": "F25", "result": "pass", "label": "Request validation (`F25.catalog-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-07-request-validation-f25-catalog-validatio.json", "failed_command": null, "variants": [ { "feature_id": "F25.catalog-validation", "family": "F25", "result": "pass", "label": "Request validation (`F25.catalog-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-07-request-validation-f25-catalog-validatio.json", "failed_command": null } ] }, { "feature_id": "F25.conversation-activation", "family": "F25", "result": "pass", "label": "Installed skill in a conversation (`F25.conversation-activation`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-26-installed-skill-in-a-conversation-f25-co.json", "failed_command": null, "variants": [ { "feature_id": "F25.conversation-activation", "family": "F25", "result": "pass", "label": "Installed skill in a conversation (`F25.conversation-activation`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-26-installed-skill-in-a-conversation-f25-co.json", "failed_command": null } ] }, { "feature_id": "F25.enable-disable", "family": "F25", "result": "pass", "label": "Disable and enable (`F25.enable-disable`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-09-disable-and-enable-f25-enable-disable.json", "failed_command": null, "variants": [ { "feature_id": "F25.enable-disable", "family": "F25", "result": "pass", "label": "Disable and enable (`F25.enable-disable`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-09-disable-and-enable-f25-enable-disable.json", "failed_command": null } ] }, { "feature_id": "F25.install-bad-frontmatter", "family": "F25", "result": "xfail", "label": "Malformed frontmatter (`F25.install-bad-frontmatter`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-12-malformed-frontmatter-f25-install-bad-fr.json", "failed_command": "control-agent-server api POST /api/skills/install --json '{\"source\": \"'\"$F/qa-f25-badyaml\"'\"}' --expect 422 --save F25.install-bad-frontmatter/install", "variants": [ { "feature_id": "F25.install-bad-frontmatter", "family": "F25", "result": "xfail", "label": "Malformed frontmatter (`F25.install-bad-frontmatter`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-12-malformed-frontmatter-f25-install-bad-fr.json", "failed_command": "control-agent-server api POST /api/skills/install --json '{\"source\": \"'\"$F/qa-f25-badyaml\"'\"}' --expect 422 --save F25.install-bad-frontmatter/install" } ] }, { "feature_id": "F25.install-conflict", "family": "F25", "result": "pass", "label": "Already installed (`F25.install-conflict`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-10-already-installed-f25-install-conflict.json", "failed_command": null, "variants": [ { "feature_id": "F25.install-conflict", "family": "F25", "result": "pass", "label": "Already installed (`F25.install-conflict`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-10-already-installed-f25-install-conflict.json", "failed_command": null } ] }, { "feature_id": "F25.install-documented-sources", "family": "F25", "result": "xfail", "label": "Documented install sources (`F25.install-documented-sources`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-16-documented-install-sources-f25-install-d.json", "failed_command": "control-agent-server api POST /api/skills/install --json '{\"source\": \"'\"$SRC\"'\"}' --timeout 240 --expect 200 --check name eq github --save F25.install-documented-sources/documented", "variants": [ { "feature_id": "F25.install-documented-sources", "family": "F25", "result": "xfail", "label": "Documented install sources (`F25.install-documented-sources`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-16-documented-install-sources-f25-install-d.json", "failed_command": "control-agent-server api POST /api/skills/install --json '{\"source\": \"'\"$SRC\"'\"}' --timeout 240 --expect 200 --check name eq github --save F25.install-documented-sources/documented" } ] }, { "feature_id": "F25.install-errors", "family": "F25", "result": "pass", "label": "Install errors (`F25.install-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-11-install-errors-f25-install-errors.json", "failed_command": null, "variants": [ { "feature_id": "F25.install-errors", "family": "F25", "result": "pass", "label": "Install errors (`F25.install-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-11-install-errors-f25-install-errors.json", "failed_command": null } ] }, { "feature_id": "F25.install-git", "family": "F25", "result": "pass", "label": "Install from a git source (`F25.install-git`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-13-install-from-a-git-source-f25-install-gi.json", "failed_command": null, "variants": [ { "feature_id": "F25.install-git", "family": "F25", "result": "pass", "label": "Install from a git source (`F25.install-git`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-13-install-from-a-git-source-f25-install-gi.json", "failed_command": null } ] }, { "feature_id": "F25.install-github", "family": "F25", "result": "pass", "label": "Install from GitHub (`F25.install-github`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-15-install-from-github-f25-install-github.json", "failed_command": null, "variants": [ { "feature_id": "F25.install-github", "family": "F25", "result": "pass", "label": "Install from GitHub (`F25.install-github`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-15-install-from-github-f25-install-github.json", "failed_command": null } ] }, { "feature_id": "F25.install-local", "family": "F25", "result": "pass", "label": "Install from a local directory (`F25.install-local`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-08-install-from-a-local-directory-f25-insta.json", "failed_command": null, "variants": [ { "feature_id": "F25.install-local", "family": "F25", "result": "pass", "label": "Install from a local directory (`F25.install-local`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-08-install-from-a-local-directory-f25-insta.json", "failed_command": null } ] }, { "feature_id": "F25.install-ref", "family": "F25", "result": "pass", "label": "Install at a ref (`F25.install-ref`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-18-install-at-a-ref-f25-install-ref.json", "failed_command": null, "variants": [ { "feature_id": "F25.install-ref", "family": "F25", "result": "pass", "label": "Install at a ref (`F25.install-ref`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-18-install-at-a-ref-f25-install-ref.json", "failed_command": null } ] }, { "feature_id": "F25.install-unknown-ref", "family": "F25", "result": "xfail", "label": "Unknown ref (`F25.install-unknown-ref`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-19-unknown-ref-f25-install-unknown-ref-know.json", "failed_command": "control-agent-server api POST /api/skills/install --json '{\"source\": \"file://'\"$G\"'\", \"repo_path\": \"skills/qa-f25-src-skill\", \"ref\": \"qa-f25-no-such-ref\", \"force\": true}' --expect 400 --save F25.install-unknown-ref/cached", "variants": [ { "feature_id": "F25.install-unknown-ref", "family": "F25", "result": "xfail", "label": "Unknown ref (`F25.install-unknown-ref`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-19-unknown-ref-f25-install-unknown-ref-know.json", "failed_command": "control-agent-server api POST /api/skills/install --json '{\"source\": \"file://'\"$G\"'\", \"repo_path\": \"skills/qa-f25-src-skill\", \"ref\": \"qa-f25-no-such-ref\", \"force\": true}' --expect 400 --save F25.install-unknown-ref/cached" } ] }, { "feature_id": "F25.invoke-skill", "family": "F25", "result": "pass", "label": "Skill invoked through `invoke_skill` (`F25.invoke-skill`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-27-skill-invoked-through-invoke-skill-f25-i.json", "failed_command": null, "variants": [ { "feature_id": "F25.invoke-skill", "family": "F25", "result": "pass", "label": "Skill invoked through `invoke_skill` (`F25.invoke-skill`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-27-skill-invoked-through-invoke-skill-f25-i.json", "failed_command": null } ] }, { "feature_id": "F25.marketplace-catalog", "family": "F25", "result": "pass", "label": "Marketplace catalog (`F25.marketplace-catalog`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-30-marketplace-catalog-f25-marketplace-cata.json", "failed_command": null, "variants": [ { "feature_id": "F25.marketplace-catalog", "family": "F25", "result": "pass", "label": "Marketplace catalog (`F25.marketplace-catalog`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-30-marketplace-catalog-f25-marketplace-cata.json", "failed_command": null } ] }, { "feature_id": "F25.marketplace-public", "family": "F25", "result": "xfail", "label": "Public marketplace catalog (`F25.marketplace-public`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-03-public-marketplace-catalog-f25-marketpla.json", "failed_command": "control-agent-server api GET /api/skills/marketplace --timeout 180 --expect 200 --check skills len-ge 1 --save F25.marketplace-public/catalog", "variants": [ { "feature_id": "F25.marketplace-public", "family": "F25", "result": "xfail", "label": "Public marketplace catalog (`F25.marketplace-public`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-03-public-marketplace-catalog-f25-marketpla.json", "failed_command": "control-agent-server api GET /api/skills/marketplace --timeout 180 --expect 200 --check skills len-ge 1 --save F25.marketplace-public/catalog" } ] }, { "feature_id": "F25.marketplace-stale-cache", "family": "F25", "result": "xfail", "label": "Catalog after a sync (`F25.marketplace-stale-cache`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-32-catalog-after-a-sync-f25-marketplace-sta.json", "failed_command": "control-agent-server api GET /api/skills/marketplace --expect 200 --check skills contains '\"name\": \"qa-f25-pub-extra\"' --save F25.marketplace-stale-cache/after-sync", "variants": [ { "feature_id": "F25.marketplace-stale-cache", "family": "F25", "result": "xfail", "label": "Catalog after a sync (`F25.marketplace-stale-cache`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-32-catalog-after-a-sync-f25-marketplace-sta.json", "failed_command": "control-agent-server api GET /api/skills/marketplace --expect 200 --check skills contains '\"name\": \"qa-f25-pub-extra\"' --save F25.marketplace-stale-cache/after-sync" } ] }, { "feature_id": "F25.name-errors", "family": "F25", "result": "pass", "label": "Unknown and invalid names (`F25.name-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-21-unknown-and-invalid-names-f25-name-error.json", "failed_command": null, "variants": [ { "feature_id": "F25.name-errors", "family": "F25", "result": "pass", "label": "Unknown and invalid names (`F25.name-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-21-unknown-and-invalid-names-f25-name-error.json", "failed_command": null } ] }, { "feature_id": "F25.public-repo", "family": "F25", "result": "pass", "label": "The real public repository (`F25.public-repo`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-02-the-real-public-repository-f25-public-re.json", "failed_command": null, "variants": [ { "feature_id": "F25.public-repo", "family": "F25", "result": "pass", "label": "The real public repository (`F25.public-repo`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-02-the-real-public-repository-f25-public-re.json", "failed_command": null } ] }, { "feature_id": "F25.refresh", "family": "F25", "result": "pass", "label": "Refresh from the source (`F25.refresh`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-17-refresh-from-the-source-f25-refresh.json", "failed_command": null, "variants": [ { "feature_id": "F25.refresh", "family": "F25", "result": "pass", "label": "Refresh from the source (`F25.refresh`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-17-refresh-from-the-source-f25-refresh.json", "failed_command": null } ] }, { "feature_id": "F25.refresh-unfetchable", "family": "F25", "result": "xfail", "label": "Refresh of a vanished source (`F25.refresh-unfetchable`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-20-refresh-of-a-vanished-source-f25-refresh.json", "failed_command": "control-agent-server api POST /api/skills/installed/qa-f25-gone/refresh --expect 400,422 --save F25.refresh-unfetchable/refresh", "variants": [ { "feature_id": "F25.refresh-unfetchable", "family": "F25", "result": "xfail", "label": "Refresh of a vanished source (`F25.refresh-unfetchable`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-20-refresh-of-a-vanished-source-f25-refresh.json", "failed_command": "control-agent-server api POST /api/skills/installed/qa-f25-gone/refresh --expect 400,422 --save F25.refresh-unfetchable/refresh" } ] }, { "feature_id": "F25.restart-persist", "family": "F25", "result": "pass", "label": "Restart (`F25.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-28-restart-f25-restart-persist.json", "failed_command": null, "variants": [ { "feature_id": "F25.restart-persist", "family": "F25", "result": "pass", "label": "Restart (`F25.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-28-restart-f25-restart-persist.json", "failed_command": null } ] }, { "feature_id": "F25.sdk-load-skills", "family": "F25", "result": "pass", "label": "SDK consumer (`F25.sdk-load-skills`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-14-sdk-consumer-f25-sdk-load-skills.json", "failed_command": null, "variants": [ { "feature_id": "F25.sdk-load-skills", "family": "F25", "result": "pass", "label": "SDK consumer (`F25.sdk-load-skills`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-14-sdk-consumer-f25-sdk-load-skills.json", "failed_command": null } ] }, { "feature_id": "F25.self-heal", "family": "F25", "result": "pass", "label": "Hand-copied and hand-deleted directories (`F25.self-heal`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-22-hand-copied-and-hand-deleted-directories.json", "failed_command": null, "variants": [ { "feature_id": "F25.self-heal", "family": "F25", "result": "pass", "label": "Hand-copied and hand-deleted directories (`F25.self-heal`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-22-hand-copied-and-hand-deleted-directories.json", "failed_command": null } ] }, { "feature_id": "F25.sync", "family": "F25", "result": "pass", "label": "Sync the public repository (`F25.sync`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-23-sync-the-public-repository-f25-sync.json", "failed_command": null, "variants": [ { "feature_id": "F25.sync", "family": "F25", "result": "pass", "label": "Sync the public repository (`F25.sync`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-23-sync-the-public-repository-f25-sync.json", "failed_command": null } ] }, { "feature_id": "F25.sync-offline", "family": "F25", "result": "xfail", "label": "Sync without a reachable origin (`F25.sync-offline`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-24-sync-without-a-reachable-origin-f25-sync.json", "failed_command": "control-agent-server api POST /api/skills/sync --expect 200 --check status eq error --save F25.sync-offline/sync", "variants": [ { "feature_id": "F25.sync-offline", "family": "F25", "result": "xfail", "label": "Sync without a reachable origin (`F25.sync-offline`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-24-sync-without-a-reachable-origin-f25-sync.json", "failed_command": "control-agent-server api POST /api/skills/sync --expect 200 --check status eq error --save F25.sync-offline/sync" } ] }, { "feature_id": "F25.ts-client", "family": "F25", "result": "pass", "label": "TypeScript client (`F25.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-31-typescript-client-f25-ts-client.json", "failed_command": null, "variants": [ { "feature_id": "F25.ts-client", "family": "F25", "result": "pass", "label": "TypeScript client (`F25.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-31-typescript-client-f25-ts-client.json", "failed_command": null } ] }, { "feature_id": "F25.uninstall", "family": "F25", "result": "pass", "label": "Uninstall (`F25.uninstall`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-33-uninstall-f25-uninstall.json", "failed_command": null, "variants": [ { "feature_id": "F25.uninstall", "family": "F25", "result": "pass", "label": "Uninstall (`F25.uninstall`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-33-uninstall-f25-uninstall.json", "failed_command": null } ] }, { "feature_id": "F26.auth", "family": "F26", "result": "pass", "label": "Session key required (`F26.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-01-session-key-required-f26-auth.json", "failed_command": null, "variants": [ { "feature_id": "F26.auth", "family": "F26", "result": "pass", "label": "Session key required (`F26.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-01-session-key-required-f26-auth.json", "failed_command": null } ] }, { "feature_id": "F26.catalog-project", "family": "F26", "result": "pass", "label": "Project plugins (`F26.catalog-project`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-02-project-plugins-f26-catalog-project.json", "failed_command": null, "variants": [ { "feature_id": "F26.catalog-project", "family": "F26", "result": "pass", "label": "Project plugins (`F26.catalog-project`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-02-project-plugins-f26-catalog-project.json", "failed_command": null } ] }, { "feature_id": "F26.catalog-user-precedence", "family": "F26", "result": "pass", "label": "User plugins and precedence (`F26.catalog-user-precedence`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-03-user-plugins-and-precedence-f26-catalog-.json", "failed_command": null, "variants": [ { "feature_id": "F26.catalog-user-precedence", "family": "F26", "result": "pass", "label": "User plugins and precedence (`F26.catalog-user-precedence`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-03-user-plugins-and-precedence-f26-catalog-.json", "failed_command": null } ] }, { "feature_id": "F26.conversation-ambient", "family": "F26", "result": "pass", "label": "Ambient load into a conversation (`F26.conversation-ambient`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-18-ambient-load-into-a-conversation-f26-con.json", "failed_command": null, "variants": [ { "feature_id": "F26.conversation-ambient", "family": "F26", "result": "pass", "label": "Ambient load into a conversation (`F26.conversation-ambient`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-18-ambient-load-into-a-conversation-f26-con.json", "failed_command": null } ] }, { "feature_id": "F26.conversation-disabled", "family": "F26", "result": "pass", "label": "Disabled plugin in a new conversation (`F26.conversation-disabled`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-19-disabled-plugin-in-a-new-conversation-f2.json", "failed_command": null, "variants": [ { "feature_id": "F26.conversation-disabled", "family": "F26", "result": "pass", "label": "Disabled plugin in a new conversation (`F26.conversation-disabled`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-19-disabled-plugin-in-a-new-conversation-f2.json", "failed_command": null } ] }, { "feature_id": "F26.conversation-resume-disabled", "family": "F26", "result": "xfail", "label": "Resume after disabling (`F26.conversation-resume-disabled`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-20-resume-after-disabling-f26-conversation-.json", "failed_command": "control-agent-server ws stop qa-f26-resume --kinds MessageEvent --contains '\"activated_skills\": [\"qa-f26-inst:qa-hello\"]' --expect-none --save F26.conversation-resume-disabled/frames", "variants": [ { "feature_id": "F26.conversation-resume-disabled", "family": "F26", "result": "xfail", "label": "Resume after disabling (`F26.conversation-resume-disabled`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-20-resume-after-disabling-f26-conversation-.json", "failed_command": "control-agent-server ws stop qa-f26-resume --kinds MessageEvent --contains '\"activated_skills\": [\"qa-f26-inst:qa-hello\"]' --expect-none --save F26.conversation-resume-disabled/frames" } ] }, { "feature_id": "F26.enable-disable", "family": "F26", "result": "pass", "label": "Enable and disable (`F26.enable-disable`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-12-enable-and-disable-f26-enable-disable.json", "failed_command": null, "variants": [ { "feature_id": "F26.enable-disable", "family": "F26", "result": "pass", "label": "Enable and disable (`F26.enable-disable`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-12-enable-and-disable-f26-enable-disable.json", "failed_command": null } ] }, { "feature_id": "F26.install-agent-plugins", "family": "F26", "result": "pass", "label": "Agent Plugins layout (`F26.install-agent-plugins`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-05-agent-plugins-layout-f26-install-agent-p.json", "failed_command": null, "variants": [ { "feature_id": "F26.install-agent-plugins", "family": "F26", "result": "pass", "label": "Agent Plugins layout (`F26.install-agent-plugins`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-05-agent-plugins-layout-f26-install-agent-p.json", "failed_command": null } ] }, { "feature_id": "F26.install-conflict", "family": "F26", "result": "pass", "label": "Duplicate install (`F26.install-conflict`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-08-duplicate-install-f26-install-conflict.json", "failed_command": null, "variants": [ { "feature_id": "F26.install-conflict", "family": "F26", "result": "pass", "label": "Duplicate install (`F26.install-conflict`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-08-duplicate-install-f26-install-conflict.json", "failed_command": null } ] }, { "feature_id": "F26.install-documented-source", "family": "F26", "result": "xfail", "label": "Documented install source (`F26.install-documented-source`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-11-documented-install-source-f26-install-do.json", "failed_command": "control-agent-server api POST /api/plugins/install --json '{\"source\": \"github:/plugins/city-weather\"}' --timeout 240 --expect 200 --check name eq city-weather --save F26.install-documented-source/documented", "variants": [ { "feature_id": "F26.install-documented-source", "family": "F26", "result": "xfail", "label": "Documented install source (`F26.install-documented-source`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-11-documented-install-source-f26-install-do.json", "failed_command": "control-agent-server api POST /api/plugins/install --json '{\"source\": \"github:/plugins/city-weather\"}' --timeout 240 --expect 200 --check name eq city-weather --save F26.install-documented-source/documented" } ] }, { "feature_id": "F26.install-errors", "family": "F26", "result": "pass", "label": "Install errors (`F26.install-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-09-install-errors-f26-install-errors.json", "failed_command": null, "variants": [ { "feature_id": "F26.install-errors", "family": "F26", "result": "pass", "label": "Install errors (`F26.install-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-09-install-errors-f26-install-errors.json", "failed_command": null } ] }, { "feature_id": "F26.install-git", "family": "F26", "result": "pass", "label": "Install from a git source (`F26.install-git`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-06-install-from-a-git-source-f26-install-gi.json", "failed_command": null, "variants": [ { "feature_id": "F26.install-git", "family": "F26", "result": "pass", "label": "Install from a git source (`F26.install-git`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-06-install-from-a-git-source-f26-install-gi.json", "failed_command": null } ] }, { "feature_id": "F26.install-github", "family": "F26", "result": "pass", "label": "Install from GitHub (`F26.install-github`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-10-install-from-github-f26-install-github.json", "failed_command": null, "variants": [ { "feature_id": "F26.install-github", "family": "F26", "result": "pass", "label": "Install from GitHub (`F26.install-github`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-10-install-from-github-f26-install-github.json", "failed_command": null } ] }, { "feature_id": "F26.install-local", "family": "F26", "result": "pass", "label": "Install from a local path (`F26.install-local`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-04-install-from-a-local-path-f26-install-lo.json", "failed_command": null, "variants": [ { "feature_id": "F26.install-local", "family": "F26", "result": "pass", "label": "Install from a local path (`F26.install-local`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-04-install-from-a-local-path-f26-install-lo.json", "failed_command": null } ] }, { "feature_id": "F26.install-repo-root-name", "family": "F26", "result": "xfail", "label": "Repository root name (`F26.install-repo-root-name`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-07-repository-root-name-f26-install-repo-ro.json", "failed_command": "test \"$N\" = qa-f26-src", "variants": [ { "feature_id": "F26.install-repo-root-name", "family": "F26", "result": "xfail", "label": "Repository root name (`F26.install-repo-root-name`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-07-repository-root-name-f26-install-repo-ro.json", "failed_command": "test \"$N\" = qa-f26-src" } ] }, { "feature_id": "F26.marketplace-cache", "family": "F26", "result": "pass", "label": "Catalog cache (`F26.marketplace-cache`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-23-catalog-cache-f26-marketplace-cache.json", "failed_command": null, "variants": [ { "feature_id": "F26.marketplace-cache", "family": "F26", "result": "pass", "label": "Catalog cache (`F26.marketplace-cache`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-23-catalog-cache-f26-marketplace-cache.json", "failed_command": null } ] }, { "feature_id": "F26.marketplace-local", "family": "F26", "result": "pass", "label": "Marketplace against a local stand-in (`F26.marketplace-local`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-22-marketplace-against-a-local-stand-in-f26.json", "failed_command": null, "variants": [ { "feature_id": "F26.marketplace-local", "family": "F26", "result": "pass", "label": "Marketplace against a local stand-in (`F26.marketplace-local`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-22-marketplace-against-a-local-stand-in-f26.json", "failed_command": null } ] }, { "feature_id": "F26.marketplace-public", "family": "F26", "result": "pass", "label": "Public marketplace (`F26.marketplace-public`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-25-public-marketplace-f26-marketplace-publi.json", "failed_command": null, "variants": [ { "feature_id": "F26.marketplace-public", "family": "F26", "result": "pass", "label": "Public marketplace (`F26.marketplace-public`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-25-public-marketplace-f26-marketplace-publi.json", "failed_command": null } ] }, { "feature_id": "F26.name-errors", "family": "F26", "result": "pass", "label": "Unknown and invalid names (`F26.name-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-13-unknown-and-invalid-names-f26-name-error.json", "failed_command": null, "variants": [ { "feature_id": "F26.name-errors", "family": "F26", "result": "pass", "label": "Unknown and invalid names (`F26.name-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-13-unknown-and-invalid-names-f26-name-error.json", "failed_command": null } ] }, { "feature_id": "F26.refresh", "family": "F26", "result": "pass", "label": "Refresh (`F26.refresh`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-14-refresh-f26-refresh.json", "failed_command": null, "variants": [ { "feature_id": "F26.refresh", "family": "F26", "result": "pass", "label": "Refresh (`F26.refresh`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-14-refresh-f26-refresh.json", "failed_command": null } ] }, { "feature_id": "F26.refresh-unfetchable", "family": "F26", "result": "xfail", "label": "Unfetchable refresh (`F26.refresh-unfetchable`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-16-unfetchable-refresh-f26-refresh-unfetcha.json", "failed_command": "control-agent-server api POST /api/plugins/installed/qa-f26-gone/refresh --expect 400,422 --save F26.refresh-unfetchable/vanished", "variants": [ { "feature_id": "F26.refresh-unfetchable", "family": "F26", "result": "xfail", "label": "Unfetchable refresh (`F26.refresh-unfetchable`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-16-unfetchable-refresh-f26-refresh-unfetcha.json", "failed_command": "control-agent-server api POST /api/plugins/installed/qa-f26-gone/refresh --expect 400,422 --save F26.refresh-unfetchable/vanished" } ] }, { "feature_id": "F26.restart-persist", "family": "F26", "result": "pass", "label": "Persistence across a restart (`F26.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-21-persistence-across-a-restart-f26-restart.json", "failed_command": null, "variants": [ { "feature_id": "F26.restart-persist", "family": "F26", "result": "pass", "label": "Persistence across a restart (`F26.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-21-persistence-across-a-restart-f26-restart.json", "failed_command": null } ] }, { "feature_id": "F26.self-heal", "family": "F26", "result": "pass", "label": "Hand-edited store (`F26.self-heal`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-15-hand-edited-store-f26-self-heal.json", "failed_command": null, "variants": [ { "feature_id": "F26.self-heal", "family": "F26", "result": "pass", "label": "Hand-edited store (`F26.self-heal`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-15-hand-edited-store-f26-self-heal.json", "failed_command": null } ] }, { "feature_id": "F26.ts-client", "family": "F26", "result": "pass", "label": "TypeScript client (`F26.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-24-typescript-client-f26-ts-client.json", "failed_command": null, "variants": [ { "feature_id": "F26.ts-client", "family": "F26", "result": "pass", "label": "TypeScript client (`F26.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-24-typescript-client-f26-ts-client.json", "failed_command": null } ] }, { "feature_id": "F26.uninstall", "family": "F26", "result": "pass", "label": "Uninstall (`F26.uninstall`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-17-uninstall-f26-uninstall.json", "failed_command": null, "variants": [ { "feature_id": "F26.uninstall", "family": "F26", "result": "pass", "label": "Uninstall (`F26.uninstall`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-17-uninstall-f26-uninstall.json", "failed_command": null } ] }, { "feature_id": "F27.agent-definitions", "family": "F27", "result": "pass", "label": "Forwarded sub-agent definitions (`F27.agent-definitions`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-21-forwarded-sub-agent-definitions-f27-agen.json", "failed_command": null, "variants": [ { "feature_id": "F27.agent-definitions", "family": "F27", "result": "pass", "label": "Forwarded sub-agent definitions (`F27.agent-definitions`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-21-forwarded-sub-agent-definitions-f27-agen.json", "failed_command": null } ] }, { "feature_id": "F27.auth", "family": "F27", "result": "pass", "label": "Session key required (`F27.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-01-session-key-required-f27-auth.json", "failed_command": null, "variants": [ { "feature_id": "F27.auth", "family": "F27", "result": "pass", "label": "Session key required (`F27.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-01-session-key-required-f27-auth.json", "failed_command": null } ] }, { "feature_id": "F27.browser-disabled", "family": "F27", "result": "fail", "label": "Browser disabled by config (`F27.browser-disabled`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-28-browser-disabled-by-config-f27-browser-d.json", "failed_command": "test -L \"$H/.cache/ms-playwright\"", "variants": [ { "feature_id": "F27.browser-disabled", "family": "F27", "result": "fail", "label": "Browser disabled by config (`F27.browser-disabled`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-28-browser-disabled-by-config-f27-browser-d.json", "failed_command": "test -L \"$H/.cache/ms-playwright\"" } ] }, { "feature_id": "F27.browser-disabled-launch", "family": "F27", "result": "fail", "label": "Default tools follow the browser setting (`F27.browser-disabled-launch`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-27-default-tools-follow-the-browser-setting.json", "failed_command": "test -L \"$H/.cache/ms-playwright\"", "variants": [ { "feature_id": "F27.browser-disabled-launch", "family": "F27", "result": "fail", "label": "Default tools follow the browser setting (`F27.browser-disabled-launch`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-27-default-tools-follow-the-browser-setting.json", "failed_command": "test -L \"$H/.cache/ms-playwright\"" } ] }, { "feature_id": "F27.browser-playwright-path", "family": "F27", "result": "fail", "label": "Playwright browsers directory (`F27.browser-playwright-path`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-25-playwright-browsers-directory-f27-browse.json", "failed_command": "control-agent-server api GET /api/tools/catalog --expect 200 --quiet --check tools contains '{\"name\": \"browser_tool_set\", \"user_selectable\": true, \"usable\": false'", "variants": [ { "feature_id": "F27.browser-playwright-path", "family": "F27", "result": "fail", "label": "Playwright browsers directory (`F27.browser-playwright-path`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-25-playwright-browsers-directory-f27-browse.json", "failed_command": "control-agent-server api GET /api/tools/catalog --expect 200 --quiet --check tools contains '{\"name\": \"browser_tool_set\", \"user_selectable\": true, \"usable\": false'" } ] }, { "feature_id": "F27.browser-probe", "family": "F27", "result": "fail", "label": "Browser probe (`F27.browser-probe`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-26-browser-probe-f27-browser-probe.json", "failed_command": "control-agent-server api GET /api/tools/catalog --expect 200 --quiet --check tools contains '{\"name\": \"browser_tool_set\", \"user_selectable\": true, \"usable\": false' --save F27.browser-probe/catalog-before", "variants": [ { "feature_id": "F27.browser-probe", "family": "F27", "result": "fail", "label": "Browser probe (`F27.browser-probe`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-26-browser-probe-f27-browser-probe.json", "failed_command": "control-agent-server api GET /api/tools/catalog --expect 200 --quiet --check tools contains '{\"name\": \"browser_tool_set\", \"user_selectable\": true, \"usable\": false' --save F27.browser-probe/catalog-before" } ] }, { "feature_id": "F27.catalog", "family": "F27", "result": "pass", "label": "Tool catalog (`F27.catalog`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-05-tool-catalog-f27-catalog.json", "failed_command": null, "variants": [ { "feature_id": "F27.catalog", "family": "F27", "result": "pass", "label": "Tool catalog (`F27.catalog`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-05-tool-catalog-f27-catalog.json", "failed_command": null } ] }, { "feature_id": "F27.catalog-docker-runtime", "family": "F27", "result": "fail", "label": "Docker conversation runtime (`F27.catalog-docker-runtime`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-31-docker-conversation-runtime-f27-catalog-.json", "failed_command": "rm \"$H/.cache/ms-playwright\"", "variants": [ { "feature_id": "F27.catalog-docker-runtime", "family": "F27", "result": "fail", "label": "Docker conversation runtime (`F27.catalog-docker-runtime`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-31-docker-conversation-runtime-f27-catalog-.json", "failed_command": "rm \"$H/.cache/ms-playwright\"" } ] }, { "feature_id": "F27.catalog-sealed", "family": "F27", "result": "pass", "label": "Client tools stay out of the catalog (`F27.catalog-sealed`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-19-client-tools-stay-out-of-the-catalog-f27.json", "failed_command": null, "variants": [ { "feature_id": "F27.catalog-sealed", "family": "F27", "result": "pass", "label": "Client tools stay out of the catalog (`F27.catalog-sealed`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-19-client-tools-stay-out-of-the-catalog-f27.json", "failed_command": null } ] }, { "feature_id": "F27.client-tool-call", "family": "F27", "result": "pass", "label": "Client tool call (`F27.client-tool-call`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-20-client-tool-call-f27-client-tool-call.json", "failed_command": null, "variants": [ { "feature_id": "F27.client-tool-call", "family": "F27", "result": "pass", "label": "Client tool call (`F27.client-tool-call`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-20-client-tool-call-f27-client-tool-call.json", "failed_command": null } ] }, { "feature_id": "F27.custom-tool-missing-on-resume", "family": "F27", "result": "xfail", "label": "Tool module gone on resume (`F27.custom-tool-missing-on-resume`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-24-tool-module-gone-on-resume-f27-custom-to.json", "failed_command": "control-agent-server api POST \"/$MCID/events\" --json '{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"Reply with the single word ok.\"}], \"run\": false}' --expect 2xx,4xx --save F27.custom-tool-missing-on-resume/send", "variants": [ { "feature_id": "F27.custom-tool-missing-on-resume", "family": "F27", "result": "xfail", "label": "Tool module gone on resume (`F27.custom-tool-missing-on-resume`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-24-tool-module-gone-on-resume-f27-custom-to.json", "failed_command": "control-agent-server api POST \"/$MCID/events\" --json '{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"Reply with the single word ok.\"}], \"run\": false}' --expect 2xx,4xx --save F27.custom-tool-missing-on-resume/send" } ] }, { "feature_id": "F27.custom-tool-modules", "family": "F27", "result": "pass", "label": "Custom tool module (`F27.custom-tool-modules`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-22-custom-tool-module-f27-custom-tool-modul.json", "failed_command": null, "variants": [ { "feature_id": "F27.custom-tool-modules", "family": "F27", "result": "pass", "label": "Custom tool module (`F27.custom-tool-modules`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-22-custom-tool-module-f27-custom-tool-modul.json", "failed_command": null } ] }, { "feature_id": "F27.hooks-absent", "family": "F27", "result": "pass", "label": "No hooks (`F27.hooks-absent`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-08-no-hooks-f27-hooks-absent.json", "failed_command": null, "variants": [ { "feature_id": "F27.hooks-absent", "family": "F27", "result": "pass", "label": "No hooks (`F27.hooks-absent`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-08-no-hooks-f27-hooks-absent.json", "failed_command": null } ] }, { "feature_id": "F27.hooks-block", "family": "F27", "result": "pass", "label": "Blocking hooks (`F27.hooks-block`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-18-blocking-hooks-f27-hooks-block.json", "failed_command": null, "variants": [ { "feature_id": "F27.hooks-block", "family": "F27", "result": "pass", "label": "Blocking hooks (`F27.hooks-block`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-18-blocking-hooks-f27-hooks-block.json", "failed_command": null } ] }, { "feature_id": "F27.hooks-conversation", "family": "F27", "result": "pass", "label": "Hooks in a conversation (`F27.hooks-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-16-hooks-in-a-conversation-f27-hooks-conver.json", "failed_command": null, "variants": [ { "feature_id": "F27.hooks-conversation", "family": "F27", "result": "pass", "label": "Hooks in a conversation (`F27.hooks-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-16-hooks-in-a-conversation-f27-hooks-conver.json", "failed_command": null } ] }, { "feature_id": "F27.hooks-formats", "family": "F27", "result": "pass", "label": "Formats (`F27.hooks-formats`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-07-formats-f27-hooks-formats.json", "failed_command": null, "variants": [ { "feature_id": "F27.hooks-formats", "family": "F27", "result": "pass", "label": "Formats (`F27.hooks-formats`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-07-formats-f27-hooks-formats.json", "failed_command": null } ] }, { "feature_id": "F27.hooks-invalid", "family": "F27", "result": "pass", "label": "Broken hooks.json (`F27.hooks-invalid`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-09-broken-hooks-json-f27-hooks-invalid.json", "failed_command": null, "variants": [ { "feature_id": "F27.hooks-invalid", "family": "F27", "result": "pass", "label": "Broken hooks.json (`F27.hooks-invalid`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-09-broken-hooks-json-f27-hooks-invalid.json", "failed_command": null } ] }, { "feature_id": "F27.hooks-not-automatic", "family": "F27", "result": "pass", "label": "Hooks are not loaded implicitly (`F27.hooks-not-automatic`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-17-hooks-are-not-loaded-implicitly-f27-hook.json", "failed_command": null, "variants": [ { "feature_id": "F27.hooks-not-automatic", "family": "F27", "result": "pass", "label": "Hooks are not loaded implicitly (`F27.hooks-not-automatic`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-17-hooks-are-not-loaded-implicitly-f27-hook.json", "failed_command": null } ] }, { "feature_id": "F27.hooks-project", "family": "F27", "result": "pass", "label": "Project hooks (`F27.hooks-project`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-06-project-hooks-f27-hooks-project.json", "failed_command": null, "variants": [ { "feature_id": "F27.hooks-project", "family": "F27", "result": "pass", "label": "Project hooks (`F27.hooks-project`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-06-project-hooks-f27-hooks-project.json", "failed_command": null } ] }, { "feature_id": "F27.import-modules", "family": "F27", "result": "pass", "label": "Startup imports (`F27.import-modules`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-23-startup-imports-f27-import-modules.json", "failed_command": null, "variants": [ { "feature_id": "F27.import-modules", "family": "F27", "result": "pass", "label": "Startup imports (`F27.import-modules`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-23-startup-imports-f27-import-modules.json", "failed_command": null } ] }, { "feature_id": "F27.server-info-browser-disabled", "family": "F27", "result": "xfail", "label": "Usable tools with the browser disabled (`F27.server-info-browser-disabled`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-29-usable-tools-with-the-browser-disabled-f.json", "failed_command": "control-agent-server api GET /server_info --auth none --expect 200 --check usable_tools contains terminal --check usable_tools not-contains browser_tool_set --save F27.server-info-browser-disabled/server-info", "variants": [ { "feature_id": "F27.server-info-browser-disabled", "family": "F27", "result": "xfail", "label": "Usable tools with the browser disabled (`F27.server-info-browser-disabled`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-29-usable-tools-with-the-browser-disabled-f.json", "failed_command": "control-agent-server api GET /server_info --auth none --expect 200 --check usable_tools contains terminal --check usable_tools not-contains browser_tool_set --save F27.server-info-browser-disabled/server-info" } ] }, { "feature_id": "F27.subagents-bad-skill", "family": "F27", "result": "xfail", "label": "Agent with a missing skill (`F27.subagents-bad-skill`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-15-agent-with-a-missing-skill-f27-subagents.json", "failed_command": "control-agent-server api POST \"/$BCID/events\" --json '{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"hi\"}], \"run\": false}' --expect 2xx,4xx --save F27.subagents-bad-skill/send", "variants": [ { "feature_id": "F27.subagents-bad-skill", "family": "F27", "result": "xfail", "label": "Agent with a missing skill (`F27.subagents-bad-skill`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-15-agent-with-a-missing-skill-f27-subagents.json", "failed_command": "control-agent-server api POST \"/$BCID/events\" --json '{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"hi\"}], \"run\": false}' --expect 2xx,4xx --save F27.subagents-bad-skill/send" } ] }, { "feature_id": "F27.subagents-browser-disabled", "family": "F27", "result": "xfail", "label": "Browser-only sub-agent with the browser disabled (`F27.subagents-browser-disabled`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-30-browser-only-sub-agent-with-the-browser-.json", "failed_command": "control-agent-server api POST /api/sub-agents --json '{\"load_user\": false, \"load_project\": false}' --expect 200 --check agents contains '\"name\": \"bash-runner\"' --check agents not-contains '\"name\": \"web-researcher\"' --save F27.subagents-browser-disabled/builtins", "variants": [ { "feature_id": "F27.subagents-browser-disabled", "family": "F27", "result": "xfail", "label": "Browser-only sub-agent with the browser disabled (`F27.subagents-browser-disabled`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-30-browser-only-sub-agent-with-the-browser-.json", "failed_command": "control-agent-server api POST /api/sub-agents --json '{\"load_user\": false, \"load_project\": false}' --expect 200 --check agents contains '\"name\": \"bash-runner\"' --check agents not-contains '\"name\": \"web-researcher\"' --save F27.subagents-browser-disabled/builtins" } ] }, { "feature_id": "F27.subagents-builtin", "family": "F27", "result": "pass", "label": "Sub-agent built-ins (`F27.subagents-builtin`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-10-sub-agent-built-ins-f27-subagents-builti.json", "failed_command": null, "variants": [ { "feature_id": "F27.subagents-builtin", "family": "F27", "result": "pass", "label": "Sub-agent built-ins (`F27.subagents-builtin`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-10-sub-agent-built-ins-f27-subagents-builti.json", "failed_command": null } ] }, { "feature_id": "F27.subagents-invalid", "family": "F27", "result": "pass", "label": "Invalid definitions (`F27.subagents-invalid`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-14-invalid-definitions-f27-subagents-invali.json", "failed_command": null, "variants": [ { "feature_id": "F27.subagents-invalid", "family": "F27", "result": "pass", "label": "Invalid definitions (`F27.subagents-invalid`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-14-invalid-definitions-f27-subagents-invali.json", "failed_command": null } ] }, { "feature_id": "F27.subagents-precedence", "family": "F27", "result": "pass", "label": "Precedence (`F27.subagents-precedence`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-13-precedence-f27-subagents-precedence.json", "failed_command": null, "variants": [ { "feature_id": "F27.subagents-precedence", "family": "F27", "result": "pass", "label": "Precedence (`F27.subagents-precedence`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-13-precedence-f27-subagents-precedence.json", "failed_command": null } ] }, { "feature_id": "F27.subagents-project", "family": "F27", "result": "pass", "label": "Project sub-agents (`F27.subagents-project`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-11-project-sub-agents-f27-subagents-project.json", "failed_command": null, "variants": [ { "feature_id": "F27.subagents-project", "family": "F27", "result": "pass", "label": "Project sub-agents (`F27.subagents-project`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-11-project-sub-agents-f27-subagents-project.json", "failed_command": null } ] }, { "feature_id": "F27.subagents-user", "family": "F27", "result": "pass", "label": "User sub-agents (`F27.subagents-user`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-12-user-sub-agents-f27-subagents-user.json", "failed_command": null, "variants": [ { "feature_id": "F27.subagents-user", "family": "F27", "result": "pass", "label": "User sub-agents (`F27.subagents-user`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-12-user-sub-agents-f27-subagents-user.json", "failed_command": null } ] }, { "feature_id": "F27.tools-list", "family": "F27", "result": "pass", "label": "Registered tools (`F27.tools-list`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-03-registered-tools-f27-tools-list.json", "failed_command": null, "variants": [ { "feature_id": "F27.tools-list", "family": "F27", "result": "pass", "label": "Registered tools (`F27.tools-list`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-03-registered-tools-f27-tools-list.json", "failed_command": null } ] }, { "feature_id": "F27.tools-redirect", "family": "F27", "result": "pass", "label": "Slash redirects (`F27.tools-redirect`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-04-slash-redirects-f27-tools-redirect.json", "failed_command": null, "variants": [ { "feature_id": "F27.tools-redirect", "family": "F27", "result": "pass", "label": "Slash redirects (`F27.tools-redirect`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-04-slash-redirects-f27-tools-redirect.json", "failed_command": null } ] }, { "feature_id": "F27.validation", "family": "F27", "result": "pass", "label": "Bad bodies (`F27.validation`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-02-bad-bodies-f27-validation.json", "failed_command": null, "variants": [ { "feature_id": "F27.validation", "family": "F27", "result": "pass", "label": "Bad bodies (`F27.validation`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-02-bad-bodies-f27-validation.json", "failed_command": null } ] }, { "feature_id": "F28.backend-approval-gate", "family": "F28", "result": "pass", "label": "Approval gate (`F28.backend-approval-gate`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-13-approval-gate-f28-backend-approval-gate.json", "failed_command": null, "variants": [ { "feature_id": "F28.backend-approval-gate", "family": "F28", "result": "pass", "label": "Approval gate (`F28.backend-approval-gate`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-13-approval-gate-f28-backend-approval-gate.json", "failed_command": null } ] }, { "feature_id": "F28.backend-inherit-environment", "family": "F28", "result": "pass", "label": "Inherited environment (`F28.backend-inherit-environment`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-21-inherited-environment-f28-backend-inheri.json", "failed_command": null, "variants": [ { "feature_id": "F28.backend-inherit-environment", "family": "F28", "result": "pass", "label": "Inherited environment (`F28.backend-inherit-environment`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-21-inherited-environment-f28-backend-inheri.json", "failed_command": null } ] }, { "feature_id": "F28.backend-logs", "family": "F28", "result": "pass", "label": "Logs while starting (`F28.backend-logs`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-18-logs-while-starting-f28-backend-logs.json", "failed_command": null, "variants": [ { "feature_id": "F28.backend-logs", "family": "F28", "result": "pass", "label": "Logs while starting (`F28.backend-logs`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-18-logs-while-starting-f28-backend-logs.json", "failed_command": null } ] }, { "feature_id": "F28.backend-prepare", "family": "F28", "result": "pass", "label": "Prepare (`F28.backend-prepare`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-14-prepare-f28-backend-prepare.json", "failed_command": null, "variants": [ { "feature_id": "F28.backend-prepare", "family": "F28", "result": "pass", "label": "Prepare (`F28.backend-prepare`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-14-prepare-f28-backend-prepare.json", "failed_command": null } ] }, { "feature_id": "F28.backend-start", "family": "F28", "result": "pass", "label": "Start (`F28.backend-start`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-15-start-f28-backend-start.json", "failed_command": null, "variants": [ { "feature_id": "F28.backend-start", "family": "F28", "result": "pass", "label": "Start (`F28.backend-start`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-15-start-f28-backend-start.json", "failed_command": null } ] }, { "feature_id": "F28.backend-start-fails", "family": "F28", "result": "pass", "label": "Failed starts (`F28.backend-start-fails`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-20-failed-starts-f28-backend-start-fails.json", "failed_command": null, "variants": [ { "feature_id": "F28.backend-start-fails", "family": "F28", "result": "pass", "label": "Failed starts (`F28.backend-start-fails`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-20-failed-starts-f28-backend-start-fails.json", "failed_command": null } ] }, { "feature_id": "F28.backend-status", "family": "F28", "result": "pass", "label": "Backend status (`F28.backend-status`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-12-backend-status-f28-backend-status.json", "failed_command": null, "variants": [ { "feature_id": "F28.backend-status", "family": "F28", "result": "pass", "label": "Backend status (`F28.backend-status`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-12-backend-status-f28-backend-status.json", "failed_command": null } ] }, { "feature_id": "F28.backend-stop", "family": "F28", "result": "fail", "label": "Stop (`F28.backend-stop`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-16-stop-f28-backend-stop.json", "failed_command": "control-agent-server api POST /api/canvas-extensions/installed/qa-f28-flaky/backend/stop --expect 200 --check state eq stopped --expect-max-ms 7000 --save F28.backend-stop/group", "variants": [ { "feature_id": "F28.backend-stop", "family": "F28", "result": "fail", "label": "Stop (`F28.backend-stop`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-16-stop-f28-backend-stop.json", "failed_command": "control-agent-server api POST /api/canvas-extensions/installed/qa-f28-flaky/backend/stop --expect 200 --check state eq stopped --expect-max-ms 7000 --save F28.backend-stop/group" } ] }, { "feature_id": "F28.broken-install", "family": "F28", "result": "pass", "label": "Broken installs (`F28.broken-install`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-11-broken-installs-f28-broken-install.json", "failed_command": null, "variants": [ { "feature_id": "F28.broken-install", "family": "F28", "result": "pass", "label": "Broken installs (`F28.broken-install`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-11-broken-installs-f28-broken-install.json", "failed_command": null } ] }, { "feature_id": "F28.bundle", "family": "F28", "result": "pass", "label": "Bundle (`F28.bundle`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-06-bundle-f28-bundle.json", "failed_command": null, "variants": [ { "feature_id": "F28.bundle", "family": "F28", "result": "pass", "label": "Bundle (`F28.bundle`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-06-bundle-f28-bundle.json", "failed_command": null } ] }, { "feature_id": "F28.disable-stops-backend", "family": "F28", "result": "pass", "label": "Disable stops the backend (`F28.disable-stops-backend`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-22-disable-stops-the-backend-f28-disable-st.json", "failed_command": null, "variants": [ { "feature_id": "F28.disable-stops-backend", "family": "F28", "result": "pass", "label": "Disable stops the backend (`F28.disable-stops-backend`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-22-disable-stops-the-backend-f28-disable-st.json", "failed_command": null } ] }, { "feature_id": "F28.enable-disable", "family": "F28", "result": "pass", "label": "Enable and disable (`F28.enable-disable`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-08-enable-and-disable-f28-enable-disable.json", "failed_command": null, "variants": [ { "feature_id": "F28.enable-disable", "family": "F28", "result": "pass", "label": "Enable and disable (`F28.enable-disable`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-08-enable-and-disable-f28-enable-disable.json", "failed_command": null } ] }, { "feature_id": "F28.force-reinstall", "family": "F28", "result": "pass", "label": "Reinstall (`F28.force-reinstall`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-17-reinstall-f28-force-reinstall.json", "failed_command": null, "variants": [ { "feature_id": "F28.force-reinstall", "family": "F28", "result": "pass", "label": "Reinstall (`F28.force-reinstall`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-17-reinstall-f28-force-reinstall.json", "failed_command": null } ] }, { "feature_id": "F28.hard-restart-orphan", "family": "F28", "result": "xfail", "label": "Backend outlives a crash (`F28.hard-restart-orphan`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-29-backend-outlives-a-crash-f28-hard-restar.json", "failed_command": "false", "variants": [ { "feature_id": "F28.hard-restart-orphan", "family": "F28", "result": "xfail", "label": "Backend outlives a crash (`F28.hard-restart-orphan`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-29-backend-outlives-a-crash-f28-hard-restar.json", "failed_command": "false" } ] }, { "feature_id": "F28.icon", "family": "F28", "result": "pass", "label": "Icon (`F28.icon`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-07-icon-f28-icon.json", "failed_command": null, "variants": [ { "feature_id": "F28.icon", "family": "F28", "result": "pass", "label": "Icon (`F28.icon`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-07-icon-f28-icon.json", "failed_command": null } ] }, { "feature_id": "F28.install-errors", "family": "F28", "result": "pass", "label": "Install errors (`F28.install-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-05-install-errors-f28-install-errors.json", "failed_command": null, "variants": [ { "feature_id": "F28.install-errors", "family": "F28", "result": "pass", "label": "Install errors (`F28.install-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-05-install-errors-f28-install-errors.json", "failed_command": null } ] }, { "feature_id": "F28.install-git", "family": "F28", "result": "pass", "label": "Install from a git URL (`F28.install-git`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-02-install-from-a-git-url-f28-install-git.json", "failed_command": null, "variants": [ { "feature_id": "F28.install-git", "family": "F28", "result": "pass", "label": "Install from a git URL (`F28.install-git`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-02-install-from-a-git-url-f28-install-git.json", "failed_command": null } ] }, { "feature_id": "F28.install-local", "family": "F28", "result": "pass", "label": "Install from a local directory (`F28.install-local`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-01-install-from-a-local-directory-f28-insta.json", "failed_command": null, "variants": [ { "feature_id": "F28.install-local", "family": "F28", "result": "pass", "label": "Install from a local directory (`F28.install-local`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-01-install-from-a-local-directory-f28-insta.json", "failed_command": null } ] }, { "feature_id": "F28.install-ref", "family": "F28", "result": "pass", "label": "Pin a git ref (`F28.install-ref`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-03-pin-a-git-ref-f28-install-ref.json", "failed_command": null, "variants": [ { "feature_id": "F28.install-ref", "family": "F28", "result": "pass", "label": "Pin a git ref (`F28.install-ref`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-03-pin-a-git-ref-f28-install-ref.json", "failed_command": null } ] }, { "feature_id": "F28.install-ref-cached", "family": "F28", "result": "xfail", "label": "Unknown ref on a cached source (`F28.install-ref-cached`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-26-unknown-ref-on-a-cached-source-f28-insta.json", "failed_command": "control-agent-server api POST /api/canvas-extensions/install --json \"{\\\"source\\\": \\\"file://$GX\\\", \\\"ref\\\": \\\"qa-no-such-ref\\\", \\\"force\\\": true}\" --expect 400 --check detail contains 'Could not read canvas extension source' --save F28.install-ref-cached/unknown-ref", "variants": [ { "feature_id": "F28.install-ref-cached", "family": "F28", "result": "xfail", "label": "Unknown ref on a cached source (`F28.install-ref-cached`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-26-unknown-ref-on-a-cached-source-f28-insta.json", "failed_command": "control-agent-server api POST /api/canvas-extensions/install --json \"{\\\"source\\\": \\\"file://$GX\\\", \\\"ref\\\": \\\"qa-no-such-ref\\\", \\\"force\\\": true}\" --expect 400 --check detail contains 'Could not read canvas extension source' --save F28.install-ref-cached/unknown-ref" } ] }, { "feature_id": "F28.install-repo-path", "family": "F28", "result": "pass", "label": "Install a subdirectory (`F28.install-repo-path`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-04-install-a-subdirectory-f28-install-repo-.json", "failed_command": null, "variants": [ { "feature_id": "F28.install-repo-path", "family": "F28", "result": "pass", "label": "Install a subdirectory (`F28.install-repo-path`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-04-install-a-subdirectory-f28-install-repo-.json", "failed_command": null } ] }, { "feature_id": "F28.list-discovery", "family": "F28", "result": "pass", "label": "Discovery by listing (`F28.list-discovery`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-10-discovery-by-listing-f28-list-discovery.json", "failed_command": null, "variants": [ { "feature_id": "F28.list-discovery", "family": "F28", "result": "pass", "label": "Discovery by listing (`F28.list-discovery`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-10-discovery-by-listing-f28-list-discovery.json", "failed_command": null } ] }, { "feature_id": "F28.prepare-corrupt-archive", "family": "F28", "result": "xfail", "label": "Non-gzip artifact (`F28.prepare-corrupt-archive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-25-non-gzip-artifact-f28-prepare-corrupt-ar.json", "failed_command": "control-agent-server api POST /api/canvas-extensions/installed/qa-f28-notgz/backend/prepare --json \"{\\\"revision\\\": \\\"$NREV\\\"}\" --expect 409 --save F28.prepare-corrupt-archive/prepare", "variants": [ { "feature_id": "F28.prepare-corrupt-archive", "family": "F28", "result": "xfail", "label": "Non-gzip artifact (`F28.prepare-corrupt-archive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-25-non-gzip-artifact-f28-prepare-corrupt-ar.json", "failed_command": "control-agent-server api POST /api/canvas-extensions/installed/qa-f28-notgz/backend/prepare --json \"{\\\"revision\\\": \\\"$NREV\\\"}\" --expect 409 --save F28.prepare-corrupt-archive/prepare" } ] }, { "feature_id": "F28.restart-persistence", "family": "F28", "result": "fail", "label": "Persistence across a restart (`F28.restart-persistence`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-23-persistence-across-a-restart-f28-restart.json", "failed_command": "control-agent-server api GET /api/canvas-extensions/installed --check canvas_extensions len-eq 6 --quiet", "variants": [ { "feature_id": "F28.restart-persistence", "family": "F28", "result": "fail", "label": "Persistence across a restart (`F28.restart-persistence`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-23-persistence-across-a-restart-f28-restart.json", "failed_command": "control-agent-server api GET /api/canvas-extensions/installed --check canvas_extensions len-eq 6 --quiet" } ] }, { "feature_id": "F28.status-exit-hang", "family": "F28", "result": "fail", "label": "Status after the leader exits (`F28.status-exit-hang`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-28-status-after-the-leader-exits-f28-status.json", "failed_command": "false", "variants": [ { "feature_id": "F28.status-exit-hang", "family": "F28", "result": "fail", "label": "Status after the leader exits (`F28.status-exit-hang`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-28-status-after-the-leader-exits-f28-status.json", "failed_command": "false" } ] }, { "feature_id": "F28.status-probe-latch", "family": "F28", "result": "xfail", "label": "Health probe latch (`F28.status-probe-latch`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-27-health-probe-latch-f28-status-probe-latc.json", "failed_command": "control-agent-server api GET /api/canvas-extensions/installed/qa-f28-flaky/backend --until-ok 5 --check state eq ready --save F28.status-probe-latch/after-recovery", "variants": [ { "feature_id": "F28.status-probe-latch", "family": "F28", "result": "xfail", "label": "Health probe latch (`F28.status-probe-latch`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-27-health-probe-latch-f28-status-probe-latc.json", "failed_command": "control-agent-server api GET /api/canvas-extensions/installed/qa-f28-flaky/backend --until-ok 5 --check state eq ready --save F28.status-probe-latch/after-recovery" } ] }, { "feature_id": "F28.ts-client", "family": "F28", "result": "pass", "label": "TypeScript client (`F28.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-19-typescript-client-f28-ts-client.json", "failed_command": null, "variants": [ { "feature_id": "F28.ts-client", "family": "F28", "result": "pass", "label": "TypeScript client (`F28.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-19-typescript-client-f28-ts-client.json", "failed_command": null } ] }, { "feature_id": "F28.uninstall", "family": "F28", "result": "fail", "label": "Uninstall (`F28.uninstall`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-24-uninstall-f28-uninstall.json", "failed_command": "control-agent-server api GET /api/canvas-extensions/installed --check canvas_extensions len-eq 5 --check canvas_extensions not-contains qa-backend --quiet", "variants": [ { "feature_id": "F28.uninstall", "family": "F28", "result": "fail", "label": "Uninstall (`F28.uninstall`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-24-uninstall-f28-uninstall.json", "failed_command": "control-agent-server api GET /api/canvas-extensions/installed --check canvas_extensions len-eq 5 --check canvas_extensions not-contains qa-backend --quiet" } ] }, { "feature_id": "F28.unknown-name", "family": "F28", "result": "pass", "label": "Unknown and invalid names (`F28.unknown-name`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-09-unknown-and-invalid-names-f28-unknown-na.json", "failed_command": null, "variants": [ { "feature_id": "F28.unknown-name", "family": "F28", "result": "pass", "label": "Unknown and invalid names (`F28.unknown-name`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-09-unknown-and-invalid-names-f28-unknown-na.json", "failed_command": null } ] }, { "feature_id": "F29.backend-crash-detected", "family": "F29", "result": "xfail", "label": "Crash seen by the bridge (`F29.backend-crash-detected`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-24-crash-seen-by-the-bridge-f29-backend-cra.json", "failed_command": "control-agent-server api GET /echo --auth none --cookie \"oh_app_backend_session=$CC\" --until-ok 15 --expect 503 --check exception eq '503: Canvas App backend is not ready' --save F29.backend-crash-detected/crashed", "variants": [ { "feature_id": "F29.backend-crash-detected", "family": "F29", "result": "xfail", "label": "Crash seen by the bridge (`F29.backend-crash-detected`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-24-crash-seen-by-the-bridge-f29-backend-cra.json", "failed_command": "control-agent-server api GET /echo --auth none --cookie \"oh_app_backend_session=$CC\" --until-ok 15 --expect 503 --check exception eq '503: Canvas App backend is not ready' --save F29.backend-crash-detected/crashed" } ] }, { "feature_id": "F29.backend-lifecycle-revokes", "family": "F29", "result": "pass", "label": "Backend stop and disable revoke (`F29.backend-lifecycle-revokes`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-23-backend-stop-and-disable-revoke-f29-back.json", "failed_command": null, "variants": [ { "feature_id": "F29.backend-lifecycle-revokes", "family": "F29", "result": "pass", "label": "Backend stop and disable revoke (`F29.backend-lifecycle-revokes`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-23-backend-stop-and-disable-revoke-f29-back.json", "failed_command": null } ] }, { "feature_id": "F29.ingress-advertised", "family": "F29", "result": "pass", "label": "Ingress advertised (`F29.ingress-advertised`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-01-ingress-advertised-f29-ingress-advertise.json", "failed_command": null, "variants": [ { "feature_id": "F29.ingress-advertised", "family": "F29", "result": "pass", "label": "Ingress advertised (`F29.ingress-advertised`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-01-ingress-advertised-f29-ingress-advertise.json", "failed_command": null } ] }, { "feature_id": "F29.ingress-config-errors", "family": "F29", "result": "pass", "label": "Ingress not configured or unusable (`F29.ingress-config-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-02-ingress-not-configured-or-unusable-f29-i.json", "failed_command": null, "variants": [ { "feature_id": "F29.ingress-config-errors", "family": "F29", "result": "pass", "label": "Ingress not configured or unusable (`F29.ingress-config-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-02-ingress-not-configured-or-unusable-f29-i.json", "failed_command": null } ] }, { "feature_id": "F29.proxy-cookie-required", "family": "F29", "result": "pass", "label": "Cookie required (`F29.proxy-cookie-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-10-cookie-required-f29-proxy-cookie-require.json", "failed_command": null, "variants": [ { "feature_id": "F29.proxy-cookie-required", "family": "F29", "result": "pass", "label": "Cookie required (`F29.proxy-cookie-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-10-cookie-required-f29-proxy-cookie-require.json", "failed_command": null } ] }, { "feature_id": "F29.proxy-credentials-stripped", "family": "F29", "result": "pass", "label": "Credentials stripped (`F29.proxy-credentials-stripped`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-14-credentials-stripped-f29-proxy-credentia.json", "failed_command": null, "variants": [ { "feature_id": "F29.proxy-credentials-stripped", "family": "F29", "result": "pass", "label": "Credentials stripped (`F29.proxy-credentials-stripped`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-14-credentials-stripped-f29-proxy-credentia.json", "failed_command": null } ] }, { "feature_id": "F29.proxy-get", "family": "F29", "result": "pass", "label": "Proxied safe methods (`F29.proxy-get`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-09-proxied-safe-methods-f29-proxy-get.json", "failed_command": null, "variants": [ { "feature_id": "F29.proxy-get", "family": "F29", "result": "pass", "label": "Proxied safe methods (`F29.proxy-get`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-09-proxied-safe-methods-f29-proxy-get.json", "failed_command": null } ] }, { "feature_id": "F29.proxy-path-traversal", "family": "F29", "result": "pass", "label": "Path traversal (`F29.proxy-path-traversal`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-18-path-traversal-f29-proxy-path-traversal.json", "failed_command": null, "variants": [ { "feature_id": "F29.proxy-path-traversal", "family": "F29", "result": "pass", "label": "Path traversal (`F29.proxy-path-traversal`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-18-path-traversal-f29-proxy-path-traversal.json", "failed_command": null } ] }, { "feature_id": "F29.proxy-request-framing", "family": "F29", "result": "xfail", "label": "Request framing (`F29.proxy-request-framing`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-15-request-framing-f29-proxy-request-framin.json", "failed_command": "control-agent-server api GET /app-backends/qa-headers/echo --auth none --cookie \"oh_app_backend_session=$FR\" --expect 200 --check method eq GET --check body eq '' --check headers.transfer-encoding missing --save F29.proxy-request-framing/get", "variants": [ { "feature_id": "F29.proxy-request-framing", "family": "F29", "result": "xfail", "label": "Request framing (`F29.proxy-request-framing`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-15-request-framing-f29-proxy-request-framin.json", "failed_command": "control-agent-server api GET /app-backends/qa-headers/echo --auth none --cookie \"oh_app_backend_session=$FR\" --expect 200 --check method eq GET --check body eq '' --check headers.transfer-encoding missing --save F29.proxy-request-framing/get" } ] }, { "feature_id": "F29.proxy-response-filter", "family": "F29", "result": "pass", "label": "Response filter (`F29.proxy-response-filter`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-16-response-filter-f29-proxy-response-filte.json", "failed_command": null, "variants": [ { "feature_id": "F29.proxy-response-filter", "family": "F29", "result": "pass", "label": "Response filter (`F29.proxy-response-filter`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-16-response-filter-f29-proxy-response-filte.json", "failed_command": null } ] }, { "feature_id": "F29.proxy-response-headers", "family": "F29", "result": "xfail", "label": "One Date and Server header (`F29.proxy-response-headers`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-17-one-date-and-server-header-f29-proxy-res.json", "failed_command": "control-agent-server api GET /echo --auth none --cookie \"oh_app_backend_session=$RH\" --expect 200 --check-header date matches '^[A-Z][a-z]{2}, [0-9]{2} [A-Z][a-z]{2} [0-9]{4} [0-9:]{8} GMT$' --check-header server not-contains ', ' --all-headers --save F29.proxy-response-headers/echo", "variants": [ { "feature_id": "F29.proxy-response-headers", "family": "F29", "result": "xfail", "label": "One Date and Server header (`F29.proxy-response-headers`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-17-one-date-and-server-header-f29-proxy-res.json", "failed_command": "control-agent-server api GET /echo --auth none --cookie \"oh_app_backend_session=$RH\" --expect 200 --check-header date matches '^[A-Z][a-z]{2}, [0-9]{2} [A-Z][a-z]{2} [0-9]{4} [0-9:]{8} GMT$' --check-header server not-contains ', ' --all-headers --save F29.proxy-response-headers/echo" } ] }, { "feature_id": "F29.proxy-root", "family": "F29", "result": "pass", "label": "App root (`F29.proxy-root`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-12-app-root-f29-proxy-root.json", "failed_command": null, "variants": [ { "feature_id": "F29.proxy-root", "family": "F29", "result": "pass", "label": "App root (`F29.proxy-root`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-12-app-root-f29-proxy-root.json", "failed_command": null } ] }, { "feature_id": "F29.proxy-root-path-query", "family": "F29", "result": "xfail", "label": "Root `path` parameter (`F29.proxy-root-path-query`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-13-root-path-parameter-f29-proxy-root-path-.json", "failed_command": "control-agent-server api GET --query path=echo --auth none --cookie \"oh_app_backend_session=$RP\" --expect 200 --check path eq / --check query.path eq echo --save F29.proxy-root-path-query/get", "variants": [ { "feature_id": "F29.proxy-root-path-query", "family": "F29", "result": "xfail", "label": "Root `path` parameter (`F29.proxy-root-path-query`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-13-root-path-parameter-f29-proxy-root-path-.json", "failed_command": "control-agent-server api GET --query path=echo --auth none --cookie \"oh_app_backend_session=$RP\" --expect 200 --check path eq / --check query.path eq echo --save F29.proxy-root-path-query/get" } ] }, { "feature_id": "F29.proxy-unsafe-methods", "family": "F29", "result": "pass", "label": "Unsafe methods (`F29.proxy-unsafe-methods`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-11-unsafe-methods-f29-proxy-unsafe-methods.json", "failed_command": null, "variants": [ { "feature_id": "F29.proxy-unsafe-methods", "family": "F29", "result": "pass", "label": "Unsafe methods (`F29.proxy-unsafe-methods`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-11-unsafe-methods-f29-proxy-unsafe-methods.json", "failed_command": null } ] }, { "feature_id": "F29.session-backend-not-ready", "family": "F29", "result": "pass", "label": "Backend not ready (`F29.session-backend-not-ready`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-08-backend-not-ready-f29-session-backend-no.json", "failed_command": null, "variants": [ { "feature_id": "F29.session-backend-not-ready", "family": "F29", "result": "pass", "label": "Backend not ready (`F29.session-backend-not-ready`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-08-backend-not-ready-f29-session-backend-no.json", "failed_command": null } ] }, { "feature_id": "F29.session-bootstrap", "family": "F29", "result": "pass", "label": "Session bootstrap (`F29.session-bootstrap`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-03-session-bootstrap-f29-session-bootstrap.json", "failed_command": null, "variants": [ { "feature_id": "F29.session-bootstrap", "family": "F29", "result": "pass", "label": "Session bootstrap (`F29.session-bootstrap`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-03-session-bootstrap-f29-session-bootstrap.json", "failed_command": null } ] }, { "feature_id": "F29.session-configured-control-origin", "family": "F29", "result": "pass", "label": "Configured control origins (`F29.session-configured-control-origin`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-25-configured-control-origins-f29-session-c.json", "failed_command": null, "variants": [ { "feature_id": "F29.session-configured-control-origin", "family": "F29", "result": "pass", "label": "Configured control origins (`F29.session-configured-control-origin`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-25-configured-control-origins-f29-session-c.json", "failed_command": null } ] }, { "feature_id": "F29.session-forwarded-proto", "family": "F29", "result": "xfail", "label": "Forwarded proto (`F29.session-forwarded-proto`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-06-forwarded-proto-f29-session-forwarded-pr.json", "failed_command": "control-agent-server api POST /session --header 'Origin: http://localhost:3000' --header 'X-Forwarded-Proto: https' --expect 200 --check ingress_url eq \"$S/\" --save F29.session-forwarded-proto/session", "variants": [ { "feature_id": "F29.session-forwarded-proto", "family": "F29", "result": "xfail", "label": "Forwarded proto (`F29.session-forwarded-proto`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-06-forwarded-proto-f29-session-forwarded-pr.json", "failed_command": "control-agent-server api POST /session --header 'Origin: http://localhost:3000' --header 'X-Forwarded-Proto: https' --expect 200 --check ingress_url eq \"$S/\" --save F29.session-forwarded-proto/session" } ] }, { "feature_id": "F29.session-ingress-host", "family": "F29", "result": "pass", "label": "Ingress host (`F29.session-ingress-host`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-05-ingress-host-f29-session-ingress-host.json", "failed_command": null, "variants": [ { "feature_id": "F29.session-ingress-host", "family": "F29", "result": "pass", "label": "Ingress host (`F29.session-ingress-host`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-05-ingress-host-f29-session-ingress-host.json", "failed_command": null } ] }, { "feature_id": "F29.session-key-required", "family": "F29", "result": "pass", "label": "Session key required (`F29.session-key-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-04-session-key-required-f29-session-key-req.json", "failed_command": null, "variants": [ { "feature_id": "F29.session-key-required", "family": "F29", "result": "pass", "label": "Session key required (`F29.session-key-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-04-session-key-required-f29-session-key-req.json", "failed_command": null } ] }, { "feature_id": "F29.session-origin-rules", "family": "F29", "result": "pass", "label": "Origin rules (`F29.session-origin-rules`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-07-origin-rules-f29-session-origin-rules.json", "failed_command": null, "variants": [ { "feature_id": "F29.session-origin-rules", "family": "F29", "result": "pass", "label": "Origin rules (`F29.session-origin-rules`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-07-origin-rules-f29-session-origin-rules.json", "failed_command": null } ] }, { "feature_id": "F29.session-revoke", "family": "F29", "result": "pass", "label": "Revoke a session (`F29.session-revoke`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-22-revoke-a-session-f29-session-revoke.json", "failed_command": null, "variants": [ { "feature_id": "F29.session-revoke", "family": "F29", "result": "pass", "label": "Revoke a session (`F29.session-revoke`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-22-revoke-a-session-f29-session-revoke.json", "failed_command": null } ] }, { "feature_id": "F29.session-ttl", "family": "F29", "result": "pass", "label": "Session expiry (`F29.session-ttl`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-28-session-expiry-f29-session-ttl.json", "failed_command": null, "variants": [ { "feature_id": "F29.session-ttl", "family": "F29", "result": "pass", "label": "Session expiry (`F29.session-ttl`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-28-session-expiry-f29-session-ttl.json", "failed_command": null } ] }, { "feature_id": "F29.sessions-not-persisted", "family": "F29", "result": "pass", "label": "Server restart (`F29.sessions-not-persisted`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-27-server-restart-f29-sessions-not-persiste.json", "failed_command": null, "variants": [ { "feature_id": "F29.sessions-not-persisted", "family": "F29", "result": "pass", "label": "Server restart (`F29.sessions-not-persisted`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-27-server-restart-f29-sessions-not-persiste.json", "failed_command": null } ] }, { "feature_id": "F29.trusted-forwarded-headers", "family": "F29", "result": "pass", "label": "Trusted forwarded headers (`F29.trusted-forwarded-headers`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-26-trusted-forwarded-headers-f29-trusted-fo.json", "failed_command": null, "variants": [ { "feature_id": "F29.trusted-forwarded-headers", "family": "F29", "result": "pass", "label": "Trusted forwarded headers (`F29.trusted-forwarded-headers`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-26-trusted-forwarded-headers-f29-trusted-fo.json", "failed_command": null } ] }, { "feature_id": "F29.uninstall-revokes", "family": "F29", "result": "pass", "label": "Uninstall (`F29.uninstall-revokes`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-29-uninstall-f29-uninstall-revokes.json", "failed_command": null, "variants": [ { "feature_id": "F29.uninstall-revokes", "family": "F29", "result": "pass", "label": "Uninstall (`F29.uninstall-revokes`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-29-uninstall-f29-uninstall-revokes.json", "failed_command": null } ] }, { "feature_id": "F29.ws-bridge", "family": "F29", "result": "pass", "label": "WebSocket bridge (`F29.ws-bridge`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-19-websocket-bridge-f29-ws-bridge.json", "failed_command": null, "variants": [ { "feature_id": "F29.ws-bridge", "family": "F29", "result": "pass", "label": "WebSocket bridge (`F29.ws-bridge`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-19-websocket-bridge-f29-ws-bridge.json", "failed_command": null } ] }, { "feature_id": "F29.ws-handshake-rejections", "family": "F29", "result": "pass", "label": "Handshake rejections (`F29.ws-handshake-rejections`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-20-handshake-rejections-f29-ws-handshake-re.json", "failed_command": null, "variants": [ { "feature_id": "F29.ws-handshake-rejections", "family": "F29", "result": "pass", "label": "Handshake rejections (`F29.ws-handshake-rejections`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-20-handshake-rejections-f29-ws-handshake-re.json", "failed_command": null } ] }, { "feature_id": "F29.ws-subprotocol", "family": "F29", "result": "xfail", "label": "Subprotocol (`F29.ws-subprotocol`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-21-subprotocol-f29-ws-subprotocol-known-bug.json", "failed_command": "control-agent-server exec --expect-output 'subprotocol=qa.v1' --expect-output '\"protocol\": \"qa.v1\"' --save F29.ws-subprotocol/bridged -- \"$PWD/.venv/bin/python\" -c 'import sys; from websockets.sync.client import connect; ws = connect(sys.argv[1], subprotocols=[\"qa.v1\"], origin=sys.argv[2], additional_headers={\"Cookie\": \"oh_app_backend_session=\" + sys.argv[3]}, proxy=None); print(\"subprotocol=\" + str(ws.subprotocol)); ws.send(\"qa-f29-proto\"); print(ws.recv(timeout=10)); ws.close()' \"ws://127.0.0.", "variants": [ { "feature_id": "F29.ws-subprotocol", "family": "F29", "result": "xfail", "label": "Subprotocol (`F29.ws-subprotocol`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-21-subprotocol-f29-ws-subprotocol-known-bug.json", "failed_command": "control-agent-server exec --expect-output 'subprotocol=qa.v1' --expect-output '\"protocol\": \"qa.v1\"' --save F29.ws-subprotocol/bridged -- \"$PWD/.venv/bin/python\" -c 'import sys; from websockets.sync.client import connect; ws = connect(sys.argv[1], subprotocols=[\"qa.v1\"], origin=sys.argv[2], additional_headers={\"Cookie\": \"oh_app_backend_session=\" + sys.argv[3]}, proxy=None); print(\"subprotocol=\" + str(ws.subprotocol)); ws.send(\"qa-f29-proto\"); print(ws.recv(timeout=10)); ws.close()' \"ws://127.0.0." } ] }, { "feature_id": "F30.auth-bearer-or-header", "family": "F30", "result": "pass", "label": "Bearer or session header (`F30.auth-bearer-or-header`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-02-bearer-or-session-header-f30-auth-bearer.json", "failed_command": null, "variants": [ { "feature_id": "F30.auth-bearer-or-header", "family": "F30", "result": "pass", "label": "Bearer or session header (`F30.auth-bearer-or-header`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-02-bearer-or-session-header-f30-auth-bearer.json", "failed_command": null } ] }, { "feature_id": "F30.auth-open-without-keys", "family": "F30", "result": "pass", "label": "No session keys (`F30.auth-open-without-keys`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-03-no-session-keys-f30-auth-open-without-ke.json", "failed_command": null, "variants": [ { "feature_id": "F30.auth-open-without-keys", "family": "F30", "result": "pass", "label": "No session keys (`F30.auth-open-without-keys`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-03-no-session-keys-f30-auth-open-without-ke.json", "failed_command": null } ] }, { "feature_id": "F30.chat-agent-tools", "family": "F30", "result": "pass", "label": "The agent runs its own tools (`F30.chat-agent-tools`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-14-the-agent-runs-its-own-tools-f30-chat-ag.json", "failed_command": null, "variants": [ { "feature_id": "F30.chat-agent-tools", "family": "F30", "result": "pass", "label": "The agent runs its own tools (`F30.chat-agent-tools`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-14-the-agent-runs-its-own-tools-f30-chat-ag.json", "failed_command": null } ] }, { "feature_id": "F30.chat-basic", "family": "F30", "result": "pass", "label": "Chat completion (`F30.chat-basic`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-04-chat-completion-f30-chat-basic.json", "failed_command": null, "variants": [ { "feature_id": "F30.chat-basic", "family": "F30", "result": "pass", "label": "Chat completion (`F30.chat-basic`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-04-chat-completion-f30-chat-basic.json", "failed_command": null } ] }, { "feature_id": "F30.chat-confirmation-409", "family": "F30", "result": "pass", "label": "Confirmation stops a new conversation (`F30.chat-confirmation-409`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-18-confirmation-stops-a-new-conversation-f3.json", "failed_command": null, "variants": [ { "feature_id": "F30.chat-confirmation-409", "family": "F30", "result": "pass", "label": "Confirmation stops a new conversation (`F30.chat-confirmation-409`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-18-confirmation-stops-a-new-conversation-f3.json", "failed_command": null } ] }, { "feature_id": "F30.chat-header-new-id", "family": "F30", "result": "pass", "label": "Caller-chosen id (`F30.chat-header-new-id`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-10-caller-chosen-id-f30-chat-header-new-id.json", "failed_command": null, "variants": [ { "feature_id": "F30.chat-header-new-id", "family": "F30", "result": "pass", "label": "Caller-chosen id (`F30.chat-header-new-id`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-10-caller-chosen-id-f30-chat-header-new-id.json", "failed_command": null } ] }, { "feature_id": "F30.chat-input-errors", "family": "F30", "result": "pass", "label": "Chat input errors (`F30.chat-input-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-15-chat-input-errors-f30-chat-input-errors.json", "failed_command": null, "variants": [ { "feature_id": "F30.chat-input-errors", "family": "F30", "result": "pass", "label": "Chat input errors (`F30.chat-input-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-15-chat-input-errors-f30-chat-input-errors.json", "failed_command": null } ] }, { "feature_id": "F30.chat-latest-user-only", "family": "F30", "result": "pass", "label": "Only the latest user turn (`F30.chat-latest-user-only`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-06-only-the-latest-user-turn-f30-chat-lates.json", "failed_command": null, "variants": [ { "feature_id": "F30.chat-latest-user-only", "family": "F30", "result": "pass", "label": "Only the latest user turn (`F30.chat-latest-user-only`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-06-only-the-latest-user-turn-f30-chat-lates.json", "failed_command": null } ] }, { "feature_id": "F30.chat-reuse-after-restart", "family": "F30", "result": "pass", "label": "Continue after a restart (`F30.chat-reuse-after-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-09-continue-after-a-restart-f30-chat-reuse-.json", "failed_command": null, "variants": [ { "feature_id": "F30.chat-reuse-after-restart", "family": "F30", "result": "pass", "label": "Continue after a restart (`F30.chat-reuse-after-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-09-continue-after-a-restart-f30-chat-reuse-.json", "failed_command": null } ] }, { "feature_id": "F30.chat-reuse-concurrent", "family": "F30", "result": "xfail", "label": "Overlapping turns on one conversation (`F30.chat-reuse-concurrent`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-13-overlapping-turns-on-one-conversation-f3.json", "failed_command": "control-agent-server state cat fixtures/qa-f30/race-x.json --check choices.0.message.content eq QA_F30_RACE_B", "variants": [ { "feature_id": "F30.chat-reuse-concurrent", "family": "F30", "result": "xfail", "label": "Overlapping turns on one conversation (`F30.chat-reuse-concurrent`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-13-overlapping-turns-on-one-conversation-f3.json", "failed_command": "control-agent-server state cat fixtures/qa-f30/race-x.json --check choices.0.message.content eq QA_F30_RACE_B" } ] }, { "feature_id": "F30.chat-reuse-confirmation", "family": "F30", "result": "xfail", "label": "Confirmation on a continued conversation (`F30.chat-reuse-confirmation`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-19-confirmation-on-a-continued-conversation.json", "failed_command": "control-agent-server state cat fixtures/qa-f30/reconf-2.json --check status eq 409 --check response.body.detail eq 'Agent run ended with status: waiting_for_confirmation' --check elapsed_ms lt 30000", "variants": [ { "feature_id": "F30.chat-reuse-confirmation", "family": "F30", "result": "xfail", "label": "Confirmation on a continued conversation (`F30.chat-reuse-confirmation`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-19-confirmation-on-a-continued-conversation.json", "failed_command": "control-agent-server state cat fixtures/qa-f30/reconf-2.json --check status eq 409 --check response.body.detail eq 'Agent run ended with status: waiting_for_confirmation' --check elapsed_ms lt 30000" } ] }, { "feature_id": "F30.chat-reuse-header", "family": "F30", "result": "pass", "label": "Continue with the header (`F30.chat-reuse-header`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-08-continue-with-the-header-f30-chat-reuse-.json", "failed_command": null, "variants": [ { "feature_id": "F30.chat-reuse-header", "family": "F30", "result": "pass", "label": "Continue with the header (`F30.chat-reuse-header`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-08-continue-with-the-header-f30-chat-reuse-.json", "failed_command": null } ] }, { "feature_id": "F30.chat-reuse-keeps-llm", "family": "F30", "result": "pass", "label": "Continuing keeps the conversation's LLM (`F30.chat-reuse-keeps-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-11-continuing-keeps-the-conversation-s-llm-.json", "failed_command": null, "variants": [ { "feature_id": "F30.chat-reuse-keeps-llm", "family": "F30", "result": "pass", "label": "Continuing keeps the conversation's LLM (`F30.chat-reuse-keeps-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-11-continuing-keeps-the-conversation-s-llm-.json", "failed_command": null } ] }, { "feature_id": "F30.chat-reuse-run-limit-429", "family": "F30", "result": "xfail", "label": "Run limit on a continued conversation (`F30.chat-reuse-run-limit-429`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-21-run-limit-on-a-continued-conversation-f3.json", "failed_command": "control-agent-server api GET \"/$CID/events/search\" --query source=user --query body=QA_F30_429_REUSE --check items len-eq 0 --save F30.chat-reuse-run-limit-429/user-messages", "variants": [ { "feature_id": "F30.chat-reuse-run-limit-429", "family": "F30", "result": "xfail", "label": "Run limit on a continued conversation (`F30.chat-reuse-run-limit-429`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-21-run-limit-on-a-continued-conversation-f3.json", "failed_command": "control-agent-server api GET \"/$CID/events/search\" --query source=user --query body=QA_F30_429_REUSE --check items len-eq 0 --save F30.chat-reuse-run-limit-429/user-messages" } ] }, { "feature_id": "F30.chat-run-error-500", "family": "F30", "result": "pass", "label": "Run ends in error (`F30.chat-run-error-500`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-17-run-ends-in-error-f30-chat-run-error-500.json", "failed_command": null, "variants": [ { "feature_id": "F30.chat-run-error-500", "family": "F30", "result": "pass", "label": "Run ends in error (`F30.chat-run-error-500`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-17-run-ends-in-error-f30-chat-run-error-500.json", "failed_command": null } ] }, { "feature_id": "F30.chat-run-limit-429", "family": "F30", "result": "pass", "label": "Run limit (`F30.chat-run-limit-429`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-20-run-limit-f30-chat-run-limit-429.json", "failed_command": null, "variants": [ { "feature_id": "F30.chat-run-limit-429", "family": "F30", "result": "pass", "label": "Run limit (`F30.chat-run-limit-429`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-20-run-limit-f30-chat-run-limit-429.json", "failed_command": null } ] }, { "feature_id": "F30.chat-stream", "family": "F30", "result": "pass", "label": "Streaming (`F30.chat-stream`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-07-streaming-f30-chat-stream.json", "failed_command": null, "variants": [ { "feature_id": "F30.chat-stream", "family": "F30", "result": "pass", "label": "Streaming (`F30.chat-stream`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-07-streaming-f30-chat-stream.json", "failed_command": null } ] }, { "feature_id": "F30.chat-system-suffix", "family": "F30", "result": "pass", "label": "System text goes to the system prompt (`F30.chat-system-suffix`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-05-system-text-goes-to-the-system-prompt-f3.json", "failed_command": null, "variants": [ { "feature_id": "F30.chat-system-suffix", "family": "F30", "result": "pass", "label": "System text goes to the system prompt (`F30.chat-system-suffix`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-05-system-text-goes-to-the-system-prompt-f3.json", "failed_command": null } ] }, { "feature_id": "F30.chat-timeout-504", "family": "F30", "result": "not-run", "label": "a run longer than the fixed 120 s answers 504 `Agent run timed out`, and the run keeps going server-side.", "run": null, "transcript": null, "variants": [ { "feature_id": "F30.chat-timeout-504", "family": "F30", "result": "not-run", "label": "a run longer than the fixed 120 s answers 504 `Agent run timed out`, and the run keeps going server-side.", "run": null, "transcript": null } ] }, { "feature_id": "F30.chat-usage-per-request", "family": "F30", "result": "xfail", "label": "Usage per request (`F30.chat-usage-per-request`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-12-usage-per-request-f30-chat-usage-per-req.json", "failed_command": "control-agent-server state cat fixtures/qa-f30/usage-2.json --check usage.prompt_tokens eq 3 --check usage.completion_tokens eq 2 --check usage.total_tokens eq 5", "variants": [ { "feature_id": "F30.chat-usage-per-request", "family": "F30", "result": "xfail", "label": "Usage per request (`F30.chat-usage-per-request`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-12-usage-per-request-f30-chat-usage-per-req.json", "failed_command": "control-agent-server state cat fixtures/qa-f30/usage-2.json --check usage.prompt_tokens eq 3 --check usage.completion_tokens eq 2 --check usage.total_tokens eq 5" } ] }, { "feature_id": "F30.models-list", "family": "F30", "result": "pass", "label": "Model list (`F30.models-list`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-01-model-list-f30-models-list.json", "failed_command": null, "variants": [ { "feature_id": "F30.models-list", "family": "F30", "result": "pass", "label": "Model list (`F30.models-list`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-01-model-list-f30-models-list.json", "failed_command": null } ] }, { "feature_id": "F30.observability-headers", "family": "F30", "result": "pass", "label": "Observability headers (`F30.observability-headers`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-16-observability-headers-f30-observability-.json", "failed_command": null, "variants": [ { "feature_id": "F30.observability-headers", "family": "F30", "result": "pass", "label": "Observability headers (`F30.observability-headers`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-16-observability-headers-f30-observability-.json", "failed_command": null } ] }, { "feature_id": "F30.openai-sdk", "family": "F30", "result": "not-run", "label": "the official OpenAI Python SDK lists models, reads the conversation header through `with_raw_response`, continues the conversation with `extra_headers`, parses a stream and creates a response.", "run": null, "transcript": null, "variants": [ { "feature_id": "F30.openai-sdk", "family": "F30", "result": "not-run", "label": "the official OpenAI Python SDK lists models, reads the conversation header through `with_raw_response`, continues the conversation with `extra_headers`, parses a stream and creates a response.", "run": null, "transcript": null } ] }, { "feature_id": "F30.responses-basic", "family": "F30", "result": "not-run", "label": "`POST /v1/responses` on a real model returns a completed `response` with one `output_text` message equal to the agent's final answer, echoes `instructions` and `metadata`, and puts the instructions in the system suffix.", "run": null, "transcript": null, "variants": [ { "feature_id": "F30.responses-basic", "family": "F30", "result": "not-run", "label": "`POST /v1/responses` on a real model returns a completed `response` with one `output_text` message equal to the agent's final answer, echoes `instructions` and `metadata`, and puts the instructions in the system suffix.", "run": null, "transcript": null } ] }, { "feature_id": "F30.responses-errors", "family": "F30", "result": "not-run", "label": "`stream`, `store: true` and `previous_response_id` are 400 before anything else is checked, bad input is 400 or 422, an unknown model 404, and nothing is created.", "run": null, "transcript": null, "variants": [ { "feature_id": "F30.responses-errors", "family": "F30", "result": "not-run", "label": "`stream`, `store: true` and `previous_response_id` are 400 before anything else is checked, bad input is 400 or 422, an unknown model 404, and nothing is created.", "run": null, "transcript": null } ] }, { "feature_id": "F30.responses-stateless-replay", "family": "F30", "result": "not-run", "label": "every responses call is a new conversation (the conversation header is ignored); replayed items become one user message wrapped in `` markers, and developer items go to the system suffix.", "run": null, "transcript": null, "variants": [ { "feature_id": "F30.responses-stateless-replay", "family": "F30", "result": "not-run", "label": "every responses call is a new conversation (the conversation header is ignored); replayed items become one user message wrapped in `` markers, and developer items go to the system suffix.", "run": null, "transcript": null } ] }, { "feature_id": "F31.config-invalid-fails-fast", "family": "F31", "result": "not-run", "label": "a non-numeric `OH_MAX_CONCURRENT_RUNS` or an out-of-range limit in the configuration file stops the server during startup instead of serving with a default.", "run": null, "transcript": null, "variants": [ { "feature_id": "F31.config-invalid-fails-fast", "family": "F31", "result": "not-run", "label": "a non-numeric `OH_MAX_CONCURRENT_RUNS` or an out-of-range limit in the configuration file stops the server during startup instead of serving with a default.", "run": null, "transcript": null } ] }, { "feature_id": "F31.config-null-override", "family": "F31", "result": "not-run", "label": "`OH__IS_NONE=1` clears a value the configuration file sets, as a non-null `OH_*` value replaces it (seen through `app_backend_public_url` in `/server_info`).", "run": null, "transcript": null, "variants": [ { "feature_id": "F31.config-null-override", "family": "F31", "result": "not-run", "label": "`OH__IS_NONE=1` clears a value the configuration file sets, as a non-null `OH_*` value replaces it (seen through `app_backend_public_url` in `/server_info`).", "run": null, "transcript": null } ] }, { "feature_id": "F31.config-precedence", "family": "F31", "result": "not-run", "label": "the configuration file is applied and an `OH_*` variable overrides the same field (seen through the lease TTL).", "run": null, "transcript": null, "variants": [ { "feature_id": "F31.config-precedence", "family": "F31", "result": "not-run", "label": "the configuration file is applied and an `OH_*` variable overrides the same field (seen through the lease TTL).", "run": null, "transcript": null } ] }, { "feature_id": "F31.config-silent-env", "family": "F31", "result": "not-run", "label": "an `OH_*` boolean other than `1`/`true` is read as false (it overrides a `true` in the configuration file) and a `Literal` variable outside its choices falls back to the default; the server starts either way, while the same invalid choice in the file stops startup.", "run": null, "transcript": null, "variants": [ { "feature_id": "F31.config-silent-env", "family": "F31", "result": "not-run", "label": "an `OH_*` boolean other than `1`/`true` is read as false (it overrides a `true` in the configuration file) and a `Literal` variable outside its choices falls back to the default; the server starts either way, while the same invalid choice in the file stops startup.", "run": null, "transcript": null } ] }, { "feature_id": "F31.crash-tool-call", "family": "F31", "result": "pass", "label": "Crash during a tool call (`F31.crash-tool-call`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-05-crash-during-a-tool-call-f31-crash-tool-.json", "failed_command": null, "variants": [ { "feature_id": "F31.crash-tool-call", "family": "F31", "result": "pass", "label": "Crash during a tool call (`F31.crash-tool-call`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-05-crash-during-a-tool-call-f31-crash-tool-.json", "failed_command": null } ] }, { "feature_id": "F31.create-capacity", "family": "F31", "result": "not-run", "label": "with every run slot taken, `POST ` answers 429 and creates nothing, on disk or in the catalog; the same request succeeds once the slot is free.", "run": null, "transcript": null, "variants": [ { "feature_id": "F31.create-capacity", "family": "F31", "result": "not-run", "label": "with every run slot taken, `POST ` answers 429 and creates nothing, on disk or in the catalog; the same request succeeds once the slot is free.", "run": null, "transcript": null } ] }, { "feature_id": "F31.eviction-skips-busy", "family": "F31", "result": "not-run", "label": "the same pass keeps a conversation with an open events socket and a running one loaded; once the socket closes and the run is interrupted, the next pass unloads both.", "run": null, "transcript": null, "variants": [ { "feature_id": "F31.eviction-skips-busy", "family": "F31", "result": "not-run", "label": "the same pass keeps a conversation with an open events socket and a running one loaded; once the socket closes and the run is interrupted, the next pass unloads both.", "run": null, "transcript": null } ] }, { "feature_id": "F31.fenced-write-no-trace", "family": "F31", "result": "not-run", "label": "a write refused for lost ownership answers a 4xx and leaves no trace: the stale owner's own GET does not report the refused title.", "run": null, "transcript": null, "variants": [ { "feature_id": "F31.fenced-write-no-trace", "family": "F31", "result": "not-run", "label": "a write refused for lost ownership answers a 4xx and leaves no trace: the stale owner's own GET does not report the refused title.", "run": null, "transcript": null } ] }, { "feature_id": "F31.idle-eviction", "family": "F31", "result": "not-run", "label": "an idle conversation is unloaded at the first eviction pass after its TTL (lease deleted, `Evicted idle` logged), stays listed with its title and status, and loads again with the same events on the next access.", "run": null, "transcript": null, "variants": [ { "feature_id": "F31.idle-eviction", "family": "F31", "result": "not-run", "label": "an idle conversation is unloaded at the first eviction pass after its TTL (lease deleted, `Evicted idle` logged), stays listed with its title and status, and loads again with the same events on the next access.", "run": null, "transcript": null } ] }, { "feature_id": "F31.lease-claimed", "family": "F31", "result": "pass", "label": "Lease claimed on load (`F31.lease-claimed`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-01-lease-claimed-on-load-f31-lease-claimed.json", "failed_command": null, "variants": [ { "feature_id": "F31.lease-claimed", "family": "F31", "result": "pass", "label": "Lease claimed on load (`F31.lease-claimed`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-01-lease-claimed-on-load-f31-lease-claimed.json", "failed_command": null } ] }, { "feature_id": "F31.lease-dead-owner-takeover", "family": "F31", "result": "fail", "label": "Dead owner takeover (`F31.lease-dead-owner-takeover`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-04-dead-owner-takeover-f31-lease-dead-owner.json", "failed_command": "false", "variants": [ { "feature_id": "F31.lease-dead-owner-takeover", "family": "F31", "result": "fail", "label": "Dead owner takeover (`F31.lease-dead-owner-takeover`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-04-dead-owner-takeover-f31-lease-dead-owner.json", "failed_command": "false" } ] }, { "feature_id": "F31.lease-disabled", "family": "F31", "result": "not-run", "label": "with `lease_ttl_seconds` 0 no lease file is written and a second server loads and writes the same conversation concurrently.", "run": null, "transcript": null, "variants": [ { "feature_id": "F31.lease-disabled", "family": "F31", "result": "not-run", "label": "with `lease_ttl_seconds` 0 no lease file is written and a second server loads and writes the same conversation concurrently.", "run": null, "transcript": null } ] }, { "feature_id": "F31.lease-expiry-fencing", "family": "F31", "result": "not-run", "label": "an owner that stops renewing (frozen, pid alive) loses the lease once it expires, and after it resumes its writes are refused and never reach disk.", "run": null, "transcript": null, "variants": [ { "feature_id": "F31.lease-expiry-fencing", "family": "F31", "result": "not-run", "label": "an owner that stops renewing (frozen, pid alive) loses the lease once it expires, and after it resumes its writes are refused and never reach disk.", "run": null, "transcript": null } ] }, { "feature_id": "F31.lease-released-graceful", "family": "F31", "result": "pass", "label": "Graceful release (`F31.lease-released-graceful`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-03-graceful-release-f31-lease-released-grac.json", "failed_command": null, "variants": [ { "feature_id": "F31.lease-released-graceful", "family": "F31", "result": "pass", "label": "Graceful release (`F31.lease-released-graceful`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-03-graceful-release-f31-lease-released-grac.json", "failed_command": null } ] }, { "feature_id": "F31.lease-renewed", "family": "F31", "result": "pass", "label": "Lease renewed (`F31.lease-renewed`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-02-lease-renewed-f31-lease-renewed.json", "failed_command": null, "variants": [ { "feature_id": "F31.lease-renewed", "family": "F31", "result": "pass", "label": "Lease renewed (`F31.lease-renewed`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-02-lease-renewed-f31-lease-renewed.json", "failed_command": null } ] }, { "feature_id": "F31.no-plaintext-at-rest", "family": "F31", "result": "pass", "label": "Nothing secret in plaintext (`F31.no-plaintext-at-rest`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-07-nothing-secret-in-plaintext-f31-no-plain.json", "failed_command": null, "variants": [ { "feature_id": "F31.no-plaintext-at-rest", "family": "F31", "result": "pass", "label": "Nothing secret in plaintext (`F31.no-plaintext-at-rest`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-07-nothing-secret-in-plaintext-f31-no-plain.json", "failed_command": null } ] }, { "feature_id": "F31.second-instance-catalog", "family": "F31", "result": "pass", "label": "Second server reads the catalog (`F31.second-instance-catalog`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-08-second-server-reads-the-catalog-f31-seco.json", "failed_command": null, "variants": [ { "feature_id": "F31.second-instance-catalog", "family": "F31", "result": "pass", "label": "Second server reads the catalog (`F31.second-instance-catalog`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-08-second-server-reads-the-catalog-f31-seco.json", "failed_command": null } ] }, { "feature_id": "F31.second-instance-lease-held", "family": "F31", "result": "pass", "label": "Second server refused while the owner lives (`F31.second-instance-lease-held`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-09-second-server-refused-while-the-owner-li.json", "failed_command": null, "variants": [ { "feature_id": "F31.second-instance-lease-held", "family": "F31", "result": "pass", "label": "Second server refused while the owner lives (`F31.second-instance-lease-held`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-09-second-server-refused-while-the-owner-li.json", "failed_command": null } ] }, { "feature_id": "F31.second-instance-takeover", "family": "F31", "result": "fail", "label": "Takeover after the owner dies (`F31.second-instance-takeover`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-10-takeover-after-the-owner-dies-f31-second.json", "failed_command": "control-agent-server logs --run \"$B\" --grep 'Taking over conversation' --expect-min $((NB + 1))", "variants": [ { "feature_id": "F31.second-instance-takeover", "family": "F31", "result": "fail", "label": "Takeover after the owner dies (`F31.second-instance-takeover`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-10-takeover-after-the-owner-dies-f31-second.json", "failed_command": "control-agent-server logs --run \"$B\" --grep 'Taking over conversation' --expect-min $((NB + 1))" } ] }, { "feature_id": "F31.state-survives-restarts", "family": "F31", "result": "pass", "label": "State survives restarts (`F31.state-survives-restarts`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-06-state-survives-restarts-f31-state-surviv.json", "failed_command": null, "variants": [ { "feature_id": "F31.state-survives-restarts", "family": "F31", "result": "pass", "label": "State survives restarts (`F31.state-survives-restarts`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-06-state-survives-restarts-f31-state-surviv.json", "failed_command": null } ] }, { "feature_id": "F31.storage-retention", "family": "F31", "result": "not-run", "label": "with the Docker runtime, `conversation_storage_retention_days` archives a stopped conversation inactive past the limit (events readable, resume and proxied routes 410, runtime directory gone), and `conversation_storage_disk_budget` sheds a stopped workspace's gitignored directories while keeping its files (blocked here: no Docker daemon).", "run": null, "transcript": null, "variants": [ { "feature_id": "F31.storage-retention", "family": "F31", "result": "not-run", "label": "with the Docker runtime, `conversation_storage_retention_days` archives a stopped conversation inactive past the limit (events readable, resume and proxied routes 410, runtime directory gone), and `conversation_storage_disk_budget` sheds a stopped workspace's gitignored directories while keeping its files (blocked here: no Docker daemon).", "run": null, "transcript": null } ] }, { "feature_id": "F31.takeover-keeps-metadata", "family": "F31", "result": "not-run", "label": "a title the owner stored after the second server started survives the second server's takeover, in its GET and in `meta.json`.", "run": null, "transcript": null, "variants": [ { "feature_id": "F31.takeover-keeps-metadata", "family": "F31", "result": "not-run", "label": "a title the owner stored after the second server started survives the second server's takeover, in its GET and in `meta.json`.", "run": null, "transcript": null } ] }, { "feature_id": "F31.tmux-dir-cleanup", "family": "F31", "result": "not-run", "label": "a server that defaulted its own per-process `TMUX_TMPDIR` removes that directory when it stops.", "run": null, "transcript": null, "variants": [ { "feature_id": "F31.tmux-dir-cleanup", "family": "F31", "result": "not-run", "label": "a server that defaulted its own per-process `TMUX_TMPDIR` removes that directory when it stops.", "run": null, "transcript": null } ] }, { "feature_id": "F32.backpressure", "family": "F32", "result": "pass", "label": "Batch and queue limits (`F32.backpressure`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-12-batch-and-queue-limits-f32-backpressure.json", "failed_command": null, "variants": [ { "feature_id": "F32.backpressure", "family": "F32", "result": "pass", "label": "Batch and queue limits (`F32.backpressure`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-12-batch-and-queue-limits-f32-backpressure.json", "failed_command": null } ] }, { "feature_id": "F32.buffer-size", "family": "F32", "result": "pass", "label": "Custom headers, path prefix and buffer size (`F32.custom-headers`, `F32.buffer-size`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-07-custom-headers-path-prefix-and-buffer-si.json", "failed_command": null, "variants": [ { "feature_id": "F32.buffer-size", "family": "F32", "result": "pass", "label": "Custom headers, path prefix and buffer size (`F32.custom-headers`, `F32.buffer-size`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-07-custom-headers-path-prefix-and-buffer-si.json", "failed_command": null } ] }, { "feature_id": "F32.buffer-stranded", "family": "F32", "result": "xfail", "label": "Events stranded behind an in-flight batch (`F32.buffer-stranded`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-08-events-stranded-behind-an-in-flight-batc.json", "failed_command": "control-agent-server sink read --path \"/qa-f32-b/events/${BID//-/}\" --contains QA_F32_BUFFER --expect-min 1 --wait 10 --save F32.buffer-stranded/after-wait", "variants": [ { "feature_id": "F32.buffer-stranded", "family": "F32", "result": "xfail", "label": "Events stranded behind an in-flight batch (`F32.buffer-stranded`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-08-events-stranded-behind-an-in-flight-batc.json", "failed_command": "control-agent-server sink read --path \"/qa-f32-b/events/${BID//-/}\" --contains QA_F32_BUFFER --expect-min 1 --wait 10 --save F32.buffer-stranded/after-wait" } ] }, { "feature_id": "F32.conversation-created", "family": "F32", "result": "pass", "label": "Conversation created (`F32.conversation-created`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-03-conversation-created-f32-conversation-cr.json", "failed_command": null, "variants": [ { "feature_id": "F32.conversation-created", "family": "F32", "result": "pass", "label": "Conversation created (`F32.conversation-created`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-03-conversation-created-f32-conversation-cr.json", "failed_command": null } ] }, { "feature_id": "F32.conversation-lifecycle", "family": "F32", "result": "pass", "label": "Conversation lifecycle posts (`F32.conversation-lifecycle`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-05-conversation-lifecycle-posts-f32-convers.json", "failed_command": null, "variants": [ { "feature_id": "F32.conversation-lifecycle", "family": "F32", "result": "pass", "label": "Conversation lifecycle posts (`F32.conversation-lifecycle`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-05-conversation-lifecycle-posts-f32-convers.json", "failed_command": null } ] }, { "feature_id": "F32.custom-headers", "family": "F32", "result": "pass", "label": "Custom headers, path prefix and buffer size (`F32.custom-headers`, `F32.buffer-size`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-07-custom-headers-path-prefix-and-buffer-si.json", "failed_command": null, "variants": [ { "feature_id": "F32.custom-headers", "family": "F32", "result": "pass", "label": "Custom headers, path prefix and buffer size (`F32.custom-headers`, `F32.buffer-size`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-07-custom-headers-path-prefix-and-buffer-si.json", "failed_command": null } ] }, { "feature_id": "F32.deferred-init", "family": "F32", "result": "not-run", "label": "a dormant pod sends nothing until `POST /api/init` delivers `telemetry` and `webhooks`; then `server_started`, conversation telemetry and webhooks flow.", "run": null, "transcript": null, "variants": [ { "feature_id": "F32.deferred-init", "family": "F32", "result": "not-run", "label": "a dormant pod sends nothing until `POST /api/init` delivers `telemetry` and `webhooks`; then `server_started`, conversation telemetry and webhooks flow.", "run": null, "transcript": null } ] }, { "feature_id": "F32.deferred-init-flag", "family": "F32", "result": "not-run", "label": "events from an initialized deferred pod report `deferred_init: true` (reproduces a real mislabel; see Gotchas).", "run": null, "transcript": null, "variants": [ { "feature_id": "F32.deferred-init-flag", "family": "F32", "result": "not-run", "label": "events from an initialized deferred pod report `deferred_init: true` (reproduces a real mislabel; see Gotchas).", "run": null, "transcript": null } ] }, { "feature_id": "F32.env-config", "family": "F32", "result": "pass", "label": "Webhooks from the environment (`F32.env-config`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-16-webhooks-from-the-environment-f32-env-co.json", "failed_command": null, "variants": [ { "feature_id": "F32.env-config", "family": "F32", "result": "pass", "label": "Webhooks from the environment (`F32.env-config`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-16-webhooks-from-the-environment-f32-env-co.json", "failed_command": null } ] }, { "feature_id": "F32.events-full-state", "family": "F32", "result": "pass", "label": "Full-state snapshot on every attach (`F32.events-full-state`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-04-full-state-snapshot-on-every-attach-f32-.json", "failed_command": null, "variants": [ { "feature_id": "F32.events-full-state", "family": "F32", "result": "pass", "label": "Full-state snapshot on every attach (`F32.events-full-state`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-04-full-state-snapshot-on-every-attach-f32-.json", "failed_command": null } ] }, { "feature_id": "F32.events-no-deltas", "family": "F32", "result": "pass", "label": "No streaming deltas on webhooks (`F32.events-no-deltas`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-06-no-streaming-deltas-on-webhooks-f32-even.json", "failed_command": null, "variants": [ { "feature_id": "F32.events-no-deltas", "family": "F32", "result": "pass", "label": "No streaming deltas on webhooks (`F32.events-no-deltas`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-06-no-streaming-deltas-on-webhooks-f32-even.json", "failed_command": null } ] }, { "feature_id": "F32.events-post", "family": "F32", "result": "pass", "label": "A run's events reach the webhook (`F32.events-post`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-01-a-run-s-events-reach-the-webhook-f32-eve.json", "failed_command": null, "variants": [ { "feature_id": "F32.events-post", "family": "F32", "result": "pass", "label": "A run's events reach the webhook (`F32.events-post`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-01-a-run-s-events-reach-the-webhook-f32-eve.json", "failed_command": null } ] }, { "feature_id": "F32.events-session-key", "family": "F32", "result": "pass", "label": "Session key and content type on every request (`F32.events-session-key`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-02-session-key-and-content-type-on-every-re.json", "failed_command": null, "variants": [ { "feature_id": "F32.events-session-key", "family": "F32", "result": "pass", "label": "Session key and content type on every request (`F32.events-session-key`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-02-session-key-and-content-type-on-every-re.json", "failed_command": null } ] }, { "feature_id": "F32.flush-delay", "family": "F32", "result": "fail", "label": "Flush timer from the first event (`F32.flush-delay`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-10-flush-timer-from-the-first-event-f32-flu.json", "failed_command": "control-agent-server sink read --path \"/qa-f32-d/events/${TID//-/}\" --expect-max 0", "variants": [ { "feature_id": "F32.flush-delay", "family": "F32", "result": "fail", "label": "Flush timer from the first event (`F32.flush-delay`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-10-flush-timer-from-the-first-event-f32-flu.json", "failed_command": "control-agent-server sink read --path \"/qa-f32-d/events/${TID//-/}\" --expect-max 0" } ] }, { "feature_id": "F32.flush-on-close", "family": "F32", "result": "pass", "label": "Flush on delete and on shutdown (`F32.flush-on-close`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-09-flush-on-delete-and-on-shutdown-f32-flus.json", "failed_command": null, "variants": [ { "feature_id": "F32.flush-on-close", "family": "F32", "result": "pass", "label": "Flush on delete and on shutdown (`F32.flush-on-close`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-09-flush-on-delete-and-on-shutdown-f32-flus.json", "failed_command": null } ] }, { "feature_id": "F32.requeue", "family": "F32", "result": "pass", "label": "Failed batches are requeued (`F32.requeue`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-14-failed-batches-are-requeued-f32-requeue.json", "failed_command": null, "variants": [ { "feature_id": "F32.requeue", "family": "F32", "result": "pass", "label": "Failed batches are requeued (`F32.requeue`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-14-failed-batches-are-requeued-f32-requeue.json", "failed_command": null } ] }, { "feature_id": "F32.retry", "family": "F32", "result": "pass", "label": "Retries against a failing receiver (`F32.retry`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-13-retries-against-a-failing-receiver-f32-r.json", "failed_command": null, "variants": [ { "feature_id": "F32.retry", "family": "F32", "result": "pass", "label": "Retries against a failing receiver (`F32.retry`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-13-retries-against-a-failing-receiver-f32-r.json", "failed_command": null } ] }, { "feature_id": "F32.slow-receiver", "family": "F32", "result": "xfail", "label": "A slow receiver blocks the next run (`F32.slow-receiver`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-15-a-slow-receiver-blocks-the-next-run-f32-.json", "failed_command": "control-agent-server api POST \"/$WCID/run\" --expect 200 --save F32.slow-receiver/run-after-finished", "variants": [ { "feature_id": "F32.slow-receiver", "family": "F32", "result": "xfail", "label": "A slow receiver blocks the next run (`F32.slow-receiver`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-15-a-slow-receiver-blocks-the-next-run-f32-.json", "failed_command": "control-agent-server api POST \"/$WCID/run\" --expect 200 --save F32.slow-receiver/run-after-finished" } ] }, { "feature_id": "F32.telemetry-consent-settings", "family": "F32", "result": "not-run", "label": "granting or denying consent with `PATCH /api/settings` `misc_settings_diff` starts or stops delivery immediately, and persisted consent enables telemetry after a restart.", "run": null, "transcript": null, "variants": [ { "feature_id": "F32.telemetry-consent-settings", "family": "F32", "result": "not-run", "label": "granting or denying consent with `PATCH /api/settings` `misc_settings_diff` starts or stops delivery immediately, and persisted consent enables telemetry after a restart.", "run": null, "transcript": null } ] }, { "feature_id": "F32.telemetry-conversation", "family": "F32", "result": "not-run", "label": "a run emits `conversation_created` and `conversation_finished` under the request's `user_id`, with a pseudonymous `conversation_ref` and without the prompt or the conversation id.", "run": null, "transcript": null, "variants": [ { "feature_id": "F32.telemetry-conversation", "family": "F32", "result": "not-run", "label": "a run emits `conversation_created` and `conversation_finished` under the request's `user_id`, with a pseudonymous `conversation_ref` and without the prompt or the conversation id.", "run": null, "transcript": null } ] }, { "feature_id": "F32.telemetry-conversation-failed", "family": "F32", "result": "not-run", "label": "a run that ends in `error` emits `conversation_failed` (`terminal_status` `error`) and `conversation_error` with the error code as `error_class`, without the error's detail text.", "run": null, "transcript": null, "variants": [ { "feature_id": "F32.telemetry-conversation-failed", "family": "F32", "result": "not-run", "label": "a run that ends in `error` emits `conversation_failed` (`terminal_status` `error`) and `conversation_error` with the error code as `error_class`, without the error's detail text.", "run": null, "transcript": null } ] }, { "feature_id": "F32.telemetry-env-override", "family": "F32", "result": "not-run", "label": "`OH_TELEMETRY_CONSENT=denied` with `OH_TELEMETRY_CONSENT_MODE=override` beats a granted setting.", "run": null, "transcript": null, "variants": [ { "feature_id": "F32.telemetry-env-override", "family": "F32", "result": "not-run", "label": "`OH_TELEMETRY_CONSENT=denied` with `OH_TELEMETRY_CONSENT_MODE=override` beats a granted setting.", "run": null, "transcript": null } ] }, { "feature_id": "F32.telemetry-kill-switch", "family": "F32", "result": "not-run", "label": "`DO_NOT_TRACK=1` beats every consent source.", "run": null, "transcript": null, "variants": [ { "feature_id": "F32.telemetry-kill-switch", "family": "F32", "result": "not-run", "label": "`DO_NOT_TRACK=1` beats every consent source.", "run": null, "transcript": null } ] }, { "feature_id": "F32.telemetry-lifecycle", "family": "F32", "result": "not-run", "label": "with the `http` exporter and consent, `server_started` arrives at boot and `server_stopped` at a graceful stop, as `{\"schema_version\": 1, \"events\": [...]}` with the configured Bearer token.", "run": null, "transcript": null, "variants": [ { "feature_id": "F32.telemetry-lifecycle", "family": "F32", "result": "not-run", "label": "with the `http` exporter and consent, `server_started` arrives at boot and `server_stopped` at a graceful stop, as `{\"schema_version\": 1, \"events\": [...]}` with the configured Bearer token.", "run": null, "transcript": null } ] }, { "feature_id": "F32.telemetry-off-by-default", "family": "F32", "result": "fail", "label": "Telemetry is off by default (`F32.telemetry-off-by-default`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-17-telemetry-is-off-by-default-f32-telemetr.json", "failed_command": "control-agent-server conversation start --placeholder-agent --no-run --prompt QA_F32_OFF --body-json '{\"user_id\": \"qa-f32-off\"}' --no-autotitle", "variants": [ { "feature_id": "F32.telemetry-off-by-default", "family": "F32", "result": "fail", "label": "Telemetry is off by default (`F32.telemetry-off-by-default`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-17-telemetry-is-off-by-default-f32-telemetr.json", "failed_command": "control-agent-server conversation start --placeholder-agent --no-run --prompt QA_F32_OFF --body-json '{\"user_id\": \"qa-f32-off\"}' --no-autotitle" } ] }, { "feature_id": "F32.telemetry-request-failed", "family": "F32", "result": "not-run", "label": "an unhandled 500 emits `request_failed` with the route template (never the concrete path), the exception class and the response's `error_id`, attributed to `X-OpenHands-Telemetry-Distinct-Id`, without the exception message.", "run": null, "transcript": null, "variants": [ { "feature_id": "F32.telemetry-request-failed", "family": "F32", "result": "not-run", "label": "an unhandled 500 emits `request_failed` with the route template (never the concrete path), the exception class and the response's `error_id`, attributed to `X-OpenHands-Telemetry-Distinct-Id`, without the exception message.", "run": null, "transcript": null } ] }, { "feature_id": "F32.telemetry-salt-and-kind", "family": "F32", "result": "not-run", "label": "without a salt `conversation_ref` is keyed by `OH_SECRET_KEY` (the same ref across restarts); `OH_TELEMETRY_SALT` re-keys it (again the same across restarts), and `OH_TELEMETRY_DEPLOYMENT_KIND` tags every event.", "run": null, "transcript": null, "variants": [ { "feature_id": "F32.telemetry-salt-and-kind", "family": "F32", "result": "not-run", "label": "without a salt `conversation_ref` is keyed by `OH_SECRET_KEY` (the same ref across restarts); `OH_TELEMETRY_SALT` re-keys it (again the same across restarts), and `OH_TELEMETRY_DEPLOYMENT_KIND` tags every event.", "run": null, "transcript": null } ] }, { "feature_id": "F32.telemetry-salt-docs", "family": "F32", "result": "not-run", "label": "the `TelemetrySpec.salt` description published in the OpenAPI schema states the real fallback key (reproduces a documentation bug; see Gotchas).", "run": null, "transcript": null, "variants": [ { "feature_id": "F32.telemetry-salt-docs", "family": "F32", "result": "not-run", "label": "the `TelemetrySpec.salt` description published in the OpenAPI schema states the real fallback key (reproduces a documentation bug; see Gotchas).", "run": null, "transcript": null } ] }, { "feature_id": "F32.telemetry-usage-buckets", "family": "F32", "result": "not-run", "label": "`conversation_finished` reports token and cost buckets from the run's real usage (reproduces a real gap; see Gotchas).", "run": null, "transcript": null, "variants": [ { "feature_id": "F32.telemetry-usage-buckets", "family": "F32", "result": "not-run", "label": "`conversation_finished` reports token and cost buckets from the run's real usage (reproduces a real gap; see Gotchas).", "run": null, "transcript": null } ] }, { "feature_id": "F32.webhook-spec-docs", "family": "F32", "result": "xfail", "label": "Webhook field descriptions (`F32.webhook-spec-docs`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-11-webhook-field-descriptions-f32-webhook-s.json", "failed_command": "control-agent-server api GET /openapi.json --auth none --quiet --check components.schemas.WebhookSpec.properties.flush_delay.description not-contains 'Timer is reset on each new event'", "variants": [ { "feature_id": "F32.webhook-spec-docs", "family": "F32", "result": "xfail", "label": "Webhook field descriptions (`F32.webhook-spec-docs`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-11-webhook-field-descriptions-f32-webhook-s.json", "failed_command": "control-agent-server api GET /openapi.json --auth none --quiet --check components.schemas.WebhookSpec.properties.flush_delay.description not-contains 'Timer is reset on each new event'" } ] }, { "feature_id": "F33.async-workspace", "family": "F33", "result": "pass", "label": "Async workspace (`F33.async-workspace`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-07-async-workspace-f33-async-workspace.json", "failed_command": null, "variants": [ { "feature_id": "F33.async-workspace", "family": "F33", "result": "pass", "label": "Async workspace (`F33.async-workspace`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-07-async-workspace-f33-async-workspace.json", "failed_command": null } ] }, { "feature_id": "F33.client-tools", "family": "F33", "result": "pass", "label": "Client-defined tool (`F33.client-tools`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-15-client-defined-tool-f33-client-tools.json", "failed_command": null, "variants": [ { "feature_id": "F33.client-tools", "family": "F33", "result": "pass", "label": "Client-defined tool (`F33.client-tools`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-15-client-defined-tool-f33-client-tools.json", "failed_command": null } ] }, { "feature_id": "F33.completion-logs", "family": "F33", "result": "not-run", "label": "an agent LLM with `log_completions=True` gets one log file per model call in its `log_completions_folder`, written by the client from the `LLMCompletionLogEvent`s the server streams and stores (the server writes no file).", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.completion-logs", "family": "F33", "result": "not-run", "label": "an agent LLM with `log_completions=True` gets one log file per model call in its `log_completions_folder`, written by the client from the `LLMCompletionLogEvent`s the server streams and stores (the server writes no file).", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-ask-title", "family": "F33", "result": "not-run", "label": "`ask_agent()` answers without adding events, `generate_title()` returns a title within `max_length` without adding events, and `set_title()` changes the server's title.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-ask-title", "family": "F33", "result": "not-run", "label": "`ask_agent()` answers without adding events, `generate_title()` returns a title within `max_length` without adding events, and `set_title()` changes the server's title.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-attach", "family": "F33", "result": "not-run", "label": "a second program reattaches by id with `RemoteConversation.attach()` or `Conversation(conversation_id=...)` and replays the same history without creating anything; `attach()` on an unknown id raises 404 while `Conversation(conversation_id=)` creates that id.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-attach", "family": "F33", "result": "not-run", "label": "a second program reattaches by id with `RemoteConversation.attach()` or `Conversation(conversation_id=...)` and replays the same history without creating anything; `attach()` on an unknown id raises 404 while `Conversation(conversation_id=)` creates that id.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-attach-large", "family": "F33", "result": "not-run", "label": "attaching to a conversation with more than 100 stored events pages through the whole history: `state.events` holds the server's events in the server's order (plus the one unpersisted socket snapshot), and the first reads of `state.confirmation_policy`, `state.security_analyzer`, `state.hook_config` and `state.agent` equal the server's.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-attach-large", "family": "F33", "result": "not-run", "label": "attaching to a conversation with more than 100 stored events pages through the whole history: `state.events` holds the server's events in the server's order (plus the one unpersisted socket snapshot), and the first reads of `state.confirmation_policy`, `state.security_analyzer`, `state.hook_config` and `state.agent` equal the server's.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-callbacks", "family": "F33", "result": "pass", "label": "Send and run to completion (`F33.conversation-run`, `F33.conversation-callbacks`, `F33.conversation-stats`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-13-send-and-run-to-completion-f33-conversat.json", "failed_command": null, "variants": [ { "feature_id": "F33.conversation-callbacks", "family": "F33", "result": "pass", "label": "Send and run to completion (`F33.conversation-run`, `F33.conversation-callbacks`, `F33.conversation-stats`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-13-send-and-run-to-completion-f33-conversat.json", "failed_command": null } ] }, { "feature_id": "F33.conversation-close", "family": "F33", "result": "not-run", "label": "`close()` keeps the server conversation when `delete_on_close=False` and deletes it when the `Conversation(...)` factory's default `delete_on_close=True` applies.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-close", "family": "F33", "result": "not-run", "label": "`close()` keeps the server conversation when `delete_on_close=False` and deletes it when the `Conversation(...)` factory's default `delete_on_close=True` applies.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-confirmation", "family": "F33", "result": "not-run", "label": "`set_confirmation_policy(AlwaysConfirm())` makes `run()` return at `waiting_for_confirmation`; `reject_pending_actions()` records a `UserRejectObservation`, never runs the action and leaves the conversation `idle`.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-confirmation", "family": "F33", "result": "not-run", "label": "`set_confirmation_policy(AlwaysConfirm())` makes `run()` return at `waiting_for_confirmation`; `reject_pending_actions()` records a `UserRejectObservation`, never runs the action and leaves the conversation `idle`.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-create", "family": "F33", "result": "pass", "label": "Create a conversation without running it (`F33.conversation-create`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-08-create-a-conversation-without-running-it.json", "failed_command": null, "variants": [ { "feature_id": "F33.conversation-create", "family": "F33", "result": "pass", "label": "Create a conversation without running it (`F33.conversation-create`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-08-create-a-conversation-without-running-it.json", "failed_command": null } ] }, { "feature_id": "F33.conversation-create-profile", "family": "F33", "result": "pass", "label": "Create from a saved agent profile (`F33.conversation-create-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-10-create-from-a-saved-agent-profile-f33-co.json", "failed_command": null, "variants": [ { "feature_id": "F33.conversation-create-profile", "family": "F33", "result": "pass", "label": "Create from a saved agent profile (`F33.conversation-create-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-10-create-from-a-saved-agent-profile-f33-co.json", "failed_command": null } ] }, { "feature_id": "F33.conversation-create-request", "family": "F33", "result": "pass", "label": "Create from a request, twice (`F33.conversation-create-request`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-09-create-from-a-request-twice-f33-conversa.json", "failed_command": null, "variants": [ { "feature_id": "F33.conversation-create-request", "family": "F33", "result": "pass", "label": "Create from a request, twice (`F33.conversation-create-request`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-09-create-from-a-request-twice-f33-conversa.json", "failed_command": null } ] }, { "feature_id": "F33.conversation-fork", "family": "F33", "result": "not-run", "label": "`fork(title=..., tags=...)` returns a `RemoteConversation` for a new server conversation with the same history, the title and the tags; `fork(agent=...)` is refused client-side.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-fork", "family": "F33", "result": "not-run", "label": "`fork(title=..., tags=...)` returns a `RemoteConversation` for a new server conversation with the same history, the title and the tags; `fork(agent=...)` is refused client-side.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-fork-branch", "family": "F33", "result": "not-run", "label": "`fork(from_event_id=..., reset_metrics=False)` returns a fork whose history ends at that event (its HEAD) with the source's stats, and `navigate_to(None)` and `navigate_to(event_id)` move the fork's HEAD without changing its events.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-fork-branch", "family": "F33", "result": "not-run", "label": "`fork(from_event_id=..., reset_metrics=False)` returns a fork whose history ends at that event (its HEAD) with the source's stats, and `navigate_to(None)` and `navigate_to(event_id)` move the fork's HEAD without changing its events.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-interrupt", "family": "F33", "result": "not-run", "label": "`interrupt()` from another thread while a blocking `run()` waits on a model call makes `run()` return at once without raising, at `paused` with one `InterruptEvent` and no `PauseEvent`, and a later `run()` reaches `finished`.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-interrupt", "family": "F33", "result": "not-run", "label": "`interrupt()` from another thread while a blocking `run()` waits on a model call makes `run()` return at once without raising, at `paused` with one `InterruptEvent` and no `PauseEvent`, and a later `run()` reaches `finished`.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-pause", "family": "F33", "result": "not-run", "label": "`pause()` from another thread during a blocking `run()` lets the tool step in flight finish, makes `run()` return with `paused` and a `PauseEvent`, and a later `run()` resumes to `finished`.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-pause", "family": "F33", "result": "not-run", "label": "`pause()` from another thread during a blocking `run()` lets the tool step in flight finish, makes `run()` return with `paused` and a `PauseEvent`, and a later `run()` resumes to `finished`.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-plugins-tag", "family": "F33", "result": "not-run", "label": "`Conversation(plugins=[PluginSource(...)])` with inline URL credentials stores the `plugins` tag with the credentials masked, and the token reaches no server state.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-plugins-tag", "family": "F33", "result": "not-run", "label": "`Conversation(plugins=[PluginSource(...)])` with inline URL credentials stores the `plugins` tag with the credentials masked, and the token reaches no server state.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-reconnect", "family": "F33", "result": "not-run", "label": "a live `RemoteConversation` survives a server restart: its events socket reconnects, a new message runs to `finished`, and `state.events` again matches the server.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-reconnect", "family": "F33", "result": "not-run", "label": "a live `RemoteConversation` survives a server restart: its events socket reconnects, a new message runs to `finished`, and `state.events` again matches the server.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-reconnect-callbacks", "family": "F33", "result": "not-run", "label": "an event the server records while a live `RemoteConversation`'s events socket is down reaches its callbacks once the socket is back, not only `state.events`.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-reconnect-callbacks", "family": "F33", "result": "not-run", "label": "an event the server records while a live `RemoteConversation`'s events socket is down reaches its callbacks once the socket is back, not only `state.events`.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-restart", "family": "F33", "result": "not-run", "label": "after a restart a new program reattaches to a finished conversation, sees the same history, and continues it to `finished`.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-restart", "family": "F33", "result": "not-run", "label": "after a restart a new program reattaches to a finished conversation, sees the same history, and continues it to `finished`.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-run", "family": "F33", "result": "pass", "label": "Send and run to completion (`F33.conversation-run`, `F33.conversation-callbacks`, `F33.conversation-stats`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-13-send-and-run-to-completion-f33-conversat.json", "failed_command": null, "variants": [ { "feature_id": "F33.conversation-run", "family": "F33", "result": "pass", "label": "Send and run to completion (`F33.conversation-run`, `F33.conversation-callbacks`, `F33.conversation-stats`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-13-send-and-run-to-completion-f33-conversat.json", "failed_command": null } ] }, { "feature_id": "F33.conversation-run-error", "family": "F33", "result": "not-run", "label": "when the model provider rejects the key, `run()` raises `ConversationRunError` carrying the server's `ConversationErrorEvent` code, and the server reports `error`.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-run-error", "family": "F33", "result": "not-run", "label": "when the model provider rejects the key, `run()` raises `ConversationRunError` carrying the server's `ConversationErrorEvent` code, and the server reports `error`.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-run-join-long-tool", "family": "F33", "result": "not-run", "label": "a second `run()` also joins a run whose current tool call lasts longer than 30 s, instead of raising.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-run-join-long-tool", "family": "F33", "result": "not-run", "label": "a second `run()` also joins a run whose current tool call lasts longer than 30 s, instead of raising.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-run-timeout", "family": "F33", "result": "not-run", "label": "`run(timeout=...)` raises `ConversationRunError` wrapping a `TimeoutError` on time while the server keeps running, and a second `run()` joins the run in flight (the server's 409 is not an error) and returns at `finished`.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-run-timeout", "family": "F33", "result": "not-run", "label": "`run(timeout=...)` raises `ConversationRunError` wrapping a `TimeoutError` on time while the server keeps running, and a second `run()` joins the run in flight (the server's 409 is not an error) and returns at `finished`.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-secrets", "family": "F33", "result": "not-run", "label": "`update_secrets()` with a plain value and with a `LookupSecret` from `get_secrets()` makes both available to the agent's commands, masked in observations and not stored in plaintext.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-secrets", "family": "F33", "result": "not-run", "label": "`update_secrets()` with a plain value and with a `LookupSecret` from `get_secrets()` makes both available to the agent's commands, masked in observations and not stored in plaintext.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-state-reread", "family": "F33", "result": "not-run", "label": "reading `state.confirmation_policy`, `state.security_analyzer`, `state.agent` and `is_confirmation_mode_active` a second time gives the same answer as the first.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-state-reread", "family": "F33", "result": "not-run", "label": "reading `state.confirmation_policy`, `state.security_analyzer`, `state.agent` and `is_confirmation_mode_active` a second time gives the same answer as the first.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-stats", "family": "F33", "result": "pass", "label": "Send and run to completion (`F33.conversation-run`, `F33.conversation-callbacks`, `F33.conversation-stats`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-13-send-and-run-to-completion-f33-conversat.json", "failed_command": null, "variants": [ { "feature_id": "F33.conversation-stats", "family": "F33", "result": "pass", "label": "Send and run to completion (`F33.conversation-run`, `F33.conversation-callbacks`, `F33.conversation-stats`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-13-send-and-run-to-completion-f33-conversat.json", "failed_command": null } ] }, { "feature_id": "F33.conversation-stop-hook", "family": "F33", "result": "pass", "label": "A Stop hook that refuses the first stop (`F33.conversation-stop-hook`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-14-a-stop-hook-that-refuses-the-first-stop-.json", "failed_command": null, "variants": [ { "feature_id": "F33.conversation-stop-hook", "family": "F33", "result": "pass", "label": "A Stop hook that refuses the first stop (`F33.conversation-stop-hook`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-14-a-stop-hook-that-refuses-the-first-stop-.json", "failed_command": null } ] }, { "feature_id": "F33.conversation-streaming", "family": "F33", "result": "not-run", "label": "with a `stream=True` agent LLM, callbacks receive the run's `StreamingDeltaEvent` frames live and the server stores none of them.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-streaming", "family": "F33", "result": "not-run", "label": "with a `stream=True` agent LLM, callbacks receive the run's `StreamingDeltaEvent` frames live and the server stores none of them.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-streaming-state", "family": "F33", "result": "not-run", "label": "streamed deltas stay out of `state.events`, which keeps matching the server's stored events.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-streaming-state", "family": "F33", "result": "not-run", "label": "streamed deltas stay out of `state.events`, which keeps matching the server's stored events.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-stuck", "family": "F33", "result": "not-run", "label": "when the server's stuck detector stops a looping agent, `run()` raises `ConversationRunError` (\"got stuck\") at once and the server reports `stuck`.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-stuck", "family": "F33", "result": "not-run", "label": "when the server's stuck detector stops a looping agent, `run()` raises `ConversationRunError` (\"got stuck\") at once and the server reports `stuck`.", "run": null, "transcript": null } ] }, { "feature_id": "F33.conversation-token-callbacks", "family": "F33", "result": "not-run", "label": "`Conversation(token_callbacks=[...])` on a remote workspace calls the token callbacks with the run's streamed deltas.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.conversation-token-callbacks", "family": "F33", "result": "not-run", "label": "`Conversation(token_callbacks=[...])` on a remote workspace calls the token callbacks with the run's streamed deltas.", "run": null, "transcript": null } ] }, { "feature_id": "F33.custom-tool-module", "family": "F33", "result": "not-run", "label": "a tool registered in the client process is sent as `tool_module_qualnames`, imported by a server started with `OH_EXTRA_PYTHON_PATH`, and runs on the server when the agent calls it; a client-registered module the server cannot import is only a logged warning; reattaching from a process without the module raises `ImportError` naming the tool.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.custom-tool-module", "family": "F33", "result": "not-run", "label": "a tool registered in the client process is sent as `tool_module_qualnames`, imported by a server started with `OH_EXTRA_PYTHON_PATH`, and runs on the server when the agent calls it; a client-registered module the server cannot import is only a logged warning; reattaching from a process without the module raises `ImportError` naming the tool.", "run": null, "transcript": null } ] }, { "feature_id": "F33.custom-tool-module-unimportable", "family": "F33", "result": "not-run", "label": "a conversation whose agent names a tool from a module the server cannot import fails with a client error (a 4xx, or a recorded conversation error), never with a 500.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.custom-tool-module-unimportable", "family": "F33", "result": "not-run", "label": "a conversation whose agent names a tool from a module the server cannot import fails with a client error (a 4xx, or a recorded conversation error), never with a 500.", "run": null, "transcript": null } ] }, { "feature_id": "F33.workspace-background", "family": "F33", "result": "pass", "label": "Background command (`F33.workspace-background`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-03-background-command-f33-workspace-backgro.json", "failed_command": null, "variants": [ { "feature_id": "F33.workspace-background", "family": "F33", "result": "pass", "label": "Background command (`F33.workspace-background`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-03-background-command-f33-workspace-backgro.json", "failed_command": null } ] }, { "feature_id": "F33.workspace-bad-key", "family": "F33", "result": "pass", "label": "Wrong session key (`F33.workspace-bad-key`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-02-wrong-session-key-f33-workspace-bad-key.json", "failed_command": null, "variants": [ { "feature_id": "F33.workspace-bad-key", "family": "F33", "result": "pass", "label": "Wrong session key (`F33.workspace-bad-key`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-02-wrong-session-key-f33-workspace-bad-key.json", "failed_command": null } ] }, { "feature_id": "F33.workspace-exec", "family": "F33", "result": "pass", "label": "Commands, server info and liveness (`F33.workspace-exec`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-01-commands-server-info-and-liveness-f33-wo.json", "failed_command": null, "variants": [ { "feature_id": "F33.workspace-exec", "family": "F33", "result": "pass", "label": "Commands, server info and liveness (`F33.workspace-exec`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-01-commands-server-info-and-liveness-f33-wo.json", "failed_command": null } ] }, { "feature_id": "F33.workspace-files", "family": "F33", "result": "pass", "label": "File round trip (`F33.workspace-files`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-04-file-round-trip-f33-workspace-files.json", "failed_command": null, "variants": [ { "feature_id": "F33.workspace-files", "family": "F33", "result": "pass", "label": "File round trip (`F33.workspace-files`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-04-file-round-trip-f33-workspace-files.json", "failed_command": null } ] }, { "feature_id": "F33.workspace-git", "family": "F33", "result": "pass", "label": "Git changes and diff (`F33.workspace-git`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-05-git-changes-and-diff-f33-workspace-git.json", "failed_command": null, "variants": [ { "feature_id": "F33.workspace-git", "family": "F33", "result": "pass", "label": "Git changes and diff (`F33.workspace-git`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-05-git-changes-and-diff-f33-workspace-git.json", "failed_command": null } ] }, { "feature_id": "F33.workspace-runtime-lifecycle", "family": "F33", "result": "not-run", "label": "`get_runtime_session_key()` and `release_runtime()` hand out a scoped worker key and stop a conversation's runtime on a server with a Docker conversation runtime.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.workspace-runtime-lifecycle", "family": "F33", "result": "not-run", "label": "`get_runtime_session_key()` and `release_runtime()` hand out a scoped worker key and stop a conversation's runtime on a server with a Docker conversation runtime.", "run": null, "transcript": null } ] }, { "feature_id": "F33.workspace-runtime-scope", "family": "F33", "result": "pass", "label": "Conversation-scoped workspace (`F33.workspace-runtime-scope`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-11-conversation-scoped-workspace-f33-worksp.json", "failed_command": null, "variants": [ { "feature_id": "F33.workspace-runtime-scope", "family": "F33", "result": "pass", "label": "Conversation-scoped workspace (`F33.workspace-runtime-scope`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-11-conversation-scoped-workspace-f33-worksp.json", "failed_command": null } ] }, { "feature_id": "F33.workspace-settings", "family": "F33", "result": "pass", "label": "LLM profiles, secrets and MCP from the server (`F33.workspace-settings`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-06-llm-profiles-secrets-and-mcp-from-the-se.json", "failed_command": null, "variants": [ { "feature_id": "F33.workspace-settings", "family": "F33", "result": "pass", "label": "LLM profiles, secrets and MCP from the server (`F33.workspace-settings`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-06-llm-profiles-secrets-and-mcp-from-the-se.json", "failed_command": null } ] }, { "feature_id": "F33.ws-unavailable", "family": "F33", "result": "not-run", "label": "when the events socket cannot connect, `Conversation(...)` raises `WebSocketConnectionError` after `OPENHANDS_REMOTE_WS_READY_TIMEOUT` seconds (the conversation it created stays on the server); with `OPENHANDS_REMOTE_WS_READY_REQUIRED=false` it continues, and `run()` returns at `finished` through the REST fallback about 30 s after the server finished, with `state.events` matching the server.", "run": null, "transcript": null, "variants": [ { "feature_id": "F33.ws-unavailable", "family": "F33", "result": "not-run", "label": "when the events socket cannot connect, `Conversation(...)` raises `WebSocketConnectionError` after `OPENHANDS_REMOTE_WS_READY_TIMEOUT` seconds (the conversation it created stays on the server); with `OPENHANDS_REMOTE_WS_READY_REQUIRED=false` it continues, and `run()` returns at `finished` through the REST fallback about 30 s after the server finished, with `state.events` matching the server.", "run": null, "transcript": null } ] }, { "feature_id": "F34.bash-client", "family": "F34", "result": "pass", "label": "Bash client (`F34.bash-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-09-bash-client-f34-bash-client.json", "failed_command": null, "variants": [ { "feature_id": "F34.bash-client", "family": "F34", "result": "pass", "label": "Bash client (`F34.bash-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-09-bash-client-f34-bash-client.json", "failed_command": null } ] }, { "feature_id": "F34.bash-websocket", "family": "F34", "result": "xfail", "label": "Bash events socket from Node (`F34.bash-websocket`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-10-bash-events-socket-from-node-f34-bash-we.json", "failed_command": "jq -e '.saw_output and .errors == []' \"$F34/bash-ws.json\"", "variants": [ { "feature_id": "F34.bash-websocket", "family": "F34", "result": "xfail", "label": "Bash events socket from Node (`F34.bash-websocket`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-10-bash-events-socket-from-node-f34-bash-we.json", "failed_command": "jq -e '.saw_output and .errors == []' \"$F34/bash-ws.json\"" } ] }, { "feature_id": "F34.conversation-branching", "family": "F34", "result": "pass", "label": "Fork, navigate and ask (`F34.conversation-branching`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-24-fork-navigate-and-ask-f34-conversation-b.json", "failed_command": null, "variants": [ { "feature_id": "F34.conversation-branching", "family": "F34", "result": "pass", "label": "Fork, navigate and ask (`F34.conversation-branching`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-24-fork-navigate-and-ask-f34-conversation-b.json", "failed_command": null } ] }, { "feature_id": "F34.conversation-confirm", "family": "F34", "result": "pass", "label": "Confirmation and secrets (`F34.conversation-confirm`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-25-confirmation-and-secrets-f34-conversatio.json", "failed_command": null, "variants": [ { "feature_id": "F34.conversation-confirm", "family": "F34", "result": "pass", "label": "Confirmation and secrets (`F34.conversation-confirm`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-25-confirmation-and-secrets-f34-conversatio.json", "failed_command": null } ] }, { "feature_id": "F34.conversation-controls", "family": "F34", "result": "pass", "label": "Title and confirmation policy (`F34.conversation-controls`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-19-title-and-confirmation-policy-f34-conver.json", "failed_command": null, "variants": [ { "feature_id": "F34.conversation-controls", "family": "F34", "result": "pass", "label": "Title and confirmation policy (`F34.conversation-controls`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-19-title-and-confirmation-policy-f34-conver.json", "failed_command": null } ] }, { "feature_id": "F34.conversation-create", "family": "F34", "result": "pass", "label": "Create a conversation (`F34.conversation-create`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-12-create-a-conversation-f34-conversation-c.json", "failed_command": null, "variants": [ { "feature_id": "F34.conversation-create", "family": "F34", "result": "pass", "label": "Create a conversation (`F34.conversation-create`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-12-create-a-conversation-f34-conversation-c.json", "failed_command": null } ] }, { "feature_id": "F34.conversation-run", "family": "F34", "result": "pass", "label": "Send and run (`F34.conversation-run`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-13-send-and-run-f34-conversation-run.json", "failed_command": null, "variants": [ { "feature_id": "F34.conversation-run", "family": "F34", "result": "pass", "label": "Send and run (`F34.conversation-run`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-13-send-and-run-f34-conversation-run.json", "failed_command": null } ] }, { "feature_id": "F34.endpoint-audit", "family": "F34", "result": "fail", "label": "Endpoint audit against this checkout (`F34.endpoint-audit`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-31-endpoint-audit-against-this-checkout-f34.json", "failed_command": "mkdir -p \"$A/home\"", "variants": [ { "feature_id": "F34.endpoint-audit", "family": "F34", "result": "fail", "label": "Endpoint audit against this checkout (`F34.endpoint-audit`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-31-endpoint-audit-against-this-checkout-f34.json", "failed_command": "mkdir -p \"$A/home\"" } ] }, { "feature_id": "F34.event-stream", "family": "F34", "result": "pass", "label": "Event stream on Node's WebSocket (`F34.event-stream`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-17-event-stream-on-node-s-websocket-f34-eve.json", "failed_command": null, "variants": [ { "feature_id": "F34.event-stream", "family": "F34", "result": "pass", "label": "Event stream on Node's WebSocket (`F34.event-stream`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-17-event-stream-on-node-s-websocket-f34-eve.json", "failed_command": null } ] }, { "feature_id": "F34.event-stream-reconnect", "family": "F34", "result": "fail", "label": "Event stream reconnect (`F34.event-stream-reconnect`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-29-event-stream-reconnect-f34-event-stream-.json", "failed_command": "jq -e '.ws | .went_down and .attempts >= 1 and .reconnected and .replayed_after >= 1 and .sent_after >= 1' \"$F34/reconnect-ws.json\"", "variants": [ { "feature_id": "F34.event-stream-reconnect", "family": "F34", "result": "fail", "label": "Event stream reconnect (`F34.event-stream-reconnect`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-29-event-stream-reconnect-f34-event-stream-.json", "failed_command": "jq -e '.ws | .went_down and .attempts >= 1 and .reconnected and .replayed_after >= 1 and .sent_after >= 1' \"$F34/reconnect-ws.json\"" } ] }, { "feature_id": "F34.event-stream-reconnect-native", "family": "F34", "result": "fail", "label": "Reconnect on Node's own WebSocket (`F34.event-stream-reconnect-native`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-30-reconnect-on-node-s-own-websocket-f34-ev.json", "failed_command": "RNID=$(control-agent-server conversation start --no-run --placeholder-agent --tools none --no-autotitle --workspace \"$F34/scoped\" --prompt 'qa-f34-reconnect: never run' --print-id)", "variants": [ { "feature_id": "F34.event-stream-reconnect-native", "family": "F34", "result": "fail", "label": "Reconnect on Node's own WebSocket (`F34.event-stream-reconnect-native`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-30-reconnect-on-node-s-own-websocket-f34-ev.json", "failed_command": "RNID=$(control-agent-server conversation start --no-run --placeholder-agent --tools none --no-autotitle --workspace \"$F34/scoped\" --prompt 'qa-f34-reconnect: never run' --print-id)" } ] }, { "feature_id": "F34.events-list", "family": "F34", "result": "pass", "label": "Events list (`F34.events-list`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-14-events-list-f34-events-list.json", "failed_command": null, "variants": [ { "feature_id": "F34.events-list", "family": "F34", "result": "pass", "label": "Events list (`F34.events-list`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-14-events-list-f34-events-list.json", "failed_command": null } ] }, { "feature_id": "F34.events-search-kind", "family": "F34", "result": "xfail", "label": "Short event kinds (`F34.events-search-kind`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-16-short-event-kinds-f34-events-search-kind.json", "failed_command": "jq -e '.short == .full and .short_count == 1' \"$F34/kind.json\"", "variants": [ { "feature_id": "F34.events-search-kind", "family": "F34", "result": "xfail", "label": "Short event kinds (`F34.events-search-kind`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-16-short-event-kinds-f34-events-search-kind.json", "failed_command": "jq -e '.short == .full and .short_count == 1' \"$F34/kind.json\"" } ] }, { "feature_id": "F34.events-unknown-id", "family": "F34", "result": "xfail", "label": "Unknown event id (`F34.events-unknown-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-15-unknown-event-id-f34-events-unknown-id-k.json", "failed_command": "jq -e '.single.http_error and .single.status == 404' \"$F34/unknown-event.json\"", "variants": [ { "feature_id": "F34.events-unknown-id", "family": "F34", "result": "xfail", "label": "Unknown event id (`F34.events-unknown-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-15-unknown-event-id-f34-events-unknown-id-k.json", "failed_command": "jq -e '.single.http_error and .single.status == 404' \"$F34/unknown-event.json\"" } ] }, { "feature_id": "F34.hooks-client", "family": "F34", "result": "pass", "label": "Hooks (`F34.hooks-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-26-hooks-f34-hooks-client.json", "failed_command": null, "variants": [ { "feature_id": "F34.hooks-client", "family": "F34", "result": "pass", "label": "Hooks (`F34.hooks-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-26-hooks-f34-hooks-client.json", "failed_command": null } ] }, { "feature_id": "F34.http-errors", "family": "F34", "result": "pass", "label": "Errors (`F34.http-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-02-errors-f34-http-errors.json", "failed_command": null, "variants": [ { "feature_id": "F34.http-errors", "family": "F34", "result": "pass", "label": "Errors (`F34.http-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-02-errors-f34-http-errors.json", "failed_command": null } ] }, { "feature_id": "F34.manager", "family": "F34", "result": "pass", "label": "Conversation manager (`F34.manager`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-20-conversation-manager-f34-manager.json", "failed_command": null, "variants": [ { "feature_id": "F34.manager", "family": "F34", "result": "pass", "label": "Conversation manager (`F34.manager`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-20-conversation-manager-f34-manager.json", "failed_command": null } ] }, { "feature_id": "F34.manager-acp", "family": "F34", "result": "pass", "label": "ACP conversations (`F34.manager-acp`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-23-acp-conversations-f34-manager-acp.json", "failed_command": null, "variants": [ { "feature_id": "F34.manager-acp", "family": "F34", "result": "pass", "label": "ACP conversations (`F34.manager-acp`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-23-acp-conversations-f34-manager-acp.json", "failed_command": null } ] }, { "feature_id": "F34.manager-create-options", "family": "F34", "result": "xfail", "label": "Create options (`F34.manager-create-options`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-22-create-options-f34-manager-create-option.json", "failed_command": "jq -e '.max_iterations == 7 and .stuck_detection == false' \"$F34/create-options.json\"", "variants": [ { "feature_id": "F34.manager-create-options", "family": "F34", "result": "xfail", "label": "Create options (`F34.manager-create-options`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-22-create-options-f34-manager-create-option.json", "failed_command": "jq -e '.max_iterations == 7 and .stuck_detection == false' \"$F34/create-options.json\"" } ] }, { "feature_id": "F34.manager-update-result", "family": "F34", "result": "xfail", "label": "Update result (`F34.manager-update-result`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-21-update-result-f34-manager-update-result-.json", "failed_command": "jq -e --arg id \"$SCID\" '.id == $id and .title == \"qa-f34-renamed\"' \"$F34/update.json\"", "variants": [ { "feature_id": "F34.manager-update-result", "family": "F34", "result": "xfail", "label": "Update result (`F34.manager-update-result`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-21-update-result-f34-manager-update-result-.json", "failed_command": "jq -e --arg id \"$SCID\" '.id == $id and .title == \"qa-f34-renamed\"' \"$F34/update.json\"" } ] }, { "feature_id": "F34.mcp-client", "family": "F34", "result": "pass", "label": "MCP test (`F34.mcp-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-27-mcp-test-f34-mcp-client.json", "failed_command": null, "variants": [ { "feature_id": "F34.mcp-client", "family": "F34", "result": "pass", "label": "MCP test (`F34.mcp-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-27-mcp-test-f34-mcp-client.json", "failed_command": null } ] }, { "feature_id": "F34.restart-reload", "family": "F34", "result": "pass", "label": "Restart (`F34.restart-reload`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-28-restart-f34-restart-reload.json", "failed_command": null, "variants": [ { "feature_id": "F34.restart-reload", "family": "F34", "result": "pass", "label": "Restart (`F34.restart-reload`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-28-restart-f34-restart-reload.json", "failed_command": null } ] }, { "feature_id": "F34.server-client", "family": "F34", "result": "pass", "label": "Server status and version gate (`F34.server-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-01-server-status-and-version-gate-f34-serve.json", "failed_command": null, "variants": [ { "feature_id": "F34.server-client", "family": "F34", "result": "pass", "label": "Server status and version gate (`F34.server-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-01-server-status-and-version-gate-f34-serve.json", "failed_command": null } ] }, { "feature_id": "F34.version-gate-old", "family": "F34", "result": "fail", "label": "Old server version (`F34.version-gate-old`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-32-old-server-version-f34-version-gate-old.json", "failed_command": "mkdir -p \"$OLD/home\" \"$OLD/persist\"", "variants": [ { "feature_id": "F34.version-gate-old", "family": "F34", "result": "fail", "label": "Old server version (`F34.version-gate-old`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-32-old-server-version-f34-version-gate-old.json", "failed_command": "mkdir -p \"$OLD/home\" \"$OLD/persist\"" } ] }, { "feature_id": "F34.workspace-binary-download", "family": "F34", "result": "xfail", "label": "Binary download through RemoteWorkspace (`F34.workspace-binary-download`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-05-binary-download-through-remoteworkspace-.json", "failed_command": "jq -e '.blob == [255, 0, 128, 254, 65] and .file_size == 5' \"$F34/binary.json\"", "variants": [ { "feature_id": "F34.workspace-binary-download", "family": "F34", "result": "xfail", "label": "Binary download through RemoteWorkspace (`F34.workspace-binary-download`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-05-binary-download-through-remoteworkspace-.json", "failed_command": "jq -e '.blob == [255, 0, 128, 254, 65] and .file_size == 5' \"$F34/binary.json\"" } ] }, { "feature_id": "F34.workspace-exec", "family": "F34", "result": "pass", "label": "Workspace commands (`F34.workspace-exec`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-03-workspace-commands-f34-workspace-exec.json", "failed_command": null, "variants": [ { "feature_id": "F34.workspace-exec", "family": "F34", "result": "pass", "label": "Workspace commands (`F34.workspace-exec`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-03-workspace-commands-f34-workspace-exec.json", "failed_command": null } ] }, { "feature_id": "F34.workspace-files", "family": "F34", "result": "pass", "label": "Workspace files (`F34.workspace-files`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-04-workspace-files-f34-workspace-files.json", "failed_command": null, "variants": [ { "feature_id": "F34.workspace-files", "family": "F34", "result": "pass", "label": "Workspace files (`F34.workspace-files`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-04-workspace-files-f34-workspace-files.json", "failed_command": null } ] }, { "feature_id": "F34.workspace-git", "family": "F34", "result": "pass", "label": "Git through RemoteWorkspace (`F34.workspace-git`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-07-git-through-remoteworkspace-f34-workspac.json", "failed_command": null, "variants": [ { "feature_id": "F34.workspace-git", "family": "F34", "result": "pass", "label": "Git through RemoteWorkspace (`F34.workspace-git`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-07-git-through-remoteworkspace-f34-workspac.json", "failed_command": null } ] }, { "feature_id": "F34.workspace-git-types", "family": "F34", "result": "xfail", "label": "Git status type (`F34.workspace-git-types`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-08-git-status-type-f34-workspace-git-types-.json", "failed_command": "$TSC \"$F34/git-types.mts\"", "variants": [ { "feature_id": "F34.workspace-git-types", "family": "F34", "result": "xfail", "label": "Git status type (`F34.workspace-git-types`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-08-git-status-type-f34-workspace-git-types-.json", "failed_command": "$TSC \"$F34/git-types.mts\"" } ] }, { "feature_id": "F34.workspace-scoped", "family": "F34", "result": "pass", "label": "Conversation-scoped workspace (`F34.workspace-scoped`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-06-conversation-scoped-workspace-f34-worksp.json", "failed_command": null, "variants": [ { "feature_id": "F34.workspace-scoped", "family": "F34", "result": "pass", "label": "Conversation-scoped workspace (`F34.workspace-scoped`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-06-conversation-scoped-workspace-f34-worksp.json", "failed_command": null } ] }, { "feature_id": "F34.workspace-session", "family": "F34", "result": "pass", "label": "Workspace session (`F34.workspace-session`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-11-workspace-session-f34-workspace-session.json", "failed_command": null, "variants": [ { "feature_id": "F34.workspace-session", "family": "F34", "result": "pass", "label": "Workspace session (`F34.workspace-session`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-11-workspace-session-f34-workspace-session.json", "failed_command": null } ] }, { "feature_id": "F34.ws-callback-client", "family": "F34", "result": "xfail", "label": "Conversation callback socket from Node (`F34.ws-callback-client`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-18-conversation-callback-socket-from-node-f.json", "failed_command": "jq -e '(.kinds | index(\"ConversationStateUpdateEvent\")) != null and .errors == []' \"$F34/callback.json\"", "variants": [ { "feature_id": "F34.ws-callback-client", "family": "F34", "result": "xfail", "label": "Conversation callback socket from Node (`F34.ws-callback-client`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-18-conversation-callback-socket-from-node-f.json", "failed_command": "jq -e '(.kinds | index(\"ConversationStateUpdateEvent\")) != null and .errors == []' \"$F34/callback.json\"" } ] } ], "raw_primary_rows": [ { "feature_id": "F01.alive", "family": "F01", "result": "pass", "label": "Liveness (`F01.alive`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-01-liveness-f01-alive.json", "failed_command": null }, { "feature_id": "F01.health", "family": "F01", "result": "pass", "label": "Health (`F01.health`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-02-health-f01-health.json", "failed_command": null }, { "feature_id": "F01.ready", "family": "F01", "result": "pass", "label": "Readiness (`F01.ready`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-03-readiness-f01-ready.json", "failed_command": null }, { "feature_id": "F01.server-info", "family": "F01", "result": "pass", "label": "Server info (`F01.server-info`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-04-server-info-f01-server-info.json", "failed_command": null }, { "feature_id": "F01.root-info", "family": "F01", "result": "pass", "label": "Root alias (`F01.root-info`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-05-root-alias-f01-root-info.json", "failed_command": null }, { "feature_id": "F01.uptime-restart", "family": "F01", "result": "pass", "label": "Uptime across a restart (`F01.uptime-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-06-uptime-across-a-restart-f01-uptime-resta.json", "failed_command": null }, { "feature_id": "F01.idle-time", "family": "F01", "result": "pass", "label": "Idle time (`F01.idle-time`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-07-idle-time-f01-idle-time.json", "failed_command": null }, { "feature_id": "F01.openapi", "family": "F01", "result": "pass", "label": "OpenAPI document (`F01.openapi`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-08-openapi-document-f01-openapi.json", "failed_command": null }, { "feature_id": "F01.static-files", "family": "F01", "result": "pass", "label": "Static files (`F01.static-files`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-09-static-files-f01-static-files.json", "failed_command": null }, { "feature_id": "F01.debug-tracebacks", "family": "F01", "result": "pass", "label": "DEBUG tracebacks (`F01.debug-tracebacks`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-10-debug-tracebacks-f01-debug-tracebacks.json", "failed_command": null }, { "feature_id": "F01.preload-tools", "family": "F01", "result": "pass", "label": "Tool preload (`F01.preload-tools`).", "run": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01", "transcript": "/workspace/verification-artifacts/runs/20261013-021702-46e6-map-f01/evidence/map-run/F01-server-status-11-tool-preload-f01-preload-tools.json", "failed_command": null }, { "feature_id": "F02.http-key-required", "family": "F02", "result": "pass", "label": "Key required on HTTP (`F02.http-key-required`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-01-key-required-on-http-f02-http-key-requir.json", "failed_command": null }, { "feature_id": "F02.http-header-only", "family": "F02", "result": "pass", "label": "Header only (`F02.http-header-only`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-02-header-only-f02-http-header-only.json", "failed_command": null }, { "feature_id": "F02.ws-first-frame", "family": "F02", "result": "pass", "label": "First-frame auth on every socket (`F02.ws-first-frame`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-03-first-frame-auth-on-every-socket-f02-ws-.json", "failed_command": null }, { "feature_id": "F02.ws-first-frame-reject", "family": "F02", "result": "pass", "label": "Rejected first frames (`F02.ws-first-frame-reject`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-04-rejected-first-frames-f02-ws-first-frame.json", "failed_command": null }, { "feature_id": "F02.ws-auth-timeout", "family": "F02", "result": "pass", "label": "Auth timeout (`F02.ws-auth-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-05-auth-timeout-f02-ws-auth-timeout.json", "failed_command": null }, { "feature_id": "F02.ws-legacy-key", "family": "F02", "result": "pass", "label": "Deprecated query and header keys (`F02.ws-legacy-key`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-06-deprecated-query-and-header-keys-f02-ws-.json", "failed_command": null }, { "feature_id": "F02.ws-legacy-reject", "family": "F02", "result": "pass", "label": "Wrong legacy key fails the handshake (`F02.ws-legacy-reject`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-07-wrong-legacy-key-fails-the-handshake-f02.json", "failed_command": null }, { "feature_id": "F02.workspace-session-mint", "family": "F02", "result": "pass", "label": "Mint the workspace cookie (`F02.workspace-session-mint`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-08-mint-the-workspace-cookie-f02-workspace-.json", "failed_command": null }, { "feature_id": "F02.workspace-cookie-access", "family": "F02", "result": "pass", "label": "The cookie alone opens workspace files (`F02.workspace-cookie-access`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-09-the-cookie-alone-opens-workspace-files-f.json", "failed_command": null }, { "feature_id": "F02.workspace-session-delete", "family": "F02", "result": "pass", "label": "Delete the workspace cookie (`F02.workspace-session-delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-10-delete-the-workspace-cookie-f02-workspac.json", "failed_command": null }, { "feature_id": "F02.multiple-keys", "family": "F02", "result": "pass", "label": "A second key (`F02.multiple-keys`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-11-a-second-key-f02-multiple-keys.json", "failed_command": null }, { "feature_id": "F02.rotate-key", "family": "F02", "result": "pass", "label": "Rotate the key (`F02.rotate-key`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-12-rotate-the-key-f02-rotate-key.json", "failed_command": null }, { "feature_id": "F02.keys-hidden-from-commands", "family": "F02", "result": "pass", "label": "Keys stay out of commands (`F02.keys-hidden-from-commands`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-13-keys-stay-out-of-commands-f02-keys-hidde.json", "failed_command": null }, { "feature_id": "F02.cors-default-origins", "family": "F02", "result": "pass", "label": "Default CORS origins (`F02.cors-default-origins`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-14-default-cors-origins-f02-cors-default-or.json", "failed_command": null }, { "feature_id": "F02.cors-configured-origins", "family": "F02", "result": "pass", "label": "Configured CORS origins (`F02.cors-configured-origins`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-15-configured-cors-origins-f02-cors-configu.json", "failed_command": null }, { "feature_id": "F02.cors-workspace-routes", "family": "F02", "result": "pass", "label": "Workspace routes accept any origin (`F02.cors-workspace-routes`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-16-workspace-routes-accept-any-origin-f02-c.json", "failed_command": null }, { "feature_id": "F02.no-auth-mode", "family": "F02", "result": "pass", "label": "Unauthenticated mode (`F02.no-auth-mode`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-17-unauthenticated-mode-f02-no-auth-mode.json", "failed_command": null }, { "feature_id": "F02.legacy-env-key", "family": "F02", "result": "pass", "label": "Legacy `SESSION_API_KEY` (`F02.legacy-env-key`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-18-legacy-session-api-key-f02-legacy-env-ke.json", "failed_command": null }, { "feature_id": "F02.default-bind", "family": "F02", "result": "fail", "label": "Default bind address (`F02.default-bind`).", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-19-default-bind-address-f02-default-bind.json", "failed_command": "control-agent-server exec --save F02.default-bind/no-key --expect-output 'Starting OpenHands Agent Server on 127.0.0.1:8000' --expect-output 'listening on 127.0.0.1:8000' --reject-output 'Binding to all interfaces' -- unshare -rn env -i PATH=\"$PATH\" QA_F02_PYTHON=\"$QA_F02_PYTHON\" bash \"$F02/bind.sh\" \"$F02/bind-no-key\"" }, { "feature_id": "F02.workspace-cookie-root-path", "family": "F02", "result": "xfail", "label": "Cookie path ignores a prefix (`F02.workspace-cookie-root-path`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-20-cookie-path-ignores-a-prefix-f02-workspa.json", "failed_command": "control-agent-server exec --run \"$PREFIXED\" --save F02.workspace-cookie-root-path/file-with-cookie -- bash \"$F02/jar.sh\" GET \"/qa-f02-prefix/$PCID/workspace/qa-f02.html\" nokey 200 'has:qa-f02-workspace-file'" }, { "feature_id": "F02.workspace-session-ipv6-loopback", "family": "F02", "result": "xfail", "label": "IPv6 loopback is not a secure context (`F02.workspace-session-ipv6-loopback`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02", "transcript": "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/map-run/F02-auth-and-sessions-21-ipv6-loopback-is-not-a-secure-context-f0.json", "failed_command": "control-agent-server api POST /api/auth/workspace-session --header 'Host: [::1]:8000' --expect 204 --save F02.workspace-session-ipv6-loopback/ipv6-host --check-header set-cookie contains SameSite=none --check-header set-cookie contains Secure --check-header set-cookie contains Partitioned" }, { "feature_id": "F03.dormant-boot", "family": "F03", "result": "pass", "label": "Dormant boot (`F03.dormant-boot`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-01-dormant-boot-f03-dormant-boot.json", "failed_command": null }, { "feature_id": "F03.dormant-gates-api", "family": "F03", "result": "pass", "label": "Dormant gate (`F03.dormant-gates-api`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-02-dormant-gate-f03-dormant-gates-api.json", "failed_command": null }, { "feature_id": "F03.dormant-gates-sockets", "family": "F03", "result": "pass", "label": "Dormant sockets (`F03.dormant-gates-sockets`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-03-dormant-sockets-f03-dormant-gates-socket.json", "failed_command": null }, { "feature_id": "F03.dormant-gates-v1", "family": "F03", "result": "pass", "label": "Dormant /v1 and app backends (`F03.dormant-gates-v1`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-04-dormant-v1-and-app-backends-f03-dormant-.json", "failed_command": null }, { "feature_id": "F03.init-key-auth", "family": "F03", "result": "pass", "label": "Init key auth (`F03.init-key-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-05-init-key-auth-f03-init-key-auth.json", "failed_command": null }, { "feature_id": "F03.init-validation", "family": "F03", "result": "pass", "label": "Init validation (`F03.init-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-06-init-validation-f03-init-validation.json", "failed_command": null }, { "feature_id": "F03.init-redacts-secret-input", "family": "F03", "result": "pass", "label": "Input redaction (`F03.init-redacts-secret-input`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-07-input-redaction-f03-init-redacts-secret-.json", "failed_command": null }, { "feature_id": "F03.init-failure-rollback", "family": "F03", "result": "pass", "label": "Failure rollback (`F03.init-failure-rollback`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-08-failure-rollback-f03-init-failure-rollba.json", "failed_command": null }, { "feature_id": "F03.init-success", "family": "F03", "result": "pass", "label": "Successful init (`F03.init-success`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-09-successful-init-f03-init-success.json", "failed_command": null }, { "feature_id": "F03.init-delivered-webhooks", "family": "F03", "result": "pass", "label": "Delivered webhooks (`F03.init-delivered-webhooks`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-10-delivered-webhooks-f03-init-delivered-we.json", "failed_command": null }, { "feature_id": "F03.init-enforces-session-keys", "family": "F03", "result": "pass", "label": "Delivered keys are enforced (`F03.init-enforces-session-keys`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-11-delivered-keys-are-enforced-f03-init-enf.json", "failed_command": null }, { "feature_id": "F03.init-applies-env", "family": "F03", "result": "pass", "label": "Init env reaches bash (`F03.init-applies-env`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-12-init-env-reaches-bash-f03-init-applies-e.json", "failed_command": null }, { "feature_id": "F03.init-keys-gate-bash-socket", "family": "F03", "result": "pass", "label": "Delivered keys gate the bash socket (`F03.init-keys-gate-bash-socket`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-13-delivered-keys-gate-the-bash-socket-f03-.json", "failed_command": null }, { "feature_id": "F03.init-opens-events-socket-and-v1", "family": "F03", "result": "pass", "label": "Events socket and /v1 open after init (`F03.init-opens-events-socket-and-v1`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-14-events-socket-and-v1-open-after-init-f03.json", "failed_command": null }, { "feature_id": "F03.init-web-url-root-path", "family": "F03", "result": "pass", "label": "Delivered web URL sets the root path (`F03.init-web-url-root-path`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-15-delivered-web-url-sets-the-root-path-f03.json", "failed_command": null }, { "feature_id": "F03.init-delivered-worktree-root", "family": "F03", "result": "pass", "label": "Delivered worktree root (`F03.init-delivered-worktree-root`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-16-delivered-worktree-root-f03-init-deliver.json", "failed_command": null }, { "feature_id": "F03.init-delivered-max-runs", "family": "F03", "result": "pass", "label": "Delivered run capacity (`F03.init-delivered-max-runs`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-17-delivered-run-capacity-f03-init-delivere.json", "failed_command": null }, { "feature_id": "F03.init-delivered-cors-origins", "family": "F03", "result": "xfail", "label": "Delivered CORS origins (`F03.init-delivered-cors-origins`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-18-delivered-cors-origins-f03-init-delivere.json", "failed_command": "control-agent-server api OPTIONS /api/settings --auth none --header 'Origin: https://qa-f03.example' --header 'Access-Control-Request-Method: GET' --expect 200 --check-header access-control-allow-origin eq https://qa-f03.example --save F03.init-delivered-cors-origins/preflight" }, { "feature_id": "F03.init-second-call-400", "family": "F03", "result": "pass", "label": "Second init is 400 (`F03.init-second-call-400`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-19-second-init-is-400-f03-init-second-call-.json", "failed_command": null }, { "feature_id": "F03.init-open-without-secret", "family": "F03", "result": "pass", "label": "Open init without a secret (`F03.init-open-without-secret`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-20-open-init-without-a-secret-f03-init-open.json", "failed_command": null }, { "feature_id": "F03.init-state-not-persisted", "family": "F03", "result": "pass", "label": "Init state is not persisted (`F03.init-state-not-persisted`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-21-init-state-is-not-persisted-f03-init-sta.json", "failed_command": null }, { "feature_id": "F03.gate-before-auth", "family": "F03", "result": "pass", "label": "Readiness precedes auth (`F03.gate-before-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-22-readiness-precedes-auth-f03-gate-before-.json", "failed_command": null }, { "feature_id": "F03.not-deferred-404", "family": "F03", "result": "pass", "label": "Not deferred is 404 (`F03.not-deferred-404`).", "run": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03", "transcript": "/workspace/verification-artifacts/runs/20261013-024346-99e7-map-f03/evidence/map-run/F03-deferred-init-23-not-deferred-is-404-f03-not-deferred-404.json", "failed_command": null }, { "feature_id": "F04.create-settings", "family": "F04", "result": "pass", "label": "Create from the saved settings (`F04.create-settings`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-01-create-from-the-saved-settings-f04-creat.json", "failed_command": null }, { "feature_id": "F04.create-inline-agent", "family": "F04", "result": "pass", "label": "Inline agent (`F04.create-inline-agent`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-02-inline-agent-f04-create-inline-agent.json", "failed_command": null }, { "feature_id": "F04.create-agent-profile", "family": "F04", "result": "pass", "label": "Stored Agent Profile (`F04.create-agent-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-03-stored-agent-profile-f04-create-agent-pr.json", "failed_command": null }, { "feature_id": "F04.create-validation", "family": "F04", "result": "pass", "label": "Create validation (`F04.create-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-04-create-validation-f04-create-validation.json", "failed_command": null }, { "feature_id": "F04.create-validation-redaction", "family": "F04", "result": "pass", "label": "422 bodies keep the LLM key out (`F04.create-validation-redaction`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-05-422-bodies-keep-the-llm-key-out-f04-crea.json", "failed_command": null }, { "feature_id": "F04.create-idempotent", "family": "F04", "result": "pass", "label": "Idempotent create with a client id (`F04.create-idempotent`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-06-idempotent-create-with-a-client-id-f04-c.json", "failed_command": null }, { "feature_id": "F04.create-initial-message-run-false", "family": "F04", "result": "xfail", "label": "`initial_message.run: false` (`F04.create-initial-message-run-false`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-07-initial-message-run-false-f04-create-ini.json", "failed_command": "false" }, { "feature_id": "F04.create-parent", "family": "F04", "result": "pass", "label": "Parent and child (`F04.create-parent`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-08-parent-and-child-f04-create-parent.json", "failed_command": null }, { "feature_id": "F04.create-worktree", "family": "F04", "result": "pass", "label": "Git worktree (`F04.create-worktree`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-09-git-worktree-f04-create-worktree.json", "failed_command": null }, { "feature_id": "F04.create-observability", "family": "F04", "result": "pass", "label": "Observability headers (`F04.create-observability`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-10-observability-headers-f04-create-observa.json", "failed_command": null }, { "feature_id": "F04.create-observability-precedence", "family": "F04", "result": "pass", "label": "Body versus headers (`F04.create-observability-precedence`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-11-body-versus-headers-f04-create-observabi.json", "failed_command": null }, { "feature_id": "F04.create-secrets", "family": "F04", "result": "pass", "label": "Start-time secrets (`F04.create-secrets`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-12-start-time-secrets-f04-create-secrets.json", "failed_command": null }, { "feature_id": "F04.create-secrets-get", "family": "F04", "result": "xfail", "label": "Start-time secret visible to GET (`F04.create-secrets-get`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-13-start-time-secret-visible-to-get-f04-cre.json", "failed_command": "control-agent-server api GET \"/$SECID\" --expect 200 --check secret_registry.secret_sources.QA_F04_TOKEN.kind eq StaticSecret --save F04.create-secrets-get/get" }, { "feature_id": "F04.create-secrets-plain", "family": "F04", "result": "xfail", "label": "Plain-string secret value (`F04.create-secrets-plain`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-14-plain-string-secret-value-f04-create-sec.json", "failed_command": "control-agent-server api POST --json \"{\\\"workspace\\\": {\\\"working_dir\\\": \\\"$W\\\"}, \\\"agent\\\": $OFFLINE_AGENT, \\\"autotitle\\\": false, \\\"secrets\\\": {\\\"QA_F04_PLAIN\\\": \\\"qa-f04-plain-value\\\"}}\" --expect 201,422 --save F04.create-secrets-plain/start" }, { "feature_id": "F04.run-real", "family": "F04", "result": "pass", "label": "A real run (`F04.run-real`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-15-a-real-run-f04-run-real.json", "failed_command": null }, { "feature_id": "F04.final-response", "family": "F04", "result": "pass", "label": "Final response (`F04.final-response`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-16-final-response-f04-final-response.json", "failed_command": null }, { "feature_id": "F04.title-llm-profile", "family": "F04", "result": "pass", "label": "Title from a saved LLM profile (`F04.title-llm-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-17-title-from-a-saved-llm-profile-f04-title.json", "failed_command": null }, { "feature_id": "F04.get", "family": "F04", "result": "pass", "label": "Read one conversation (`F04.get`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-18-read-one-conversation-f04-get.json", "failed_command": null }, { "feature_id": "F04.batch-get", "family": "F04", "result": "pass", "label": "Batch get (`F04.batch-get`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-19-batch-get-f04-batch-get.json", "failed_command": null }, { "feature_id": "F04.batch-get-limit", "family": "F04", "result": "xfail", "label": "100 ids in one batch (`F04.batch-get-limit`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-20-100-ids-in-one-batch-f04-batch-get-limit.json", "failed_command": "control-agent-server exec --expect-output QA_BATCH_OK --save F04.batch-get-limit/hundred -- python3 \"$F/qa-f04-batch.py\" \"$IDS100\" 400,422" }, { "feature_id": "F04.search-page", "family": "F04", "result": "pass", "label": "Search pages (`F04.search-page`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-21-search-pages-f04-search-page.json", "failed_command": null }, { "feature_id": "F04.search-filter", "family": "F04", "result": "pass", "label": "Status filter (`F04.search-filter`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-22-status-filter-f04-search-filter.json", "failed_command": null }, { "feature_id": "F04.count", "family": "F04", "result": "pass", "label": "Count (`F04.count`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-23-count-f04-count.json", "failed_command": null }, { "feature_id": "F04.patch-title", "family": "F04", "result": "pass", "label": "Rename (`F04.patch-title`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-24-rename-f04-patch-title.json", "failed_command": null }, { "feature_id": "F04.patch-tags", "family": "F04", "result": "pass", "label": "Retag (`F04.patch-tags`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-25-retag-f04-patch-tags.json", "failed_command": null }, { "feature_id": "F04.patch-errors", "family": "F04", "result": "pass", "label": "PATCH errors (`F04.patch-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-26-patch-errors-f04-patch-errors.json", "failed_command": null }, { "feature_id": "F04.patch-whitespace-title", "family": "F04", "result": "xfail", "label": "Whitespace-only title (`F04.patch-whitespace-title`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-27-whitespace-only-title-f04-patch-whitespa.json", "failed_command": "control-agent-server api PATCH \"/$CHILD\" --json '{\"title\": \" \"}' --expect 422 --save F04.patch-whitespace-title/patch" }, { "feature_id": "F04.patch-unknown-id", "family": "F04", "result": "xfail", "label": "PATCH on an unknown id (`F04.patch-unknown-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-28-patch-on-an-unknown-id-f04-patch-unknown.json", "failed_command": "control-agent-server api PATCH \"/$NOPE\" --json '{\"title\": \"QA nobody\"}' --expect 404 --save F04.patch-unknown-id/unknown" }, { "feature_id": "F04.delete", "family": "F04", "result": "pass", "label": "Delete (`F04.delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-29-delete-f04-delete.json", "failed_command": null }, { "feature_id": "F04.delete-unknown-id", "family": "F04", "result": "xfail", "label": "DELETE on an unknown id (`F04.delete-unknown-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-30-delete-on-an-unknown-id-f04-delete-unkno.json", "failed_command": "control-agent-server api DELETE \"/$NOPE\" --expect 404 --save F04.delete-unknown-id/unknown" }, { "feature_id": "F04.restart-persist", "family": "F04", "result": "pass", "label": "Restart (`F04.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-31-restart-f04-restart-persist.json", "failed_command": null }, { "feature_id": "F04.read-no-hydrate", "family": "F04", "result": "pass", "label": "Cold reads do not load a conversation (`F04.read-no-hydrate`).", "run": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04", "transcript": "/workspace/verification-artifacts/runs/20261013-024705-d55f-map-f04/evidence/map-run/F04-conversation-lifecycle-32-cold-reads-do-not-load-a-conversation-f0.json", "failed_command": null }, { "feature_id": "F05.run-queued", "family": "F05", "result": "pass", "label": "Run queued input on a real model (`F05.run-queued`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-01-run-queued-input-on-a-real-model-f05-run.json", "failed_command": null }, { "feature_id": "F05.run-finished-noop", "family": "F05", "result": "pass", "label": "Run with nothing to do (`F05.run-finished-noop`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-02-run-with-nothing-to-do-f05-run-finished-.json", "failed_command": null }, { "feature_id": "F05.run-new-input", "family": "F05", "result": "pass", "label": "New input after finishing (`F05.run-new-input`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-03-new-input-after-finishing-f05-run-new-in.json", "failed_command": null }, { "feature_id": "F05.pause-idle", "family": "F05", "result": "pass", "label": "Pause an idle conversation (`F05.pause-idle`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-04-pause-an-idle-conversation-f05-pause-idl.json", "failed_command": null }, { "feature_id": "F05.pause-repeat", "family": "F05", "result": "pass", "label": "Pause twice (`F05.pause-repeat`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-05-pause-twice-f05-pause-repeat.json", "failed_command": null }, { "feature_id": "F05.run-resume-paused", "family": "F05", "result": "pass", "label": "Resume a paused conversation (`F05.run-resume-paused`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-06-resume-a-paused-conversation-f05-run-res.json", "failed_command": null }, { "feature_id": "F05.interrupt-idle", "family": "F05", "result": "pass", "label": "Interrupt with nothing running (`F05.interrupt-idle`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-07-interrupt-with-nothing-running-f05-inter.json", "failed_command": null }, { "feature_id": "F05.run-error", "family": "F05", "result": "pass", "label": "A run that fails (`F05.run-error`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-08-a-run-that-fails-f05-run-error.json", "failed_command": null }, { "feature_id": "F05.pause-terminal", "family": "F05", "result": "pass", "label": "Pause and interrupt a terminal conversation (`F05.pause-terminal`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-09-pause-and-interrupt-a-terminal-conversat.json", "failed_command": null }, { "feature_id": "F05.run-error-responsive", "family": "F05", "result": "xfail", "label": "Responsive while a run fails (`F05.run-error-responsive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-10-responsive-while-a-run-fails-f05-run-err.json", "failed_command": "test \"$SLOWEST\" -le 1500" }, { "feature_id": "F05.unknown-id-run", "family": "F05", "result": "pass", "label": "Run an unknown id (`F05.unknown-id-run`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-11-run-an-unknown-id-f05-unknown-id-run.json", "failed_command": null }, { "feature_id": "F05.unknown-id-pause", "family": "F05", "result": "xfail", "label": "Pause or interrupt an unknown id (`F05.unknown-id-pause`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-12-pause-or-interrupt-an-unknown-id-f05-unk.json", "failed_command": "control-agent-server api POST \"/$NOPE/pause\" --expect 404 --save F05.unknown-id-pause/pause" }, { "feature_id": "F05.control-errors", "family": "F05", "result": "pass", "label": "Auth and malformed ids (`F05.control-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-13-auth-and-malformed-ids-f05-control-error.json", "failed_command": null }, { "feature_id": "F05.run-conflict", "family": "F05", "result": "pass", "label": "Run while running (`F05.run-conflict`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-14-run-while-running-f05-run-conflict.json", "failed_command": null }, { "feature_id": "F05.pause-running", "family": "F05", "result": "pass", "label": "Pause during a long model call (`F05.pause-running`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-15-pause-during-a-long-model-call-f05-pause.json", "failed_command": null }, { "feature_id": "F05.pause-running-get", "family": "F05", "result": "xfail", "label": "GET after a mid-call pause (`F05.pause-running-get`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-16-get-after-a-mid-call-pause-f05-pause-run.json", "failed_command": "control-agent-server api GET \"/$P\" --check execution_status eq paused --quiet --save F05.pause-running-get/get" }, { "feature_id": "F05.pause-running-release", "family": "F05", "result": "pass", "label": "Release a paused step with interrupt (`F05.pause-running-release`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-17-release-a-paused-step-with-interrupt-f05.json", "failed_command": null }, { "feature_id": "F05.interrupt-running", "family": "F05", "result": "pass", "label": "Interrupt a running conversation (`F05.interrupt-running`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-18-interrupt-a-running-conversation-f05-int.json", "failed_command": null }, { "feature_id": "F05.run-after-interrupt", "family": "F05", "result": "pass", "label": "Resume after an interrupt (`F05.run-after-interrupt`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-19-resume-after-an-interrupt-f05-run-after-.json", "failed_command": null }, { "feature_id": "F05.pause-tool-step", "family": "F05", "result": "pass", "label": "Pause during a tool call (`F05.pause-tool-step`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-20-pause-during-a-tool-call-f05-pause-tool-.json", "failed_command": null }, { "feature_id": "F05.interrupt-tool-call", "family": "F05", "result": "pass", "label": "Interrupt during a tool call (`F05.interrupt-tool-call`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-21-interrupt-during-a-tool-call-f05-interru.json", "failed_command": null }, { "feature_id": "F05.run-capacity", "family": "F05", "result": "pass", "label": "Run capacity (`F05.run-capacity`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-22-run-capacity-f05-run-capacity.json", "failed_command": null }, { "feature_id": "F05.stuck-detected", "family": "F05", "result": "pass", "label": "A run that loops ends `stuck` (`F05.stuck-detected`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-23-a-run-that-loops-ends-stuck-f05-stuck-de.json", "failed_command": null }, { "feature_id": "F05.stuck-rerun", "family": "F05", "result": "pass", "label": "Run a stuck conversation again (`F05.stuck-rerun`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-24-run-a-stuck-conversation-again-f05-stuck.json", "failed_command": null }, { "feature_id": "F05.stuck-new-input", "family": "F05", "result": "pass", "label": "New input un-sticks it (`F05.stuck-new-input`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-25-new-input-un-sticks-it-f05-stuck-new-inp.json", "failed_command": null }, { "feature_id": "F05.max-iterations-stop", "family": "F05", "result": "pass", "label": "Iteration cap (`F05.max-iterations-stop`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-26-iteration-cap-f05-max-iterations-stop.json", "failed_command": null }, { "feature_id": "F05.max-iterations-resume", "family": "F05", "result": "pass", "label": "Resume after the cap (`F05.max-iterations-resume`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-27-resume-after-the-cap-f05-max-iterations-.json", "failed_command": null }, { "feature_id": "F05.restart-running-graceful", "family": "F05", "result": "pass", "label": "Graceful restart while running (`F05.restart-running-graceful`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-28-graceful-restart-while-running-f05-resta.json", "failed_command": null }, { "feature_id": "F05.restart-keeps-status", "family": "F05", "result": "pass", "label": "Statuses survive the restart (`F05.restart-keeps-status`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-29-statuses-survive-the-restart-f05-restart.json", "failed_command": null }, { "feature_id": "F05.restart-hard-running", "family": "F05", "result": "fail", "label": "Crash while running (`F05.restart-hard-running`).", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-30-crash-while-running-f05-restart-hard-run.json", "failed_command": "control-agent-server api GET \"/$K\" --check execution_status eq error --save F05.restart-hard-running/get" }, { "feature_id": "F05.pause-running-crash", "family": "F05", "result": "fail", "label": "Crash after a mid-call pause (`F05.pause-running-crash`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05", "transcript": "/workspace/verification-artifacts/runs/20261013-025211-dd9d-map-f05/evidence/map-run/F05-run-pause-interrupt-31-crash-after-a-mid-call-pause-f05-pause-r.json", "failed_command": "control-agent-server api POST \"/$K/run\" --expect 200 --check success eq true" }, { "feature_id": "F06.send-no-run", "family": "F06", "result": "pass", "label": "Append a message without running (`F06.send-no-run`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-01-append-a-message-without-running-f06-sen.json", "failed_command": null }, { "feature_id": "F06.search-page", "family": "F06", "result": "pass", "label": "Paging and sort order (`F06.search-page`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-02-paging-and-sort-order-f06-search-page.json", "failed_command": null }, { "feature_id": "F06.search-validation", "family": "F06", "result": "pass", "label": "Bad search parameters (`F06.search-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-03-bad-search-parameters-f06-search-validat.json", "failed_command": null }, { "feature_id": "F06.search-filters", "family": "F06", "result": "pass", "label": "Filters (`F06.search-filters`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-04-filters-f06-search-filters.json", "failed_command": null }, { "feature_id": "F06.search-kind-short", "family": "F06", "result": "xfail", "label": "Short kind names (`F06.search-kind-short`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-05-short-kind-names-f06-search-kind-short-k.json", "failed_command": "control-agent-server api GET \"/$CID/events/search\" --query kind=MessageEvent --expect 200 --check items len-eq 2 --save F06.search-kind-short/short-name" }, { "feature_id": "F06.search-time-window", "family": "F06", "result": "pass", "label": "Time window (`F06.search-time-window`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-06-time-window-f06-search-time-window.json", "failed_command": null }, { "feature_id": "F06.count", "family": "F06", "result": "pass", "label": "Count (`F06.count`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-07-count-f06-count.json", "failed_command": null }, { "feature_id": "F06.get-one", "family": "F06", "result": "pass", "label": "Get one event (`F06.get-one`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-08-get-one-event-f06-get-one.json", "failed_command": null }, { "feature_id": "F06.unknown-event", "family": "F06", "result": "xfail", "label": "Unknown event id (`F06.unknown-event`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-09-unknown-event-id-f06-unknown-event-known.json", "failed_command": "control-agent-server api GET \"/$CID/events/00000000-0000-4000-8000-0000000000e6\" --expect 404 --save F06.unknown-event/get-one" }, { "feature_id": "F06.batch-get", "family": "F06", "result": "pass", "label": "Batch get (`F06.batch-get`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-10-batch-get-f06-batch-get.json", "failed_command": null }, { "feature_id": "F06.not-found-auth", "family": "F06", "result": "pass", "label": "Unknown conversation, bad id, no key (`F06.not-found-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-11-unknown-conversation-bad-id-no-key-f06-n.json", "failed_command": null }, { "feature_id": "F06.send-validation", "family": "F06", "result": "pass", "label": "Message validation (`F06.send-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-12-message-validation-f06-send-validation.json", "failed_command": null }, { "feature_id": "F06.send-image", "family": "F06", "result": "pass", "label": "Image content (`F06.send-image`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-13-image-content-f06-send-image.json", "failed_command": null }, { "feature_id": "F06.send-non-user-role", "family": "F06", "result": "xfail", "label": "Non-user role (`F06.send-non-user-role`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-14-non-user-role-f06-send-non-user-role-kno.json", "failed_command": "control-agent-server api POST \"/$CID/events\" --json '{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"QA_F06_ASSISTANT\"}]}' --expect 4xx --save F06.send-non-user-role/assistant" }, { "feature_id": "F06.ws-snapshot", "family": "F06", "result": "pass", "label": "First frame is a state snapshot (`F06.ws-snapshot`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-15-first-frame-is-a-state-snapshot-f06-ws-s.json", "failed_command": null }, { "feature_id": "F06.ws-handshake", "family": "F06", "result": "pass", "label": "Handshake: auth and close codes (`F06.ws-handshake`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-16-handshake-auth-and-close-codes-f06-ws-ha.json", "failed_command": null }, { "feature_id": "F06.ws-delete-close", "family": "F06", "result": "xfail", "label": "Deleting a conversation closes its sockets (`F06.ws-delete-close`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-17-deleting-a-conversation-closes-its-socke.json", "failed_command": "false" }, { "feature_id": "F06.ws-replay-all", "family": "F06", "result": "pass", "label": "Replay everything (`F06.ws-replay-all`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-18-replay-everything-f06-ws-replay-all.json", "failed_command": null }, { "feature_id": "F06.ws-replay-since", "family": "F06", "result": "pass", "label": "Replay since a timestamp (`F06.ws-replay-since`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-19-replay-since-a-timestamp-f06-ws-replay-s.json", "failed_command": null }, { "feature_id": "F06.ws-bad-frame", "family": "F06", "result": "pass", "label": "Bad frames on the socket (`F06.ws-bad-frame`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-20-bad-frames-on-the-socket-f06-ws-bad-fram.json", "failed_command": null }, { "feature_id": "F06.ws-send", "family": "F06", "result": "pass", "label": "Send a message over the socket (`F06.ws-send`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-21-send-a-message-over-the-socket-f06-ws-se.json", "failed_command": null }, { "feature_id": "F06.send-run", "family": "F06", "result": "pass", "label": "Run a turn and watch it live (`F06.send-run`, `F06.ws-live`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-22-run-a-turn-and-watch-it-live-f06-send-ru.json", "failed_command": null }, { "feature_id": "F06.ws-live", "family": "F06", "result": "pass", "label": "Run a turn and watch it live (`F06.send-run`, `F06.ws-live`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-22-run-a-turn-and-watch-it-live-f06-send-ru.json", "failed_command": null }, { "feature_id": "F06.search-timestamp-order", "family": "F06", "result": "xfail", "label": "Log order is timestamp order (`F06.search-timestamp-order`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-23-log-order-is-timestamp-order-f06-search-.json", "failed_command": "control-agent-server api GET \"/$LIVE/events/count\" --query timestamp__gte=\"$OBS_T\" --check . ge \"$FROM_OBS\" --save F06.search-timestamp-order/count-since-observation" }, { "feature_id": "F06.send-resets-finished", "family": "F06", "result": "pass", "label": "A message reopens a finished conversation (`F06.send-resets-finished`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-24-a-message-reopens-a-finished-conversatio.json", "failed_command": null }, { "feature_id": "F06.restart-persist", "family": "F06", "result": "pass", "label": "History survives a restart (`F06.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-25-history-survives-a-restart-f06-restart-p.json", "failed_command": null }, { "feature_id": "F06.send-run-limit", "family": "F06", "result": "pass", "label": "Run capacity full (`F06.send-run-limit`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-26-run-capacity-full-f06-send-run-limit.json", "failed_command": null }, { "feature_id": "F06.send-while-running", "family": "F06", "result": "pass", "label": "A message while the agent runs (`F06.send-while-running`).", "run": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06", "transcript": "/workspace/verification-artifacts/runs/20261013-030025-5cd8-map-f06/evidence/map-run/F06-conversation-events-27-a-message-while-the-agent-runs-f06-send-.json", "failed_command": null }, { "feature_id": "F07.sync-first", "family": "F07", "result": "pass", "label": "Sync comes first (`F07.sync-first`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-01-sync-comes-first-f07-sync-first.json", "failed_command": null }, { "feature_id": "F07.snapshot-transient", "family": "F07", "result": "pass", "label": "State snapshot (`F07.snapshot-transient`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-02-state-snapshot-f07-snapshot-transient.json", "failed_command": null }, { "feature_id": "F07.full-replay", "family": "F07", "result": "pass", "label": "Full replay (`F07.full-replay`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-03-full-replay-f07-full-replay.json", "failed_command": null }, { "feature_id": "F07.seq-on-disk", "family": "F07", "result": "pass", "label": "Seq is the on-disk index (`F07.seq-on-disk`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-04-seq-is-the-on-disk-index-f07-seq-on-disk.json", "failed_command": null }, { "feature_id": "F07.resume-cursor", "family": "F07", "result": "pass", "label": "Resume from a cursor (`F07.resume-cursor`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-05-resume-from-a-cursor-f07-resume-cursor.json", "failed_command": null }, { "feature_id": "F07.error-frame", "family": "F07", "result": "pass", "label": "Invalid message (`F07.error-frame`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-06-invalid-message-f07-error-frame.json", "failed_command": null }, { "feature_id": "F07.error-non-json", "family": "F07", "result": "pass", "label": "Non-JSON and binary frames (`F07.error-non-json`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-07-non-json-and-binary-frames-f07-error-non.json", "failed_command": null }, { "feature_id": "F07.reject-non-user", "family": "F07", "result": "pass", "label": "Non-user role (`F07.reject-non-user`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-08-non-user-role-f07-reject-non-user.json", "failed_command": null }, { "feature_id": "F07.auth-frame-ignored", "family": "F07", "result": "pass", "label": "Redundant auth frame (`F07.auth-frame-ignored`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-09-redundant-auth-frame-f07-auth-frame-igno.json", "failed_command": null }, { "feature_id": "F07.auth", "family": "F07", "result": "pass", "label": "Authentication (`F07.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-10-authentication-f07-auth.json", "failed_command": null }, { "feature_id": "F07.unknown-conversation", "family": "F07", "result": "pass", "label": "Unknown conversation (`F07.unknown-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-11-unknown-conversation-f07-unknown-convers.json", "failed_command": null }, { "feature_id": "F07.bad-handshake", "family": "F07", "result": "pass", "label": "Malformed handshake (`F07.bad-handshake`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-12-malformed-handshake-f07-bad-handshake.json", "failed_command": null }, { "feature_id": "F07.live-durable", "family": "F07", "result": "pass", "label": "Live durable frame (`F07.live-durable`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-13-live-durable-frame-f07-live-durable.json", "failed_command": null }, { "feature_id": "F07.live-order", "family": "F07", "result": "xfail", "label": "Live frames in seq order (`F07.live-order`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-14-live-frames-in-seq-order-f07-live-order-.json", "failed_command": "control-agent-server state cat \"evidence/F07.live-order/order-$ORD.json\" --check frames.3.frame.seq eq 0 --check frames.4.frame.seq eq 1 --check frames.5.frame.seq eq 2" }, { "feature_id": "F07.replay-then-live", "family": "F07", "result": "pass", "label": "Replay and live tail, each event once (`F07.replay-then-live`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-15-replay-and-live-tail-each-event-once-f07.json", "failed_command": null }, { "feature_id": "F07.send-message", "family": "F07", "result": "pass", "label": "Send a message on the socket (`F07.send-message`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-16-send-a-message-on-the-socket-f07-send-me.json", "failed_command": null }, { "feature_id": "F07.restart-resume", "family": "F07", "result": "pass", "label": "Cursor survives a restart (`F07.restart-resume`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-17-cursor-survives-a-restart-f07-restart-re.json", "failed_command": null }, { "feature_id": "F07.progress-frames", "family": "F07", "result": "pass", "label": "Streaming progress frames (`F07.progress-frames`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-18-streaming-progress-frames-f07-progress-f.json", "failed_command": null }, { "feature_id": "F07.progress-not-replayed", "family": "F07", "result": "pass", "label": "Progress is never replayed (`F07.progress-not-replayed`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-19-progress-is-never-replayed-f07-progress-.json", "failed_command": null }, { "feature_id": "F07.item-aborted", "family": "F07", "result": "pass", "label": "Interrupted stream (`F07.item-aborted`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-20-interrupted-stream-f07-item-aborted.json", "failed_command": null }, { "feature_id": "F07.frame-too-large", "family": "F07", "result": "pass", "label": "Oversized event (`F07.frame-too-large`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-21-oversized-event-f07-frame-too-large.json", "failed_command": null }, { "feature_id": "F07.slow-consumer", "family": "F07", "result": "pass", "label": "Stalled reader (`F07.slow-consumer`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-22-stalled-reader-f07-slow-consumer.json", "failed_command": null }, { "feature_id": "F07.subscriber-limit", "family": "F07", "result": "pass", "label": "Subscriber cap (`F07.subscriber-limit`).", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-23-subscriber-cap-f07-subscriber-limit.json", "failed_command": null }, { "feature_id": "F07.deleted-close", "family": "F07", "result": "xfail", "label": "Deleting closes the socket (`F07.deleted-close`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07", "transcript": "/workspace/verification-artifacts/runs/20261013-024935-e05c-map-f07/evidence/map-run/F07-session-socket-24-deleting-closes-the-socket-f07-deleted-c.json", "failed_command": "false" }, { "feature_id": "F08.policy-set", "family": "F08", "result": "pass", "label": "Set the confirmation policy (`F08.policy-set`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-01-set-the-confirmation-policy-f08-policy-s.json", "failed_command": null }, { "feature_id": "F08.policy-errors", "family": "F08", "result": "pass", "label": "Policy errors (`F08.policy-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-02-policy-errors-f08-policy-errors.json", "failed_command": null }, { "feature_id": "F08.always-confirm-wait", "family": "F08", "result": "pass", "label": "Wait for confirmation (`F08.always-confirm-wait`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-03-wait-for-confirmation-f08-always-confirm.json", "failed_command": null }, { "feature_id": "F08.confirm-accept", "family": "F08", "result": "pass", "label": "Accept (`F08.confirm-accept`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-04-accept-f08-confirm-accept.json", "failed_command": null }, { "feature_id": "F08.confirm-reject", "family": "F08", "result": "pass", "label": "Reject (`F08.confirm-reject`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-05-reject-f08-confirm-reject.json", "failed_command": null }, { "feature_id": "F08.confirm-errors", "family": "F08", "result": "pass", "label": "Confirmation errors (`F08.confirm-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-06-confirmation-errors-f08-confirm-errors.json", "failed_command": null }, { "feature_id": "F08.confirm-capacity", "family": "F08", "result": "pass", "label": "Accept with the run capacity full (`F08.confirm-capacity`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-07-accept-with-the-run-capacity-full-f08-co.json", "failed_command": null }, { "feature_id": "F08.confirm-real", "family": "F08", "result": "pass", "label": "A real model asks first (`F08.confirm-real`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-08-a-real-model-asks-first-f08-confirm-real.json", "failed_command": null }, { "feature_id": "F08.analyzer-set-clear", "family": "F08", "result": "pass", "label": "Set and clear the analyzer (`F08.analyzer-set-clear`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-09-set-and-clear-the-analyzer-f08-analyzer-.json", "failed_command": null }, { "feature_id": "F08.analyzer-errors", "family": "F08", "result": "pass", "label": "Analyzer errors (`F08.analyzer-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-10-analyzer-errors-f08-analyzer-errors.json", "failed_command": null }, { "feature_id": "F08.confirm-risky", "family": "F08", "result": "pass", "label": "Risk-based confirmation (`F08.confirm-risky`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-11-risk-based-confirmation-f08-confirm-risk.json", "failed_command": null }, { "feature_id": "F08.secrets-update", "family": "F08", "result": "pass", "label": "Update secrets (`F08.secrets-update`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-12-update-secrets-f08-secrets-update.json", "failed_command": null }, { "feature_id": "F08.secrets-visible", "family": "F08", "result": "xfail", "label": "Runtime secret visible to GET (`F08.secrets-visible`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-13-runtime-secret-visible-to-get-f08-secret.json", "failed_command": "control-agent-server api GET \"/$KID\" --check secret_registry.secret_sources.QA_F08_FRESH.kind eq StaticSecret --save F08.secrets-visible/get" }, { "feature_id": "F08.secrets-errors", "family": "F08", "result": "pass", "label": "Secret errors (`F08.secrets-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-14-secret-errors-f08-secrets-errors.json", "failed_command": null }, { "feature_id": "F08.secrets-bad-value", "family": "F08", "result": "xfail", "label": "Non-string secret value (`F08.secrets-bad-value`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-15-non-string-secret-value-f08-secrets-bad-.json", "failed_command": "control-agent-server api POST \"/$KID/secrets\" --json '{\"secrets\": {\"QA_F08_NUMBER\": 5}}' --expect 422 --save F08.secrets-bad-value/number" }, { "feature_id": "F08.secrets-terminal", "family": "F08", "result": "pass", "label": "Secrets in the terminal (`F08.secrets-terminal`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-16-secrets-in-the-terminal-f08-secrets-term.json", "failed_command": null }, { "feature_id": "F08.secrets-override", "family": "F08", "result": "pass", "label": "Override a start-time secret (`F08.secrets-override`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-17-override-a-start-time-secret-f08-secrets.json", "failed_command": null }, { "feature_id": "F08.secrets-lookup", "family": "F08", "result": "pass", "label": "Lookup secret (`F08.secrets-lookup`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-18-lookup-secret-f08-secrets-lookup.json", "failed_command": null }, { "feature_id": "F08.secrets-lookup-self", "family": "F08", "result": "pass", "label": "Lookup of the server's own secret (`F08.secrets-lookup-self`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-19-lookup-of-the-server-s-own-secret-f08-se.json", "failed_command": null }, { "feature_id": "F08.secrets-profile-scope", "family": "F08", "result": "pass", "label": "Agent Profile secret scope (`F08.secrets-profile-scope`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-20-agent-profile-secret-scope-f08-secrets-p.json", "failed_command": null }, { "feature_id": "F08.secrets-no-cipher", "family": "F08", "result": "pass", "label": "No cipher (`F08.secrets-no-cipher`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-21-no-cipher-f08-secrets-no-cipher.json", "failed_command": null }, { "feature_id": "F08.secrets-restart", "family": "F08", "result": "fail", "label": "Restart with a saved secret (`F08.secrets-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-22-restart-with-a-saved-secret-f08-secrets-.json", "failed_command": "control-agent-server doctor" }, { "feature_id": "F08.secrets-unsaved-restart", "family": "F08", "result": "xfail", "label": "Unsaved secret restart (`F08.secrets-unsaved-restart`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08", "transcript": "/workspace/verification-artifacts/runs/20261013-021713-afec-map-f08/evidence/map-run/F08-confirmation-security-secrets-23-unsaved-secret-restart-f08-secrets-unsav.json", "failed_command": "control-agent-server api GET \"/$UNS\" --check secret_registry.secret_sources.QA_F08_UNSAVED.kind eq StaticSecret --save F08.secrets-unsaved-restart/get" }, { "feature_id": "F09.auth-required", "family": "F09", "result": "pass", "label": "Key required (`F09.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-01-key-required-f09-auth-required.json", "failed_command": null }, { "feature_id": "F09.switch-llm", "family": "F09", "result": "pass", "label": "Switch to a caller-supplied LLM (`F09.switch-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-02-switch-to-a-caller-supplied-llm-f09-swit.json", "failed_command": null }, { "feature_id": "F09.switch-llm-ws", "family": "F09", "result": "pass", "label": "Socket view of a switch (`F09.switch-llm-ws`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-03-socket-view-of-a-switch-f09-switch-llm-w.json", "failed_command": null }, { "feature_id": "F09.switch-llm-errors", "family": "F09", "result": "pass", "label": "Switch errors (`F09.switch-llm-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-04-switch-errors-f09-switch-llm-errors.json", "failed_command": null }, { "feature_id": "F09.switch-llm-encrypted-key", "family": "F09", "result": "pass", "label": "Mid-conversation switch with a cipher token (`F09.switch-llm-encrypted-key`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-05-mid-conversation-switch-with-a-cipher-to.json", "failed_command": null }, { "feature_id": "F09.switch-llm-provider-connection", "family": "F09", "result": "pass", "label": "Switch through a provider connection (`F09.switch-llm-provider-connection`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-06-switch-through-a-provider-connection-f09.json", "failed_command": null }, { "feature_id": "F09.switch-profile", "family": "F09", "result": "pass", "label": "Switch to a saved profile (`F09.switch-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-07-switch-to-a-saved-profile-f09-switch-pro.json", "failed_command": null }, { "feature_id": "F09.switch-profile-errors", "family": "F09", "result": "pass", "label": "Profile switch errors (`F09.switch-profile-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-08-profile-switch-errors-f09-switch-profile.json", "failed_command": null }, { "feature_id": "F09.switch-profile-edited", "family": "F09", "result": "xfail", "label": "Switch to an edited profile (`F09.switch-profile-edited`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-09-switch-to-an-edited-profile-f09-switch-p.json", "failed_command": "control-agent-server api GET \"/$EID\" --quiet --check agent.llm.model eq openai/qa-f09-edit-2 --save F09.switch-profile-edited/get" }, { "feature_id": "F09.switch-llm-registered-usage-id", "family": "F09", "result": "xfail", "label": "Switch onto a registered usage id (`F09.switch-llm-registered-usage-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-10-switch-onto-a-registered-usage-id-f09-sw.json", "failed_command": "control-agent-server api GET \"/$KID\" --quiet --check agent.llm.model eq deepseek/deepseek-v4-pro --save F09.switch-llm-registered-usage-id/get" }, { "feature_id": "F09.switch-llm-before-first-run", "family": "F09", "result": "xfail", "label": "Switch before the first run (`F09.switch-llm-before-first-run`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-11-switch-before-the-first-run-f09-switch-l.json", "failed_command": "control-agent-server api GET \"/$BID\" --until-ok 20 --quiet --check stats.usage_to_metrics.qa-f09-early.accumulated_token_usage.prompt_tokens gt 0 --save F09.switch-llm-before-first-run/stats" }, { "feature_id": "F09.switch-llm-mid-run", "family": "F09", "result": "fail", "label": "Switch during a running turn (`F09.switch-llm-mid-run`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-12-switch-during-a-running-turn-f09-switch-.json", "failed_command": "control-agent-server ws read qa-f09-midrun --kinds ActionEvent --contains 'sleep 20' --expect-min 1 --wait 120" }, { "feature_id": "F09.switch-llm-mid-run-get", "family": "F09", "result": "fail", "label": "GET right after a mid-run switch (`F09.switch-llm-mid-run-get`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-13-get-right-after-a-mid-run-switch-f09-swi.json", "failed_command": "control-agent-server ws read qa-f09-midget --kinds ActionEvent --contains 'sleep 20' --expect-min 1 --wait 120" }, { "feature_id": "F09.acp-model-native", "family": "F09", "result": "pass", "label": "ACP switch on a regular conversation (`F09.acp-model-native`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-14-acp-switch-on-a-regular-conversation-f09.json", "failed_command": null }, { "feature_id": "F09.acp-model-deferred", "family": "F09", "result": "pass", "label": "ACP model before the first run (`F09.acp-model-deferred`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-15-acp-model-before-the-first-run-f09-acp-m.json", "failed_command": null }, { "feature_id": "F09.acp-model-blank", "family": "F09", "result": "xfail", "label": "Blank ACP model (`F09.acp-model-blank`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-16-blank-acp-model-f09-acp-model-blank-know.json", "failed_command": "control-agent-server api POST \"/$AID2/switch_acp_model\" --json '{\"model\": \" \"}' --expect 400,422 --save F09.acp-model-blank/blank" }, { "feature_id": "F09.switch-llm-acp-refused", "family": "F09", "result": "xfail", "label": "LLM switches on an ACP conversation (`F09.switch-llm-acp-refused`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-17-llm-switches-on-an-acp-conversation-f09-.json", "failed_command": "control-agent-server api POST \"/$AID3/switch_profile\" --json '{\"profile_name\": \"deepseek-pro\"}' --expect 400,409,422 --save F09.switch-llm-acp-refused/switch-profile" }, { "feature_id": "F09.acp-model-live", "family": "F09", "result": "pass", "label": "Live ACP switch (`F09.acp-model-live`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-18-live-acp-switch-f09-acp-model-live.json", "failed_command": null }, { "feature_id": "F09.acp-model-live-unknown", "family": "F09", "result": "xfail", "label": "Unknown model on a live session (`F09.acp-model-live-unknown`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-19-unknown-model-on-a-live-session-f09-acp-.json", "failed_command": "control-agent-server api POST \"/$LID/switch_acp_model\" --json '{\"model\": \"qa-f09-nope\"}' --expect 400,422 --save F09.acp-model-live-unknown/switch" }, { "feature_id": "F09.acp-model-live-timeout", "family": "F09", "result": "pass", "label": "Live switch that times out (`F09.acp-model-live-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-20-live-switch-that-times-out-f09-acp-model.json", "failed_command": null }, { "feature_id": "F09.ts-client", "family": "F09", "result": "pass", "label": "TypeScript client (`F09.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-21-typescript-client-f09-ts-client.json", "failed_command": null }, { "feature_id": "F09.ts-client-acp-docs", "family": "F09", "result": "xfail", "label": "TypeScript docstrings for a pre-run ACP switch (`F09.ts-client-acp-docs`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-22-typescript-docstrings-for-a-pre-run-acp-.json", "failed_command": "false" }, { "feature_id": "F09.load-plugin", "family": "F09", "result": "pass", "label": "Load a plugin (`F09.load-plugin`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-23-load-a-plugin-f09-load-plugin.json", "failed_command": null }, { "feature_id": "F09.load-plugin-hook-runs", "family": "F09", "result": "pass", "label": "The loaded hook runs (`F09.load-plugin-hook-runs`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-24-the-loaded-hook-runs-f09-load-plugin-hoo.json", "failed_command": null }, { "feature_id": "F09.load-plugin-errors", "family": "F09", "result": "pass", "label": "Plugin load errors (`F09.load-plugin-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-25-plugin-load-errors-f09-load-plugin-error.json", "failed_command": null }, { "feature_id": "F09.load-plugin-twice", "family": "F09", "result": "xfail", "label": "Load the same plugin twice (`F09.load-plugin-twice`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-26-load-the-same-plugin-twice-f09-load-plug.json", "failed_command": "control-agent-server api GET \"/$QID\" --quiet --check hook_config.user_prompt_submit len-eq 1 --save F09.load-plugin-twice/get" }, { "feature_id": "F09.load-plugin-sdk", "family": "F09", "result": "pass", "label": "Load a plugin from the Python SDK (`F09.load-plugin-sdk`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-27-load-a-plugin-from-the-python-sdk-f09-lo.json", "failed_command": null }, { "feature_id": "F09.restart-persist", "family": "F09", "result": "pass", "label": "Restart (`F09.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09", "transcript": "/workspace/verification-artifacts/runs/20261013-021812-2254-map-f09/evidence/map-run/F09-model-switching-plugins-28-restart-f09-restart-persist.json", "failed_command": null }, { "feature_id": "F10.fork-whole", "family": "F10", "result": "pass", "label": "Fork the whole log (`F10.fork-whole`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-01-fork-the-whole-log-f10-fork-whole.json", "failed_command": null }, { "feature_id": "F10.fork-named", "family": "F10", "result": "pass", "label": "Named fork (`F10.fork-named`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-02-named-fork-f10-fork-named.json", "failed_command": null }, { "feature_id": "F10.fork-from-event", "family": "F10", "result": "pass", "label": "Fork from an event (`F10.fork-from-event`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-03-fork-from-an-event-f10-fork-from-event.json", "failed_command": null }, { "feature_id": "F10.fork-metrics", "family": "F10", "result": "pass", "label": "Metrics on a fork (`F10.fork-metrics`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-04-metrics-on-a-fork-f10-fork-metrics.json", "failed_command": null }, { "feature_id": "F10.fork-independent", "family": "F10", "result": "pass", "label": "Fork and source diverge (`F10.fork-independent`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-05-fork-and-source-diverge-f10-fork-indepen.json", "failed_command": null }, { "feature_id": "F10.fork-errors", "family": "F10", "result": "pass", "label": "Unknown, taken and invalid input (`F10.fork-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-06-unknown-taken-and-invalid-input-f10-fork.json", "failed_command": null }, { "feature_id": "F10.fork-tags-consistent", "family": "F10", "result": "xfail", "label": "Fork tags agree with the stored metadata (`F10.fork-tags-consistent`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-07-fork-tags-agree-with-the-stored-metadata.json", "failed_command": "control-agent-server state cat meta.json --conversation \"$TF\" --check title eq 'QA F10 tag fork' --check tags.qa eq \"$TF_QA\"" }, { "feature_id": "F10.fork-keeps-credentials", "family": "F10", "result": "xfail", "label": "The fork keeps the model keys (`F10.fork-keeps-credentials`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-08-the-fork-keeps-the-model-keys-f10-fork-k.json", "failed_command": "control-agent-server state cat base_state.json --conversation \"$KF\" --max-chars 200 --check agent.llm.api_key exists --check agent.condenser.llm.api_key exists" }, { "feature_id": "F10.fork-run-memory", "family": "F10", "result": "xfail", "label": "A fork runs with the source's memory (`F10.fork-run-memory`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-09-a-fork-runs-with-the-source-s-memory-f10.json", "failed_command": "control-agent-server conversation events \"$AF\" --kinds ConversationErrorEvent --contains LLMAuthenticationError --expect-count 0 --save F10.fork-run-memory/auth-errors" }, { "feature_id": "F10.navigate-event", "family": "F10", "result": "pass", "label": "Navigate to an earlier event (`F10.navigate-event`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-10-navigate-to-an-earlier-event-f10-navigat.json", "failed_command": null }, { "feature_id": "F10.navigate-branch", "family": "F10", "result": "pass", "label": "The next event branches (`F10.navigate-branch`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-11-the-next-event-branches-f10-navigate-bra.json", "failed_command": null }, { "feature_id": "F10.navigate-empty", "family": "F10", "result": "pass", "label": "Navigate to the empty tree (`F10.navigate-empty`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-12-navigate-to-the-empty-tree-f10-navigate-.json", "failed_command": null }, { "feature_id": "F10.navigate-errors", "family": "F10", "result": "pass", "label": "Unknown, missing and unauthenticated (`F10.navigate-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-13-unknown-missing-and-unauthenticated-f10-.json", "failed_command": null }, { "feature_id": "F10.navigate-final-response", "family": "F10", "result": "xfail", "label": "Final response after navigating back (`F10.navigate-final-response`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-14-final-response-after-navigating-back-f10.json", "failed_command": "control-agent-server api GET \"/$AN/agent_final_response\" --check response eq '' --save F10.navigate-final-response/after-navigate" }, { "feature_id": "F10.condense-llm", "family": "F10", "result": "pass", "label": "Condense with DeepSeek (`F10.condense-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-15-condense-with-deepseek-f10-condense-llm.json", "failed_command": null }, { "feature_id": "F10.condense-no-condenser", "family": "F10", "result": "pass", "label": "No request-handling condenser (`F10.condense-no-condenser`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-16-no-request-handling-condenser-f10-conden.json", "failed_command": null }, { "feature_id": "F10.condense-status", "family": "F10", "result": "xfail", "label": "Refusal status (`F10.condense-status`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-17-refusal-status-f10-condense-status-known.json", "failed_command": "control-agent-server api POST \"/$NC/condense\" --expect 4xx --save F10.condense-status/disabled" }, { "feature_id": "F10.condense-llm-failure", "family": "F10", "result": "pass", "label": "Unreachable condenser model (`F10.condense-llm-failure`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-18-unreachable-condenser-model-f10-condense.json", "failed_command": null }, { "feature_id": "F10.condense-errors", "family": "F10", "result": "pass", "label": "Unknown, malformed and unauthenticated (`F10.condense-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-19-unknown-malformed-and-unauthenticated-f1.json", "failed_command": null }, { "feature_id": "F10.ask-agent", "family": "F10", "result": "pass", "label": "Ask a side question (`F10.ask-agent`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-20-ask-a-side-question-f10-ask-agent.json", "failed_command": null }, { "feature_id": "F10.ask-agent-errors", "family": "F10", "result": "pass", "label": "Bad input, no key, unreachable model (`F10.ask-agent-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-21-bad-input-no-key-unreachable-model-f10-a.json", "failed_command": null }, { "feature_id": "F10.ask-agent-running", "family": "F10", "result": "pass", "label": "Side question during a run (`F10.ask-agent-running`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-22-side-question-during-a-run-f10-ask-agent.json", "failed_command": null }, { "feature_id": "F10.ask-agent-unknown", "family": "F10", "result": "xfail", "label": "Unknown conversation (`F10.ask-agent-unknown`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-23-unknown-conversation-f10-ask-agent-unkno.json", "failed_command": "control-agent-server api POST \"/$NOPE/ask_agent\" --json '{\"question\": \"hi\"}' --expect 404 --save F10.ask-agent-unknown/unknown" }, { "feature_id": "F10.restart-persist", "family": "F10", "result": "fail", "label": "State after a restart (`F10.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-24-state-after-a-restart-f10-restart-persis.json", "failed_command": "control-agent-server doctor" }, { "feature_id": "F10.restart-cold-calls", "family": "F10", "result": "pass", "label": "Condense and ask on a cold conversation (`F10.restart-cold-calls`).", "run": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10", "transcript": "/workspace/verification-artifacts/runs/20261013-021915-949e-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-25-condense-and-ask-on-a-cold-conversation-.json", "failed_command": null }, { "feature_id": "F11.goal-validation", "family": "F11", "result": "pass", "label": "Invalid goal requests (`F11.goal-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-01-invalid-goal-requests-f11-goal-validatio.json", "failed_command": null }, { "feature_id": "F11.goal-unknown-id", "family": "F11", "result": "pass", "label": "Unknown and malformed ids (`F11.goal-unknown-id`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-02-unknown-and-malformed-ids-f11-goal-unkno.json", "failed_command": null }, { "feature_id": "F11.goal-auth", "family": "F11", "result": "pass", "label": "Authentication (`F11.goal-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-03-authentication-f11-goal-auth.json", "failed_command": null }, { "feature_id": "F11.goal-stop-noop", "family": "F11", "result": "pass", "label": "Stop with nothing to stop (`F11.goal-stop-noop`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-04-stop-with-nothing-to-stop-f11-goal-stop-.json", "failed_command": null }, { "feature_id": "F11.goal-resume-none", "family": "F11", "result": "pass", "label": "Resume with no goal (`F11.goal-resume-none`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-05-resume-with-no-goal-f11-goal-resume-none.json", "failed_command": null }, { "feature_id": "F11.goal-complete", "family": "F11", "result": "pass", "label": "A goal on a real model (`F11.goal-complete`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-06-a-goal-on-a-real-model-f11-goal-complete.json", "failed_command": null }, { "feature_id": "F11.goal-ws", "family": "F11", "result": "pass", "label": "Goal status on the socket (`F11.goal-ws`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-07-goal-status-on-the-socket-f11-goal-ws.json", "failed_command": null }, { "feature_id": "F11.goal-in-place", "family": "F11", "result": "pass", "label": "Same conversation, no new one (`F11.goal-in-place`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-08-same-conversation-no-new-one-f11-goal-in.json", "failed_command": null }, { "feature_id": "F11.goal-followup", "family": "F11", "result": "pass", "label": "Incomplete verdict re-prompts (`F11.goal-followup`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-09-incomplete-verdict-re-prompts-f11-goal-f.json", "failed_command": null }, { "feature_id": "F11.goal-capped", "family": "F11", "result": "pass", "label": "Round cap (`F11.goal-capped`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-10-round-cap-f11-goal-capped.json", "failed_command": null }, { "feature_id": "F11.goal-resume-finished", "family": "F11", "result": "pass", "label": "Nothing to resume after the end (`F11.goal-resume-finished`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-11-nothing-to-resume-after-the-end-f11-goal.json", "failed_command": null }, { "feature_id": "F11.goal-busy", "family": "F11", "result": "pass", "label": "Busy while a round is in flight (`F11.goal-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-12-busy-while-a-round-is-in-flight-f11-goal.json", "failed_command": null }, { "feature_id": "F11.goal-stop", "family": "F11", "result": "pass", "label": "Stop an active loop (`F11.goal-stop`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-13-stop-an-active-loop-f11-goal-stop.json", "failed_command": null }, { "feature_id": "F11.goal-superseded", "family": "F11", "result": "pass", "label": "A user message supersedes the loop (`F11.goal-superseded`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-14-a-user-message-supersedes-the-loop-f11-g.json", "failed_command": null }, { "feature_id": "F11.goal-superseded-socket", "family": "F11", "result": "pass", "label": "A socket message supersedes the loop (`F11.goal-superseded-socket`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-15-a-socket-message-supersedes-the-loop-f11.json", "failed_command": null }, { "feature_id": "F11.goal-interrupt-run", "family": "F11", "result": "pass", "label": "Interrupt the round (`F11.goal-interrupt-run`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-16-interrupt-the-round-f11-goal-interrupt-r.json", "failed_command": null }, { "feature_id": "F11.goal-busy-run", "family": "F11", "result": "pass", "label": "Busy with an ordinary run (`F11.goal-busy-run`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-17-busy-with-an-ordinary-run-f11-goal-busy-.json", "failed_command": null }, { "feature_id": "F11.goal-resume", "family": "F11", "result": "pass", "label": "Resume a stopped loop (`F11.goal-resume`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-18-resume-a-stopped-loop-f11-goal-resume.json", "failed_command": null }, { "feature_id": "F11.goal-restart-resume", "family": "F11", "result": "pass", "label": "Resume after a restart (`F11.goal-restart-resume`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-19-resume-after-a-restart-f11-goal-restart-.json", "failed_command": null }, { "feature_id": "F11.goal-run-error", "family": "F11", "result": "fail", "label": "The round's provider fails (`F11.goal-run-error`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-20-the-round-s-provider-fails-f11-goal-run-.json", "failed_command": "control-agent-server ws read f11-e --contains '\"key\": \"goal\", \"value\": {\"active\": false, \"status\": \"interrupted\", \"iteration\": 0' --wait 60" }, { "feature_id": "F11.goal-judge-error", "family": "F11", "result": "pass", "label": "The judge fails (`F11.goal-judge-error`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-21-the-judge-fails-f11-goal-judge-error.json", "failed_command": null }, { "feature_id": "F11.goal-capacity", "family": "F11", "result": "pass", "label": "Capacity (`F11.goal-capacity`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-22-capacity-f11-goal-capacity.json", "failed_command": null }, { "feature_id": "F11.goal-delete-active", "family": "F11", "result": "pass", "label": "Delete a conversation with an active loop (`F11.goal-delete-active`).", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-23-delete-a-conversation-with-an-active-loo.json", "failed_command": null }, { "feature_id": "F11.goal-cap-overrun", "family": "F11", "result": "xfail", "label": "A resume after a failed last audit overruns the cap (`F11.goal-cap-overrun`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-24-a-resume-after-a-failed-last-audit-overr.json", "failed_command": "jq -e '[.events[].value | select(.iteration > .max_iterations)] | length == 0'" }, { "feature_id": "F11.goal-restart-active", "family": "F11", "result": "xfail", "label": "A loop active at a restart stays \"running\" (`F11.goal-restart-active`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11", "transcript": "/workspace/verification-artifacts/runs/20261013-022002-af50-map-f11/evidence/map-run/F11-goals-25-a-loop-active-at-a-restart-stays-running.json", "failed_command": "jq -e '.events[-1].value.active == false and .events[-1].value.status == \"interrupted\"'" }, { "feature_id": "F13.auth", "family": "F13", "result": "pass", "label": "Auth (`F13.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-01-auth-f13-auth.json", "failed_command": null }, { "feature_id": "F13.execute-sync", "family": "F13", "result": "pass", "label": "Synchronous execute (`F13.execute-sync`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-02-synchronous-execute-f13-execute-sync.json", "failed_command": null }, { "feature_id": "F13.global-cwd", "family": "F13", "result": "pass", "label": "Global cwd (`F13.global-cwd`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-03-global-cwd-f13-global-cwd.json", "failed_command": null }, { "feature_id": "F13.env-sanitized", "family": "F13", "result": "pass", "label": "Sanitized environment (`F13.env-sanitized`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-04-sanitized-environment-f13-env-sanitized.json", "failed_command": null }, { "feature_id": "F13.env-config-credentials", "family": "F13", "result": "pass", "label": "Other credential variables (`F13.env-config-credentials`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-05-other-credential-variables-f13-env-confi.json", "failed_command": null }, { "feature_id": "F13.start-async", "family": "F13", "result": "pass", "label": "Asynchronous start (`F13.start-async`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-06-asynchronous-start-f13-start-async.json", "failed_command": null }, { "feature_id": "F13.ws-live", "family": "F13", "result": "pass", "label": "Live and replayed socket frames (`F13.ws-live`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-07-live-and-replayed-socket-frames-f13-ws-l.json", "failed_command": null }, { "feature_id": "F13.ws-command", "family": "F13", "result": "pass", "label": "Commands over the socket (`F13.ws-command`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-08-commands-over-the-socket-f13-ws-command.json", "failed_command": null }, { "feature_id": "F13.ws-resend-all", "family": "F13", "result": "pass", "label": "Deprecated replay flag (`F13.ws-resend-all`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-09-deprecated-replay-flag-f13-ws-resend-all.json", "failed_command": null }, { "feature_id": "F13.ws-subscriber-cap", "family": "F13", "result": "pass", "label": "Subscriber cap (`F13.ws-subscriber-cap`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-10-subscriber-cap-f13-ws-subscriber-cap.json", "failed_command": null }, { "feature_id": "F13.get-events", "family": "F13", "result": "pass", "label": "Single and batch reads (`F13.get-events`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-11-single-and-batch-reads-f13-get-events.json", "failed_command": null }, { "feature_id": "F13.search", "family": "F13", "result": "pass", "label": "Search (`F13.search`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-12-search-f13-search.json", "failed_command": null }, { "feature_id": "F13.timestamp-offset", "family": "F13", "result": "xfail", "label": "Offset timestamps (`F13.timestamp-offset`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-13-offset-timestamps-f13-timestamp-offset-k.json", "failed_command": "control-agent-server api GET /api/bash/bash_events/search --query kind__eq=BashCommand --query timestamp__gte=\"$GTE_PLUS2\" --check items contains \"$TZ_CMD\" --save F13.timestamp-offset/plus-two" }, { "feature_id": "F13.output-chunks", "family": "F13", "result": "pass", "label": "Output chunks (`F13.output-chunks`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-14-output-chunks-f13-output-chunks.json", "failed_command": null }, { "feature_id": "F13.order-gt-paging", "family": "F13", "result": "xfail", "label": "Order filter and paging (`F13.order-gt-paging`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-15-order-filter-and-paging-f13-order-gt-pag.json", "failed_command": "control-agent-server api GET /api/bash/bash_events/search --query command_id__eq=\"$BIG_CMD\" --query order__gt=0 --query limit=1 --check items len-eq 1 --check items.0.order eq 1 --quiet --save F13.order-gt-paging/limit-1" }, { "feature_id": "F13.execute-large-output", "family": "F13", "result": "xfail", "label": "Large output through execute (`F13.execute-large-output`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-16-large-output-through-execute-f13-execute.json", "failed_command": "control-agent-server api POST /api/bash/execute_bash_command --json '{\"command\": \"yes a | head -c 1100000; echo QA_F13_TAIL\", \"timeout\": 60}' --expect 200 --check exit_code eq 0 --check stdout len-eq 1100012 --quiet --save F13.execute-large-output/execute" }, { "feature_id": "F13.timeout", "family": "F13", "result": "pass", "label": "Timeout (`F13.timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-17-timeout-f13-timeout.json", "failed_command": null }, { "feature_id": "F13.stop", "family": "F13", "result": "pass", "label": "Stop one command (`F13.stop`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-18-stop-one-command-f13-stop.json", "failed_command": null }, { "feature_id": "F13.stop-idempotent", "family": "F13", "result": "pass", "label": "Stop is idempotent (`F13.stop-idempotent`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-19-stop-is-idempotent-f13-stop-idempotent.json", "failed_command": null }, { "feature_id": "F13.scoped-cwd", "family": "F13", "result": "pass", "label": "Scoped cwd (`F13.scoped-cwd`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-20-scoped-cwd-f13-scoped-cwd.json", "failed_command": null }, { "feature_id": "F13.scoped-routes", "family": "F13", "result": "pass", "label": "Scoped routes (`F13.scoped-routes`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-21-scoped-routes-f13-scoped-routes.json", "failed_command": null }, { "feature_id": "F13.scoped-isolation", "family": "F13", "result": "pass", "label": "Scope isolation (`F13.scoped-isolation`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-22-scope-isolation-f13-scoped-isolation.json", "failed_command": null }, { "feature_id": "F13.scoped-unknown", "family": "F13", "result": "pass", "label": "Unknown conversation (`F13.scoped-unknown`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-23-unknown-conversation-f13-scoped-unknown.json", "failed_command": null }, { "feature_id": "F13.persist-restart", "family": "F13", "result": "pass", "label": "Restart (`F13.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-24-restart-f13-persist-restart.json", "failed_command": null }, { "feature_id": "F13.clear", "family": "F13", "result": "pass", "label": "Clear (`F13.clear`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-25-clear-f13-clear.json", "failed_command": null }, { "feature_id": "F13.retention", "family": "F13", "result": "pass", "label": "Retention (`F13.retention`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-26-retention-f13-retention.json", "failed_command": null }, { "feature_id": "F13.retention-rolling", "family": "F13", "result": "pass", "label": "Rolling retention (`F13.retention-rolling`).", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-27-rolling-retention-f13-retention-rolling.json", "failed_command": null }, { "feature_id": "F13.scoped-bad-id", "family": "F13", "result": "xfail", "label": "Malformed conversation id (`F13.scoped-bad-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-28-malformed-conversation-id-f13-scoped-bad.json", "failed_command": "control-agent-server api GET /not-a-uuid/bash/bash_events/search --expect 422 --save F13.scoped-bad-id/search" }, { "feature_id": "F13.scoped-idle-eviction", "family": "F13", "result": "xfail", "label": "Idle eviction kills a running command (`F13.scoped-idle-eviction`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13", "transcript": "/workspace/verification-artifacts/runs/20261013-021707-182d-map-f13/evidence/map-run/F13-bash-29-idle-eviction-kills-a-running-command-f1.json", "failed_command": "pgrep -f \"$EV_MARK\"" }, { "feature_id": "F15.auth", "family": "F15", "result": "pass", "label": "Session key required (`F15.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-01-session-key-required-f15-auth.json", "failed_command": null }, { "feature_id": "F15.changes", "family": "F15", "result": "pass", "label": "Working-tree changes (`F15.changes`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-02-working-tree-changes-f15-changes.json", "failed_command": null }, { "feature_id": "F15.changes-statuses", "family": "F15", "result": "pass", "label": "Staged deletions, additions and renames (`F15.changes-statuses`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-03-staged-deletions-additions-and-renames-f.json", "failed_command": null }, { "feature_id": "F15.changes-base", "family": "F15", "result": "pass", "label": "Default display base (`F15.changes-base`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-04-default-display-base-f15-changes-base.json", "failed_command": null }, { "feature_id": "F15.changes-ref", "family": "F15", "result": "pass", "label": "Explicit base and its errors (`F15.changes-ref`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-05-explicit-base-and-its-errors-f15-changes.json", "failed_command": null }, { "feature_id": "F15.changes-nested-repo", "family": "F15", "result": "pass", "label": "Nested repository (`F15.changes-nested-repo`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-06-nested-repository-f15-changes-nested-rep.json", "failed_command": null }, { "feature_id": "F15.changes-repo-below-path", "family": "F15", "result": "xfail", "label": "Repository below the requested directory (`F15.changes-repo-below-path`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-07-repository-below-the-requested-directory.json", "failed_command": "control-agent-server api GET /api/git/changes --query path=\"$F/ws\" --expect 200 --check . len-eq 2 --check 0.path eq proj/README.md --check 1.path eq proj/notes.txt --save F15.changes-repo-below-path/ws" }, { "feature_id": "F15.changes-special-names", "family": "F15", "result": "xfail", "label": "File names with a space or non-ASCII characters (`F15.changes-special-names`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-08-file-names-with-a-space-or-non-ascii-cha.json", "failed_command": "control-agent-server api GET /api/git/changes --query path=\"$F/names\" --expect 200 --check . len-eq 1 --check 0.status eq UPDATED --check 0.path eq 'my notes.md' --save F15.changes-special-names/space" }, { "feature_id": "F15.changes-subdir-paths", "family": "F15", "result": "xfail", "label": "Subdirectory of a repository (`F15.changes-subdir-paths`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-09-subdirectory-of-a-repository-f15-changes.json", "failed_command": "python3 - \"$F/subdir.json\" \"$F/subdir/src\" \"$F/subdir\" <<'PY'\nimport json, os, sys\nchanges = json.load(open(sys.argv[1]))\npaths = [c[\"path\"] for c in changes if c[\"status\"] != \"DELETED\"]\nbases = [b for b in sys.argv[2:] if all(os.path.exists(os.path.join(b, p)) for p in paths)]\nassert paths and bases, f\"no single base resolves every path: {changes}\"\nPY" }, { "feature_id": "F15.non-repo", "family": "F15", "result": "pass", "label": "Not a repository, or no commits yet (`F15.non-repo`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-10-not-a-repository-or-no-commits-yet-f15-n.json", "failed_command": null }, { "feature_id": "F15.diff", "family": "F15", "result": "pass", "label": "Per-file diff (`F15.diff`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-11-per-file-diff-f15-diff.json", "failed_command": null }, { "feature_id": "F15.diff-commit", "family": "F15", "result": "pass", "label": "Diff of one commit (`F15.diff-commit`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-12-diff-of-one-commit-f15-diff-commit.json", "failed_command": null }, { "feature_id": "F15.diff-errors", "family": "F15", "result": "pass", "label": "Diff errors (`F15.diff-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-13-diff-errors-f15-diff-errors.json", "failed_command": null }, { "feature_id": "F15.diff-whitespace", "family": "F15", "result": "xfail", "label": "Phantom whitespace diff (`F15.diff-whitespace`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-14-phantom-whitespace-diff-f15-diff-whitesp.json", "failed_command": "python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); assert d[\"original\"] == d[\"modified\"], d' \"$F/ws-$name.json\"" }, { "feature_id": "F15.diff-deleted-file", "family": "F15", "result": "xfail", "label": "Diff of a deleted file (`F15.diff-deleted-file`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-15-diff-of-a-deleted-file-f15-diff-deleted-.json", "failed_command": "control-agent-server api GET /api/git/diff --query path=\"$F/deleted/src/util.py\" --expect 200 --check original eq \"$(git -C \"$REPO\" show HEAD:src/util.py)\" --check modified eq '' --save F15.diff-deleted-file/diff" }, { "feature_id": "F15.commits", "family": "F15", "result": "fail", "label": "Commit history (`F15.commits`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-16-commit-history-f15-commits.json", "failed_command": "control-agent-server api GET /api/git/commits --query path=\"$REPO\" --expect 200 --check commits len-eq 2 --check has_more eq false --check commits.0.sha eq \"$C2\" --check commits.0.subject eq 'Add util' --check commits.0.short_sha eq \"$(git -C \"$REPO\" log -1 --format=%h)\" --check commits.0.author eq 'QA Verify' --check commits.0.timestamp matches '^\\d{4}-\\d\\d-\\d\\dT\\d\\d:\\d\\d:\\d\\d[+-]\\d\\d:\\d\\d$' --check commits.1.sha eq \"$C1\" --check commits.1.subject eq 'Initial commit' --save F15.commits/all" }, { "feature_id": "F15.commit-changes", "family": "F15", "result": "pass", "label": "Files of one commit (`F15.commit-changes`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-17-files-of-one-commit-f15-commit-changes.json", "failed_command": null }, { "feature_id": "F15.scoped", "family": "F15", "result": "pass", "label": "Conversation-scoped views (`F15.scoped`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-18-conversation-scoped-views-f15-scoped.json", "failed_command": null }, { "feature_id": "F15.scoped-guard", "family": "F15", "result": "pass", "label": "Workspace guard (`F15.scoped-guard`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-19-workspace-guard-f15-scoped-guard.json", "failed_command": null }, { "feature_id": "F15.scoped-restart", "family": "F15", "result": "pass", "label": "After a restart (`F15.scoped-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-20-after-a-restart-f15-scoped-restart.json", "failed_command": null }, { "feature_id": "F15.sdk-workspace", "family": "F15", "result": "pass", "label": "Python SDK workspace (`F15.sdk-workspace`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-21-python-sdk-workspace-f15-sdk-workspace.json", "failed_command": null }, { "feature_id": "F15.ts-client", "family": "F15", "result": "fail", "label": "TypeScript client (`F15.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-22-typescript-client-f15-ts-client.json", "failed_command": "( cd clients/typescript && npm ci && npm run build )" }, { "feature_id": "F15.agent-changes", "family": "F15", "result": "pass", "label": "What the agent changed (`F15.agent-changes`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-23-what-the-agent-changed-f15-agent-changes.json", "failed_command": null }, { "feature_id": "F15.repo-search-token", "family": "F15", "result": "pass", "label": "Token lookup (`F15.repo-search-token`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-24-token-lookup-f15-repo-search-token.json", "failed_command": null }, { "feature_id": "F15.repo-search-errors", "family": "F15", "result": "pass", "label": "Provider and parameter validation (`F15.repo-search-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-25-provider-and-parameter-validation-f15-re.json", "failed_command": null }, { "feature_id": "F15.repo-search-unreachable", "family": "F15", "result": "fail", "label": "GitHub unreachable (`F15.repo-search-unreachable`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-26-github-unreachable-f15-repo-search-unrea.json", "failed_command": "control-agent-server doctor" }, { "feature_id": "F15.repo-search-timeout", "family": "F15", "result": "fail", "label": "GitHub timeout (`F15.repo-search-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15/evidence/map-run/F15-git-27-github-timeout-f15-repo-search-timeout.json", "failed_command": "control-agent-server api GET /api/git/repositories/search --query provider=github --expect 504 --expect-max-ms 30000 --check detail eq 'Internal Server Error' --check exception eq '504: GitHub repository search timed out' --check . not-contains qa-f15-not-a-real-token --save F15.repo-search-timeout/silent-proxy" }, { "feature_id": "F15.repo-search-github", "family": "F15", "result": "blocked", "label": "Live GitHub listing (`F15.repo-search-github`), blocked.", "run": "/workspace/verification-artifacts/runs/20261013-021709-764e-map-f15", "transcript": null, "failed_command": null }, { "feature_id": "F16.root-index", "family": "F16", "result": "pass", "label": "Root index (`F16.root-index`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-01-root-index-f16-root-index.json", "failed_command": null }, { "feature_id": "F16.file-bytes", "family": "F16", "result": "pass", "label": "File bytes and headers (`F16.file-bytes`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-02-file-bytes-and-headers-f16-file-bytes.json", "failed_command": null }, { "feature_id": "F16.content-types", "family": "F16", "result": "pass", "label": "Content types (`F16.content-types`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-03-content-types-f16-content-types.json", "failed_command": null }, { "feature_id": "F16.dir-index", "family": "F16", "result": "pass", "label": "Directory index and missing files (`F16.dir-index`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-04-directory-index-and-missing-files-f16-di.json", "failed_command": null }, { "feature_id": "F16.range", "family": "F16", "result": "pass", "label": "Byte ranges (`F16.range`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-05-byte-ranges-f16-range.json", "failed_command": null }, { "feature_id": "F16.live-content", "family": "F16", "result": "pass", "label": "Live content (`F16.live-content`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-06-live-content-f16-live-content.json", "failed_command": null }, { "feature_id": "F16.traversal-rejected", "family": "F16", "result": "pass", "label": "Traversal (`F16.traversal-rejected`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-07-traversal-f16-traversal-rejected.json", "failed_command": null }, { "feature_id": "F16.symlink-escape", "family": "F16", "result": "pass", "label": "Symlinks (`F16.symlink-escape`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-08-symlinks-f16-symlink-escape.json", "failed_command": null }, { "feature_id": "F16.unknown-conversation", "family": "F16", "result": "pass", "label": "Unknown and malformed ids (`F16.unknown-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-09-unknown-and-malformed-ids-f16-unknown-co.json", "failed_command": null }, { "feature_id": "F16.workspace-dir-missing", "family": "F16", "result": "pass", "label": "Workspace directory gone (`F16.workspace-dir-missing`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-10-workspace-directory-gone-f16-workspace-d.json", "failed_command": null }, { "feature_id": "F16.deleted-conversation", "family": "F16", "result": "pass", "label": "Deleted conversation (`F16.deleted-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-11-deleted-conversation-f16-deleted-convers.json", "failed_command": null }, { "feature_id": "F16.worktree-served", "family": "F16", "result": "pass", "label": "Worktree conversation (`F16.worktree-served`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-12-worktree-conversation-f16-worktree-serve.json", "failed_command": null }, { "feature_id": "F16.auth", "family": "F16", "result": "pass", "label": "Header or cookie (`F16.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-13-header-or-cookie-f16-auth.json", "failed_command": null }, { "feature_id": "F16.agent-artifact", "family": "F16", "result": "pass", "label": "The agent's own artifact (`F16.agent-artifact`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-14-the-agent-s-own-artifact-f16-agent-artif.json", "failed_command": null }, { "feature_id": "F16.restart", "family": "F16", "result": "pass", "label": "Across a restart (`F16.restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-15-across-a-restart-f16-restart.json", "failed_command": null }, { "feature_id": "F16.vscode-disabled", "family": "F16", "result": "pass", "label": "VS Code disabled (`F16.vscode-disabled`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-16-vs-code-disabled-f16-vscode-disabled.json", "failed_command": null }, { "feature_id": "F16.vscode-scoped-guards", "family": "F16", "result": "pass", "label": "Scoped guards (`F16.vscode-scoped-guards`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-17-scoped-guards-f16-vscode-scoped-guards.json", "failed_command": null }, { "feature_id": "F16.vscode-auth", "family": "F16", "result": "pass", "label": "VS Code routes are header-only (`F16.vscode-auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-18-vs-code-routes-are-header-only-f16-vscod.json", "failed_command": null }, { "feature_id": "F16.vscode-enabled-status", "family": "F16", "result": "fail", "label": "Enabled but not running (`F16.vscode-enabled-status`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-19-enabled-but-not-running-f16-vscode-enabl.json", "failed_command": "control-agent-server doctor" }, { "feature_id": "F16.vscode-url-shape", "family": "F16", "result": "fail", "label": "URL shape (`F16.vscode-url-shape`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-20-url-shape-f16-vscode-url-shape.json", "failed_command": "control-agent-server api GET /api/vscode/url --expect 200 --check url eq \"http://localhost:$VPORT/?tkn=$KEY&folder=workspace\" --save F16.vscode-url-shape/global" }, { "feature_id": "F16.vscode-url-not-running", "family": "F16", "result": "xfail", "label": "Not running, still advertised (`F16.vscode-url-not-running`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-21-not-running-still-advertised-f16-vscode-.json", "failed_command": "control-agent-server api GET /api/vscode/url --expect 200 --check . eq '{\"url\": null}' --save F16.vscode-url-not-running/global" }, { "feature_id": "F16.vscode-folder-encoding", "family": "F16", "result": "xfail", "label": "Folder with `&` and `#` (`F16.vscode-folder-encoding`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-22-folder-with-and-f16-vscode-folder-encodi.json", "failed_command": "python3 -c \"$ROUNDTRIP\" \"$OWS\"" }, { "feature_id": "F16.vscode-url-no-key", "family": "F16", "result": "fail", "label": "No key, no URL (`F16.vscode-url-no-key`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-23-no-key-no-url-f16-vscode-url-no-key.json", "failed_command": "NK=$(control-agent-server launch --new --no-auth --vscode --name f16-nokey --config-json '{\"vscode_port\": '\"$VPORT\"'}' --print-run)" }, { "feature_id": "F16.vscode-url-after-init", "family": "F16", "result": "fail", "label": "Key delivered by init, no URL (`F16.vscode-url-after-init`).", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-24-key-delivered-by-init-no-url-f16-vscode-.json", "failed_command": "DI=$(control-agent-server launch --new --no-auth --deferred-init --vscode --name f16-deferred --config-json '{\"vscode_port\": '\"$VPORT\"'}' --print-run)" }, { "feature_id": "F16.docker-root-cookie", "family": "F16", "result": "xfail", "label": "Docker-mode root and the cookie (`F16.docker-root-cookie`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-25-docker-mode-root-and-the-cookie-f16-dock.json", "failed_command": "control-agent-server api GET \"/$Z/workspace\" --run \"$DK\" --auth none --jar qa-f16-docker --expect 404 --save F16.docker-root-cookie/slashless" }, { "feature_id": "F16.vscode-running", "family": "F16", "result": "blocked", "label": "VS Code actually running (`F16.vscode-running`), blocked.", "run": "/workspace/verification-artifacts/runs/20261013-021937-8e5b-map-f16", "transcript": null, "failed_command": null }, { "feature_id": "F17.list-empty", "family": "F17", "result": "pass", "label": "Empty registry (`F17.list-empty`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-01-empty-registry-f17-list-empty.json", "failed_command": null }, { "feature_id": "F17.noop-delete-no-file", "family": "F17", "result": "xfail", "label": "A refused delete on a fresh directory (`F17.noop-delete-no-file`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-02-a-refused-delete-on-a-fresh-directory-f1.json", "failed_command": "control-agent-server state ls home/.openhands/workspaces.json --expect-count 0" }, { "feature_id": "F17.add-workspaces", "family": "F17", "result": "pass", "label": "Register a repository and a plain directory (`F17.add-workspaces`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-03-register-a-repository-and-a-plain-direct.json", "failed_command": null }, { "feature_id": "F17.wire-format", "family": "F17", "result": "pass", "label": "Wire format (`F17.wire-format`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-04-wire-format-f17-wire-format.json", "failed_command": null }, { "feature_id": "F17.idempotent-add", "family": "F17", "result": "pass", "label": "Idempotent re-add (`F17.idempotent-add`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-05-idempotent-re-add-f17-idempotent-add.json", "failed_command": null }, { "feature_id": "F17.delete-workspace", "family": "F17", "result": "pass", "label": "Delete a workspace (`F17.delete-workspace`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-06-delete-a-workspace-f17-delete-workspace.json", "failed_command": null }, { "feature_id": "F17.delete-not-found", "family": "F17", "result": "pass", "label": "Unknown paths (`F17.delete-not-found`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-07-unknown-paths-f17-delete-not-found.json", "failed_command": null }, { "feature_id": "F17.noop-write-untouched", "family": "F17", "result": "xfail", "label": "No-op writes leave the file alone (`F17.noop-write-untouched`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-08-no-op-writes-leave-the-file-alone-f17-no.json", "failed_command": "test \"$(stat -c '%i %y' \"$WS\")\" = \"$STAMP\"" }, { "feature_id": "F17.paths-opaque", "family": "F17", "result": "pass", "label": "Paths are opaque strings (`F17.paths-opaque`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-09-paths-are-opaque-strings-f17-paths-opaqu.json", "failed_command": null }, { "feature_id": "F17.validation", "family": "F17", "result": "pass", "label": "Validation (`F17.validation`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-10-validation-f17-validation.json", "failed_command": null }, { "feature_id": "F17.add-parents", "family": "F17", "result": "pass", "label": "Register parents (`F17.add-parents`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-11-register-parents-f17-add-parents.json", "failed_command": null }, { "feature_id": "F17.parent-children", "family": "F17", "result": "pass", "label": "Children of a parent (`F17.parent-children`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-12-children-of-a-parent-f17-parent-children.json", "failed_command": null }, { "feature_id": "F17.delete-parent", "family": "F17", "result": "pass", "label": "Delete a parent (`F17.delete-parent`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-13-delete-a-parent-f17-delete-parent.json", "failed_command": null }, { "feature_id": "F17.lists-independent", "family": "F17", "result": "pass", "label": "Two independent lists (`F17.lists-independent`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-14-two-independent-lists-f17-lists-independ.json", "failed_command": null }, { "feature_id": "F17.persist-restart", "family": "F17", "result": "pass", "label": "Persistence across a restart (`F17.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-15-persistence-across-a-restart-f17-persist.json", "failed_command": null }, { "feature_id": "F17.auth-required", "family": "F17", "result": "pass", "label": "Session key required (`F17.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-16-session-key-required-f17-auth-required.json", "failed_command": null }, { "feature_id": "F17.concurrent-adds", "family": "F17", "result": "pass", "label": "Concurrent writes (`F17.concurrent-adds`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-17-concurrent-writes-f17-concurrent-adds.json", "failed_command": null }, { "feature_id": "F17.shared-dir", "family": "F17", "result": "pass", "label": "Two servers, one file (`F17.shared-dir`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-18-two-servers-one-file-f17-shared-dir.json", "failed_command": null }, { "feature_id": "F17.ts-client", "family": "F17", "result": "pass", "label": "TypeScript client (`F17.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-19-typescript-client-f17-ts-client.json", "failed_command": null }, { "feature_id": "F17.corrupted-file", "family": "F17", "result": "pass", "label": "Corrupted file (`F17.corrupted-file`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-20-corrupted-file-f17-corrupted-file.json", "failed_command": null }, { "feature_id": "F17.future-schema", "family": "F17", "result": "pass", "label": "Future or invalid schema (`F17.future-schema`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-21-future-or-invalid-schema-f17-future-sche.json", "failed_command": null }, { "feature_id": "F17.io-error", "family": "F17", "result": "pass", "label": "Unreadable file (`F17.io-error`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-22-unreadable-file-f17-io-error.json", "failed_command": null }, { "feature_id": "F17.noop-write-readonly", "family": "F17", "result": "fail", "label": "No-op writes on read-only storage (`F17.noop-write-readonly`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-23-no-op-writes-on-read-only-storage-f17-no.json", "failed_command": "chattr +i \"$PDIR\"" }, { "feature_id": "F17.persistence-location", "family": "F17", "result": "pass", "label": "Location without `OH_PERSISTENCE_DIR` (`F17.persistence-location`).", "run": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17", "transcript": "/workspace/verification-artifacts/runs/20261013-022059-b8c2-map-f17/evidence/map-run/F17-workspaces-registry-24-location-without-oh-persistence-dir-f17-.json", "failed_command": null }, { "feature_id": "F18.get-defaults", "family": "F18", "result": "pass", "label": "Fresh defaults (`F18.get-defaults`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-01-fresh-defaults-f18-get-defaults.json", "failed_command": null }, { "feature_id": "F18.agent-schema", "family": "F18", "result": "pass", "label": "Agent settings schema (`F18.agent-schema`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-02-agent-settings-schema-f18-agent-schema.json", "failed_command": null }, { "feature_id": "F18.conversation-schema", "family": "F18", "result": "pass", "label": "Conversation settings schema (`F18.conversation-schema`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-03-conversation-settings-schema-f18-convers.json", "failed_command": null }, { "feature_id": "F18.auth-required", "family": "F18", "result": "pass", "label": "Key required (`F18.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-04-key-required-f18-auth-required.json", "failed_command": null }, { "feature_id": "F18.patch-llm", "family": "F18", "result": "pass", "label": "Set the LLM (`F18.patch-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-05-set-the-llm-f18-patch-llm.json", "failed_command": null }, { "feature_id": "F18.expose-modes", "family": "F18", "result": "pass", "label": "Exposure modes (`F18.expose-modes`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-06-exposure-modes-f18-expose-modes.json", "failed_command": null }, { "feature_id": "F18.roundtrip", "family": "F18", "result": "pass", "label": "Post back what GET returned (`F18.roundtrip`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-07-post-back-what-get-returned-f18-roundtri.json", "failed_command": null }, { "feature_id": "F18.agent-diff-merge", "family": "F18", "result": "pass", "label": "Sparse agent diff and unset (`F18.agent-diff-merge`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-08-sparse-agent-diff-and-unset-f18-agent-di.json", "failed_command": null }, { "feature_id": "F18.conversation-diff", "family": "F18", "result": "pass", "label": "Conversation settings (`F18.conversation-diff`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-09-conversation-settings-f18-conversation-d.json", "failed_command": null }, { "feature_id": "F18.misc-settings-merge", "family": "F18", "result": "pass", "label": "Opaque misc settings (`F18.misc-settings-merge`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-10-opaque-misc-settings-f18-misc-settings-m.json", "failed_command": null }, { "feature_id": "F18.patch-validation", "family": "F18", "result": "pass", "label": "Rejected PATCH bodies (`F18.patch-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-11-rejected-patch-bodies-f18-patch-validati.json", "failed_command": null }, { "feature_id": "F18.active-profile-pointer", "family": "F18", "result": "pass", "label": "Active LLM profile pointer (`F18.active-profile-pointer`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-12-active-llm-profile-pointer-f18-active-pr.json", "failed_command": null }, { "feature_id": "F18.agent-kind-switch", "family": "F18", "result": "pass", "label": "Agent kind switch (`F18.agent-kind-switch`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-13-agent-kind-switch-f18-agent-kind-switch.json", "failed_command": null }, { "feature_id": "F18.secret-put-list-get", "family": "F18", "result": "pass", "label": "Upsert, list and read a secret (`F18.secret-put-list-get`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-14-upsert-list-and-read-a-secret-f18-secret.json", "failed_command": null }, { "feature_id": "F18.secret-delete", "family": "F18", "result": "pass", "label": "Delete a secret (`F18.secret-delete`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-15-delete-a-secret-f18-secret-delete.json", "failed_command": null }, { "feature_id": "F18.secret-name-validation", "family": "F18", "result": "pass", "label": "Secret names (`F18.secret-name-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-16-secret-names-f18-secret-name-validation.json", "failed_command": null }, { "feature_id": "F18.encrypted-at-rest", "family": "F18", "result": "pass", "label": "Encrypted at rest (`F18.encrypted-at-rest`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-17-encrypted-at-rest-f18-encrypted-at-rest.json", "failed_command": null }, { "feature_id": "F18.secret-scope-agent-profile", "family": "F18", "result": "pass", "label": "Secrets scoped by an agent profile (`F18.secret-scope-agent-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-18-secrets-scoped-by-an-agent-profile-f18-s.json", "failed_command": null }, { "feature_id": "F18.sdk-remote-workspace", "family": "F18", "result": "pass", "label": "SDK consumer (`F18.sdk-remote-workspace`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-19-sdk-consumer-f18-sdk-remote-workspace.json", "failed_command": null }, { "feature_id": "F18.sdk-clone-repos", "family": "F18", "result": "pass", "label": "SDK clones with the stored token (`F18.sdk-clone-repos`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-20-sdk-clones-with-the-stored-token-f18-sdk.json", "failed_command": null }, { "feature_id": "F18.ts-client", "family": "F18", "result": "pass", "label": "TypeScript client (`F18.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-21-typescript-client-f18-ts-client.json", "failed_command": null }, { "feature_id": "F18.persist-restart", "family": "F18", "result": "pass", "label": "Persistence across a restart (`F18.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-22-persistence-across-a-restart-f18-persist.json", "failed_command": null }, { "feature_id": "F18.corrupted-files", "family": "F18", "result": "pass", "label": "Corrupted files (`F18.corrupted-files`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-23-corrupted-files-f18-corrupted-files.json", "failed_command": null }, { "feature_id": "F18.no-cipher", "family": "F18", "result": "pass", "label": "No cipher (`F18.no-cipher`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-24-no-cipher-f18-no-cipher.json", "failed_command": null }, { "feature_id": "F18.secret-empty-value", "family": "F18", "result": "pass", "label": "Placeholder secret value (`F18.secret-empty-value`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-25-placeholder-secret-value-f18-secret-empt.json", "failed_command": null }, { "feature_id": "F18.secret-empty-value", "family": "F18", "result": "pass", "label": "Empty secret value (`F18.secret-empty-value`).", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-26-empty-secret-value-f18-secret-empty-valu.json", "failed_command": null }, { "feature_id": "F18.dangling-pointers", "family": "F18", "result": "xfail", "label": "Dangling agent-profile pointer (`F18.dangling-pointers`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-27-dangling-agent-profile-pointer-f18-dangl.json", "failed_command": "control-agent-server api PATCH /api/settings --json '{\"active_agent_profile_id\": \"00000000-0000-4000-8000-0000000000f1\"}' --expect 404,422 --save F18.dangling-pointers/agent-profile-id" }, { "feature_id": "F18.dangling-pointers", "family": "F18", "result": "xfail", "label": "Dangling meta-profile pointer (`F18.dangling-pointers`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-28-dangling-meta-profile-pointer-f18-dangli.json", "failed_command": "control-agent-server api PATCH /api/settings --json '{\"active_meta_profile\": \"qa-f18-no-router\"}' --expect 404,422 --save F18.dangling-pointers/meta-profile" }, { "feature_id": "F18.secret-scope-profile-errors", "family": "F18", "result": "xfail", "label": "Agent profile that no longer loads (`F18.secret-scope-profile-errors`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-29-agent-profile-that-no-longer-loads-f18-s.json", "failed_command": "control-agent-server api GET /api/settings/secrets --query \"agent_profile_id=$BROKEN\" --expect 4xx --save F18.secret-scope-profile-errors/unloadable" }, { "feature_id": "F18.secret-scope-profile-errors", "family": "F18", "result": "xfail", "label": "Agent-profile store busy (`F18.secret-scope-profile-errors`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-30-agent-profile-store-busy-f18-secret-scop.json", "failed_command": "control-agent-server api GET /api/settings/secrets --query agent_profile_id=00000000-0000-4000-8000-0000000000f1 --timeout 60 --expect 503 --save F18.secret-scope-profile-errors/secrets-busy" }, { "feature_id": "F18.secret-name-contract", "family": "F18", "result": "xfail", "label": "Documented status for an invalid name (`F18.secret-name-contract`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-31-documented-status-for-an-invalid-name-f1.json", "failed_command": "control-agent-server api GET /api/settings/secrets/1bad --expect \"${CLAIMED:-422}\" --save F18.secret-name-contract/invalid-name" }, { "feature_id": "F18.secret-key-rotation", "family": "F18", "result": "xfail", "label": "Cipher key rotation (`F18.secret-key-rotation`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18", "transcript": "/workspace/verification-artifacts/runs/20261013-021811-ba26-map-f18/evidence/map-run/F18-settings-and-secrets-32-cipher-key-rotation-f18-secret-key-rotat.json", "failed_command": "control-agent-server state cat home/.openhands/secrets.json --check custom_secrets.QA_F18_TOKEN.secret matches '^gAAAAA'" }, { "feature_id": "F19.auth", "family": "F19", "result": "pass", "label": "Auth (`F19.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-01-auth-f19-auth.json", "failed_command": null }, { "feature_id": "F19.settings-create", "family": "F19", "result": "pass", "label": "Create one server (`F19.settings-create`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-02-create-one-server-f19-settings-create.json", "failed_command": null }, { "feature_id": "F19.settings-validation", "family": "F19", "result": "pass", "label": "Invalid servers (`F19.settings-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-03-invalid-servers-f19-settings-validation.json", "failed_command": null }, { "feature_id": "F19.settings-empty-server", "family": "F19", "result": "xfail", "label": "Server with neither command nor url (`F19.settings-empty-server`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-04-server-with-neither-command-nor-url-f19-.json", "failed_command": "false" }, { "feature_id": "F19.settings-patch", "family": "F19", "result": "pass", "label": "Sparse patch (`F19.settings-patch`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-05-sparse-patch-f19-settings-patch.json", "failed_command": null }, { "feature_id": "F19.settings-delete", "family": "F19", "result": "pass", "label": "Delete one server (`F19.settings-delete`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-06-delete-one-server-f19-settings-delete.json", "failed_command": null }, { "feature_id": "F19.test-validation", "family": "F19", "result": "pass", "label": "Connection test body validation (`F19.test-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-07-connection-test-body-validation-f19-test.json", "failed_command": null }, { "feature_id": "F19.test-stdio", "family": "F19", "result": "pass", "label": "Test a stdio server (`F19.test-stdio`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-08-test-a-stdio-server-f19-test-stdio.json", "failed_command": null }, { "feature_id": "F19.test-tool-errors", "family": "F19", "result": "fail", "label": "Tool failures stay in `tool_result` (`F19.test-tool-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-09-tool-failures-stay-in-tool-result-f19-te.json", "failed_command": "control-agent-server api POST /api/mcp/test --timeout 20 --expect-max-ms 15000 --save F19.test-tool-errors/slow-tool --json \"$(jq -nc --argjson s \"$TOOLS_SRV\" '{server: $s, timeout: 3, tool_call: {name: \"qa_sleep\", arguments: {seconds: 30}}}')\" --expect 200 --check ok eq true --check tool_result.is_error eq true --check tool_result.text contains 'timed out after 3.0 seconds'" }, { "feature_id": "F19.test-connect-failures", "family": "F19", "result": "pass", "label": "Connection failures (`F19.test-connect-failures`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-10-connection-failures-f19-test-connect-fai.json", "failed_command": null }, { "feature_id": "F19.test-timeout", "family": "F19", "result": "pass", "label": "Handshake timeout (`F19.test-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-11-handshake-timeout-f19-test-timeout.json", "failed_command": null }, { "feature_id": "F19.test-encrypted-env", "family": "F19", "result": "pass", "label": "Encrypted env from settings (`F19.test-encrypted-env`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-12-encrypted-env-from-settings-f19-test-enc.json", "failed_command": null }, { "feature_id": "F19.test-remote-bearer", "family": "F19", "result": "pass", "label": "Remote server behind a bearer token (`F19.test-remote-bearer`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-13-remote-server-behind-a-bearer-token-f19-.json", "failed_command": null }, { "feature_id": "F19.oauth-start-errors", "family": "F19", "result": "pass", "label": "OAuth start refusals and early outcomes (`F19.oauth-start-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-14-oauth-start-refusals-and-early-outcomes-.json", "failed_command": null }, { "feature_id": "F19.oauth-unknown-job", "family": "F19", "result": "pass", "label": "Unknown jobs (`F19.oauth-unknown-job`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-15-unknown-jobs-f19-oauth-unknown-job.json", "failed_command": null }, { "feature_id": "F19.oauth-flow", "family": "F19", "result": "pass", "label": "Full OAuth consent (`F19.oauth-flow`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-16-full-oauth-consent-f19-oauth-flow.json", "failed_command": null }, { "feature_id": "F19.oauth-callback-replay", "family": "F19", "result": "xfail", "label": "Replayed callback (`F19.oauth-callback-replay`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-17-replayed-callback-f19-oauth-callback-rep.json", "failed_command": "control-agent-server api POST \"/api/mcp/oauth/callback/$J\" --json \"$RBODY\" --expect 200,4xx --save F19.oauth-callback-replay/replay" }, { "feature_id": "F19.oauth-callback-bad-port", "family": "F19", "result": "xfail", "label": "Port out of range (`F19.oauth-callback-bad-port`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-18-port-out-of-range-f19-oauth-callback-bad.json", "failed_command": "control-agent-server api POST \"/api/mcp/oauth/callback/$JB\" --json '{\"callback_url\":\"http://localhost:99999/callback?code=x\"}' --expect 400 --save F19.oauth-callback-bad-port/port-99999" }, { "feature_id": "F19.oauth-callback-validation", "family": "F19", "result": "pass", "label": "Callback URL checks on a live job (`F19.oauth-callback-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-19-callback-url-checks-on-a-live-job-f19-oa.json", "failed_command": null }, { "feature_id": "F19.oauth-consent-denied", "family": "F19", "result": "pass", "label": "Denied consent (`F19.oauth-consent-denied`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-20-denied-consent-f19-oauth-consent-denied.json", "failed_command": null }, { "feature_id": "F19.oauth-consent-timeout", "family": "F19", "result": "pass", "label": "Consent timeout (`F19.oauth-consent-timeout`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-21-consent-timeout-f19-oauth-consent-timeou.json", "failed_command": null }, { "feature_id": "F19.oauth-inline-writeback", "family": "F19", "result": "pass", "label": "Inline OAuth server refresh (`F19.oauth-inline-writeback`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-22-inline-oauth-server-refresh-f19-oauth-in.json", "failed_command": null }, { "feature_id": "F19.oauth-stored-state", "family": "F19", "result": "pass", "label": "Stored OAuth state (`F19.oauth-stored-state`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-23-stored-oauth-state-f19-oauth-stored-stat.json", "failed_command": null }, { "feature_id": "F19.disabled-server", "family": "F19", "result": "pass", "label": "Disabled servers stay off (`F19.disabled-server`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-24-disabled-servers-stay-off-f19-disabled-s.json", "failed_command": null }, { "feature_id": "F19.conversation-tool", "family": "F19", "result": "pass", "label": "The agent uses the MCP tool (`F19.conversation-tool`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-25-the-agent-uses-the-mcp-tool-f19-conversa.json", "failed_command": null }, { "feature_id": "F19.oauth-refresh-writeback", "family": "F19", "result": "pass", "label": "Stored OAuth token refresh (`F19.oauth-refresh-writeback`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-26-stored-oauth-token-refresh-f19-oauth-ref.json", "failed_command": null }, { "feature_id": "F19.conversation-secrets-at-rest", "family": "F19", "result": "pass", "label": "Credentials at rest (`F19.conversation-secrets-at-rest`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-27-credentials-at-rest-f19-conversation-sec.json", "failed_command": null }, { "feature_id": "F19.restart", "family": "F19", "result": "pass", "label": "Restart (`F19.restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19", "transcript": "/workspace/verification-artifacts/runs/20261013-021940-8441-map-f19/evidence/map-run/F19-mcp-28-restart-f19-restart.json", "failed_command": null }, { "feature_id": "F20.auth-required", "family": "F20", "result": "pass", "label": "Key required (`F20.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-01-key-required-f20-auth-required.json", "failed_command": null }, { "feature_id": "F20.save", "family": "F20", "result": "pass", "label": "Save (`F20.save`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-02-save-f20-save.json", "failed_command": null }, { "feature_id": "F20.key-encrypted-at-rest", "family": "F20", "result": "pass", "label": "Encrypted at rest (`F20.key-encrypted-at-rest`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-03-encrypted-at-rest-f20-key-encrypted-at-r.json", "failed_command": null }, { "feature_id": "F20.get-secrets", "family": "F20", "result": "pass", "label": "Reading the key (`F20.get-secrets`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-04-reading-the-key-f20-get-secrets.json", "failed_command": null }, { "feature_id": "F20.save-encrypted-roundtrip", "family": "F20", "result": "pass", "label": "Encrypted round trip (`F20.save-encrypted-roundtrip`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-05-encrypted-round-trip-f20-save-encrypted-.json", "failed_command": null }, { "feature_id": "F20.save-without-secrets", "family": "F20", "result": "pass", "label": "Save without secrets (`F20.save-without-secrets`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-06-save-without-secrets-f20-save-without-se.json", "failed_command": null }, { "feature_id": "F20.not-found", "family": "F20", "result": "pass", "label": "Unknown names (`F20.not-found`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-07-unknown-names-f20-not-found.json", "failed_command": null }, { "feature_id": "F20.name-rules", "family": "F20", "result": "pass", "label": "Name rules (`F20.name-rules`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-08-name-rules-f20-name-rules.json", "failed_command": null }, { "feature_id": "F20.corrupt-files", "family": "F20", "result": "pass", "label": "Corrupted files (`F20.corrupt-files`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-09-corrupted-files-f20-corrupt-files.json", "failed_command": null }, { "feature_id": "F20.store-busy", "family": "F20", "result": "pass", "label": "Store busy (`F20.store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-10-store-busy-f20-store-busy.json", "failed_command": null }, { "feature_id": "F20.store-busy-responsive", "family": "F20", "result": "xfail", "label": "Lock wait blocks the server (`F20.store-busy-responsive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-11-lock-wait-blocks-the-server-f20-store-bu.json", "failed_command": "jq -e '.elapsed_ms <= 3000' \"$F20/busy-alive.json\"" }, { "feature_id": "F20.profile-limit", "family": "F20", "result": "pass", "label": "Profile limit (`F20.profile-limit`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-12-profile-limit-f20-profile-limit.json", "failed_command": null }, { "feature_id": "F20.validate-live", "family": "F20", "result": "pass", "label": "Validate a working config (`F20.validate-live`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-13-validate-a-working-config-f20-validate-l.json", "failed_command": null }, { "feature_id": "F20.validate-rejects", "family": "F20", "result": "pass", "label": "Validate rejects (`F20.validate-rejects`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-14-validate-rejects-f20-validate-rejects.json", "failed_command": null }, { "feature_id": "F20.validate-bad-key-type", "family": "F20", "result": "xfail", "label": "DeepSeek 401 type (`F20.validate-bad-key-type`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-15-deepseek-401-type-f20-validate-bad-key-t.json", "failed_command": "jq -e '.response.body.error.type == \"LLMAuthenticationError\"' \"$F20/bad-key.json\"" }, { "feature_id": "F20.validate-provider-errors", "family": "F20", "result": "pass", "label": "Provider failures (`F20.validate-provider-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-16-provider-failures-f20-validate-provider-.json", "failed_command": null }, { "feature_id": "F20.validate-subscription-login", "family": "F20", "result": "pass", "label": "Subscription without a login (`F20.validate-subscription-login`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-17-subscription-without-a-login-f20-validat.json", "failed_command": null }, { "feature_id": "F20.validate-unreachable-bounded", "family": "F20", "result": "xfail", "label": "Unreachable endpoint is slow (`F20.validate-unreachable-bounded`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-18-unreachable-endpoint-is-slow-f20-validat.json", "failed_command": "jq -e '.elapsed_ms <= 30000' \"$F20/unreachable.json\"" }, { "feature_id": "F20.validate-hang-bounded", "family": "F20", "result": "xfail", "label": "Hanging provider is slower (`F20.validate-hang-bounded`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-19-hanging-provider-is-slower-f20-validate-.json", "failed_command": "test \"$RC\" = 0" }, { "feature_id": "F20.activate", "family": "F20", "result": "pass", "label": "Activate (`F20.activate`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-20-activate-f20-activate.json", "failed_command": null }, { "feature_id": "F20.activate-conversation", "family": "F20", "result": "pass", "label": "Activation drives conversations (`F20.activate-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-21-activation-drives-conversations-f20-acti.json", "failed_command": null }, { "feature_id": "F20.activate-acp", "family": "F20", "result": "pass", "label": "ACP settings (`F20.activate-acp`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-22-acp-settings-f20-activate-acp.json", "failed_command": null }, { "feature_id": "F20.sdk-get-llm", "family": "F20", "result": "pass", "label": "SDK consumer (`F20.sdk-get-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-23-sdk-consumer-f20-sdk-get-llm.json", "failed_command": null }, { "feature_id": "F20.ts-client", "family": "F20", "result": "pass", "label": "TypeScript client (`F20.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-24-typescript-client-f20-ts-client.json", "failed_command": null }, { "feature_id": "F20.activate-dangling-provider", "family": "F20", "result": "pass", "label": "Dangling provider link (`F20.activate-dangling-provider`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-25-dangling-provider-link-f20-activate-dang.json", "failed_command": null }, { "feature_id": "F20.activate-linked-provider", "family": "F20", "result": "pass", "label": "Linked provider (`F20.activate-linked-provider`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-26-linked-provider-f20-activate-linked-prov.json", "failed_command": null }, { "feature_id": "F20.sdk-get-llm-linked", "family": "F20", "result": "pass", "label": "SDK on a linked profile (`F20.sdk-get-llm-linked`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-27-sdk-on-a-linked-profile-f20-sdk-get-llm-.json", "failed_command": null }, { "feature_id": "F20.no-cipher", "family": "F20", "result": "pass", "label": "No cipher (`F20.no-cipher`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-28-no-cipher-f20-no-cipher.json", "failed_command": null }, { "feature_id": "F20.persist-restart", "family": "F20", "result": "pass", "label": "Restart (`F20.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-29-restart-f20-persist-restart.json", "failed_command": null }, { "feature_id": "F20.rename", "family": "F20", "result": "pass", "label": "Rename (`F20.rename`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-30-rename-f20-rename.json", "failed_command": null }, { "feature_id": "F20.rename-errors", "family": "F20", "result": "pass", "label": "Rename errors (`F20.rename-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-31-rename-errors-f20-rename-errors.json", "failed_command": null }, { "feature_id": "F20.delete-guarded", "family": "F20", "result": "pass", "label": "Guarded delete (`F20.delete-guarded`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-32-guarded-delete-f20-delete-guarded.json", "failed_command": null }, { "feature_id": "F20.delete", "family": "F20", "result": "pass", "label": "Delete (`F20.delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-33-delete-f20-delete.json", "failed_command": null }, { "feature_id": "F20.json-alias-guard", "family": "F20", "result": "xfail", "label": "Delete through the `.json` alias (`F20.json-alias-guard`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-34-delete-through-the-json-alias-f20-json-a.json", "failed_command": "jq -e '.status == 409' \"$F20/alias-delete.json\"" }, { "feature_id": "F20.secret-key-rotation", "family": "F20", "result": "xfail", "label": "Secret key rotation (`F20.secret-key-rotation`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20", "transcript": "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-35-secret-key-rotation-f20-secret-key-rotat.json", "failed_command": "control-agent-server api POST /api/profiles/qa-rotate/activate --expect 4xx --save F20.secret-key-rotation/activate" }, { "feature_id": "F21.auth-required", "family": "F21", "result": "pass", "label": "No key, no access (`F21.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-01-no-key-no-access-f21-auth-required.json", "failed_command": null }, { "feature_id": "F21.lazy-seed", "family": "F21", "result": "pass", "label": "Lazy seed (`F21.lazy-seed`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-02-lazy-seed-f21-lazy-seed.json", "failed_command": null }, { "feature_id": "F21.seed-no-ghost-llm", "family": "F21", "result": "pass", "label": "No ghost LLM profile (`F21.seed-no-ghost-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-03-no-ghost-llm-profile-f21-seed-no-ghost-l.json", "failed_command": null }, { "feature_id": "F21.seed-acp", "family": "F21", "result": "pass", "label": "ACP seed (`F21.seed-acp`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-04-acp-seed-f21-seed-acp.json", "failed_command": null }, { "feature_id": "F21.save-create", "family": "F21", "result": "pass", "label": "Create (`F21.save-create`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-05-create-f21-save-create.json", "failed_command": null }, { "feature_id": "F21.save-overwrite", "family": "F21", "result": "pass", "label": "Overwrite (`F21.save-overwrite`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-06-overwrite-f21-save-overwrite.json", "failed_command": null }, { "feature_id": "F21.acp-kind", "family": "F21", "result": "pass", "label": "ACP profile (`F21.acp-kind`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-07-acp-profile-f21-acp-kind.json", "failed_command": null }, { "feature_id": "F21.save-validation", "family": "F21", "result": "pass", "label": "Save validation (`F21.save-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-08-save-validation-f21-save-validation.json", "failed_command": null }, { "feature_id": "F21.name-rules", "family": "F21", "result": "pass", "label": "Name rules (`F21.name-rules`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-09-name-rules-f21-name-rules.json", "failed_command": null }, { "feature_id": "F21.not-found", "family": "F21", "result": "pass", "label": "Unknown names and ids (`F21.not-found`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-10-unknown-names-and-ids-f21-not-found.json", "failed_command": null }, { "feature_id": "F21.activate", "family": "F21", "result": "pass", "label": "Activate by id (`F21.activate`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-11-activate-by-id-f21-activate.json", "failed_command": null }, { "feature_id": "F21.activate-corrupt-settings", "family": "F21", "result": "pass", "label": "Activate over a corrupt settings file (`F21.activate-corrupt-settings`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-12-activate-over-a-corrupt-settings-file-f2.json", "failed_command": null }, { "feature_id": "F21.rename", "family": "F21", "result": "pass", "label": "Rename (`F21.rename`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-13-rename-f21-rename.json", "failed_command": null }, { "feature_id": "F21.delete", "family": "F21", "result": "pass", "label": "Delete (`F21.delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-14-delete-f21-delete.json", "failed_command": null }, { "feature_id": "F21.delete-active-acp-reset", "family": "F21", "result": "pass", "label": "Active ACP reset (`F21.delete-active-acp-reset`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-15-active-acp-reset-f21-delete-active-acp-r.json", "failed_command": null }, { "feature_id": "F21.materialize-valid", "family": "F21", "result": "pass", "label": "Materialize a resolvable profile (`F21.materialize-valid`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-16-materialize-a-resolvable-profile-f21-mat.json", "failed_command": null }, { "feature_id": "F21.materialize-dangling", "family": "F21", "result": "pass", "label": "Dangling references (`F21.materialize-dangling`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-17-dangling-references-f21-materialize-dang.json", "failed_command": null }, { "feature_id": "F21.materialize-draft", "family": "F21", "result": "pass", "label": "Draft preview (`F21.materialize-draft`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-18-draft-preview-f21-materialize-draft.json", "failed_command": null }, { "feature_id": "F21.limit", "family": "F21", "result": "pass", "label": "At most 50 (`F21.limit`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-19-at-most-50-f21-limit.json", "failed_command": null }, { "feature_id": "F21.corrupt-file", "family": "F21", "result": "pass", "label": "Corrupt file (`F21.corrupt-file`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-20-corrupt-file-f21-corrupt-file.json", "failed_command": null }, { "feature_id": "F21.conversation-start", "family": "F21", "result": "pass", "label": "Start a conversation from a profile (`F21.conversation-start`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-21-start-a-conversation-from-a-profile-f21-.json", "failed_command": null }, { "feature_id": "F21.secret-scope", "family": "F21", "result": "pass", "label": "Secret scope (`F21.secret-scope`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-22-secret-scope-f21-secret-scope.json", "failed_command": null }, { "feature_id": "F21.launch-additions", "family": "F21", "result": "pass", "label": "Launch additions (`F21.launch-additions`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-23-launch-additions-f21-launch-additions.json", "failed_command": null }, { "feature_id": "F21.ts-client", "family": "F21", "result": "pass", "label": "TypeScript client (`F21.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-24-typescript-client-f21-ts-client.json", "failed_command": null }, { "feature_id": "F21.restart", "family": "F21", "result": "pass", "label": "Restart (`F21.restart`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-25-restart-f21-restart.json", "failed_command": null }, { "feature_id": "F21.acp-skill-sourcing", "family": "F21", "result": "pass", "label": "ACP skill sourcing (`F21.acp-skill-sourcing`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-26-acp-skill-sourcing-f21-acp-skill-sourcin.json", "failed_command": null }, { "feature_id": "F21.store-busy", "family": "F21", "result": "pass", "label": "Store busy (`F21.store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-27-store-busy-f21-store-busy.json", "failed_command": null }, { "feature_id": "F21.launch-store-busy", "family": "F21", "result": "pass", "label": "Busy store at conversation start (`F21.launch-store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-28-busy-store-at-conversation-start-f21-lau.json", "failed_command": null }, { "feature_id": "F21.busy-blocks-server", "family": "F21", "result": "xfail", "label": "Lock wait stalls the server (`F21.busy-blocks-server`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-29-lock-wait-stalls-the-server-f21-busy-blo.json", "failed_command": "control-agent-server api GET /alive --auth none --expect 200 --timeout 60 --expect-max-ms 2000 --save F21.busy-blocks-server/alive" }, { "feature_id": "F21.dangling-llm-report", "family": "F21", "result": "xfail", "label": "How a dangling LLM reference is reported (`F21.dangling-llm-report`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-30-how-a-dangling-llm-reference-is-reported.json", "failed_command": "jq -e '.dangling_llm_profile_ref == \"qa-f21-missing\"' \"$RAW\" > /dev/null" }, { "feature_id": "F21.ts-payload-types", "family": "F21", "result": "xfail", "label": "TypeScript payload types (`F21.ts-payload-types`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-31-typescript-payload-types-f21-ts-payload-.json", "failed_command": "$TSC \"$D/exported.ts\"" }, { "feature_id": "F21.json-suffix-identity", "family": "F21", "result": "xfail", "label": "`.json` names lose their identity (`F21.json-suffix-identity`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21", "transcript": "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-32-json-names-lose-their-identity-f21-json-.json", "failed_command": "control-agent-server api GET /api/agent-profiles/qa-f21-suffix.json --check profile.id eq \"$J1\" --check profile.revision eq 1" }, { "feature_id": "F22.auth-required", "family": "F22", "result": "pass", "label": "Key required (`F22.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-01-key-required-f22-auth-required.json", "failed_command": null }, { "feature_id": "F22.save-classes", "family": "F22", "result": "pass", "label": "Save in classes mode (`F22.save-classes`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-02-save-in-classes-mode-f22-save-classes.json", "failed_command": null }, { "feature_id": "F22.save-direct", "family": "F22", "result": "pass", "label": "Save in direct mode (`F22.save-direct`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-03-save-in-direct-mode-f22-save-direct.json", "failed_command": null }, { "feature_id": "F22.validation", "family": "F22", "result": "pass", "label": "Validation (`F22.validation`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-04-validation-f22-validation.json", "failed_command": null }, { "feature_id": "F22.empty-router", "family": "F22", "result": "xfail", "label": "Router with no route (`F22.empty-router`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-05-router-with-no-route-f22-empty-router-kn.json", "failed_command": "control-agent-server api POST /api/meta-profiles/qa-empty --json '{\"classifier_model\": \"deepseek-flash\"}' --expect 422 --save F22.empty-router/save" }, { "feature_id": "F22.overwrite", "family": "F22", "result": "pass", "label": "Overwrite (`F22.overwrite`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-06-overwrite-f22-overwrite.json", "failed_command": null }, { "feature_id": "F22.name-rules", "family": "F22", "result": "pass", "label": "Name rules (`F22.name-rules`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-07-name-rules-f22-name-rules.json", "failed_command": null }, { "feature_id": "F22.not-found", "family": "F22", "result": "pass", "label": "Unknown names (`F22.not-found`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-08-unknown-names-f22-not-found.json", "failed_command": null }, { "feature_id": "F22.activate", "family": "F22", "result": "pass", "label": "Activate (`F22.activate`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-09-activate-f22-activate.json", "failed_command": null }, { "feature_id": "F22.route-classes", "family": "F22", "result": "pass", "label": "Classes-mode routing (`F22.route-classes`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-10-classes-mode-routing-f22-route-classes.json", "failed_command": null }, { "feature_id": "F22.route-direct", "family": "F22", "result": "pass", "label": "Direct routing (`F22.route-direct`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-11-direct-routing-f22-route-direct.json", "failed_command": null }, { "feature_id": "F22.route-missing-profile", "family": "F22", "result": "pass", "label": "Missing LLM profile (`F22.route-missing-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-12-missing-llm-profile-f22-route-missing-pr.json", "failed_command": null }, { "feature_id": "F22.persist-restart", "family": "F22", "result": "pass", "label": "Restart (`F22.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-13-restart-f22-persist-restart.json", "failed_command": null }, { "feature_id": "F22.delete", "family": "F22", "result": "pass", "label": "Delete (`F22.delete`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-14-delete-f22-delete.json", "failed_command": null }, { "feature_id": "F22.delete-active-clears", "family": "F22", "result": "pass", "label": "Delete the active one (`F22.delete-active-clears`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-15-delete-the-active-one-f22-delete-active-.json", "failed_command": null }, { "feature_id": "F22.acp-clears-pointer", "family": "F22", "result": "pass", "label": "ACP clears the pointer (`F22.acp-clears-pointer`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-16-acp-clears-the-pointer-f22-acp-clears-po.json", "failed_command": null }, { "feature_id": "F22.activate-acp", "family": "F22", "result": "xfail", "label": "Activation on ACP settings (`F22.activate-acp`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-17-activation-on-acp-settings-f22-activate-.json", "failed_command": "jq -e '.status >= 400 and .status < 500' \"$F22/acp-activate.json\"" }, { "feature_id": "F22.corrupt-files", "family": "F22", "result": "pass", "label": "Corrupted files (`F22.corrupt-files`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-18-corrupted-files-f22-corrupt-files.json", "failed_command": null }, { "feature_id": "F22.corrupt-settings", "family": "F22", "result": "xfail", "label": "Unreadable settings (`F22.corrupt-settings`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-19-unreadable-settings-f22-corrupt-settings.json", "failed_command": "jq -e '.status == 409' \"$F22/corrupt-settings-activate.json\"" }, { "feature_id": "F22.store-busy", "family": "F22", "result": "fail", "label": "Store busy (`F22.store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-20-store-busy-f22-store-busy.json", "failed_command": "test $((SECONDS - T0)) -le 33" }, { "feature_id": "F22.store-busy-responsive", "family": "F22", "result": "xfail", "label": "Lock wait blocks the server (`F22.store-busy-responsive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-21-lock-wait-blocks-the-server-f22-store-bu.json", "failed_command": "test \"$SLOW\" = 0" }, { "feature_id": "F22.limit", "family": "F22", "result": "pass", "label": "Limit (`F22.limit`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-22-limit-f22-limit.json", "failed_command": null }, { "feature_id": "F22.ts-client", "family": "F22", "result": "pass", "label": "TypeScript client (`F22.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-23-typescript-client-f22-ts-client.json", "failed_command": null }, { "feature_id": "F22.persistence-dir", "family": "F22", "result": "xfail", "label": "Persistence directory with `~` (`F22.persistence-dir`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "transcript": "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-24-persistence-directory-with-f22-persisten.json", "failed_command": "control-agent-server state ls 'home/qa-f22-tilde/meta-profiles/qa-tilde.json' --run \"$T\" --expect-count 1" }, { "feature_id": "F23.auth-required", "family": "F23", "result": "pass", "label": "Key required (`F23.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-01-key-required-f23-auth-required.json", "failed_command": null }, { "feature_id": "F23.providers", "family": "F23", "result": "pass", "label": "Providers (`F23.providers`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-02-providers-f23-providers.json", "failed_command": null }, { "feature_id": "F23.models", "family": "F23", "result": "pass", "label": "All models (`F23.models`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-03-all-models-f23-models.json", "failed_command": null }, { "feature_id": "F23.models-filter", "family": "F23", "result": "pass", "label": "Provider filter (`F23.models-filter`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-04-provider-filter-f23-models-filter.json", "failed_command": null }, { "feature_id": "F23.models-no-bedrock", "family": "F23", "result": "xfail", "label": "No Bedrock ids under a provider (`F23.models-no-bedrock`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-05-no-bedrock-ids-under-a-provider-f23-mode.json", "failed_command": "jq -e 'map(select(test(\"^deepseek\\\\.\"))) | debug | length == 0'" }, { "feature_id": "F23.verified-models", "family": "F23", "result": "pass", "label": "Verified models (`F23.verified-models`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-06-verified-models-f23-verified-models.json", "failed_command": null }, { "feature_id": "F23.catalog-ts-client", "family": "F23", "result": "pass", "label": "TypeScript client (`F23.catalog-ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-07-typescript-client-f23-catalog-ts-client.json", "failed_command": null }, { "feature_id": "F23.conn-create", "family": "F23", "result": "pass", "label": "Create (`F23.conn-create`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-08-create-f23-conn-create.json", "failed_command": null }, { "feature_id": "F23.conn-key-encrypted", "family": "F23", "result": "pass", "label": "Key hidden and encrypted (`F23.conn-key-encrypted`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-09-key-hidden-and-encrypted-f23-conn-key-en.json", "failed_command": null }, { "feature_id": "F23.no-cipher", "family": "F23", "result": "pass", "label": "No cipher (`F23.no-cipher`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-10-no-cipher-f23-no-cipher.json", "failed_command": null }, { "feature_id": "F23.conn-create-validation", "family": "F23", "result": "pass", "label": "Create validation (`F23.conn-create-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-11-create-validation-f23-conn-create-valida.json", "failed_command": null }, { "feature_id": "F23.conn-create-blank-key", "family": "F23", "result": "xfail", "label": "Blank keys on create (`F23.conn-create-blank-key`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-12-blank-keys-on-create-f23-conn-create-bla.json", "failed_command": "test \"$S_SPACE $S_MASK\" = '422 422'" }, { "feature_id": "F23.linked-run", "family": "F23", "result": "pass", "label": "Linked profile runs on the connection (`F23.linked-run`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-13-linked-profile-runs-on-the-connection-f2.json", "failed_command": null }, { "feature_id": "F23.conn-update", "family": "F23", "result": "pass", "label": "Update (`F23.conn-update`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-14-update-f23-conn-update.json", "failed_command": null }, { "feature_id": "F23.conn-update-validation", "family": "F23", "result": "pass", "label": "Update validation (`F23.conn-update-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-15-update-validation-f23-conn-update-valida.json", "failed_command": null }, { "feature_id": "F23.conn-patch-blank-key", "family": "F23", "result": "xfail", "label": "Blank keys on update (`F23.conn-patch-blank-key`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-16-blank-keys-on-update-f23-conn-patch-blan.json", "failed_command": "test \"$APPLIED\" = '[sk-qa-f23-default][sk-qa-f23-default][sk-qa-f23-default]'" }, { "feature_id": "F23.conn-rotate-key", "family": "F23", "result": "pass", "label": "Rotate the key (`F23.conn-rotate-key`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-17-rotate-the-key-f23-conn-rotate-key.json", "failed_command": null }, { "feature_id": "F23.validate-linked-draft", "family": "F23", "result": "xfail", "label": "Validate a linked draft (`F23.validate-linked-draft`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-18-validate-a-linked-draft-f23-validate-lin.json", "failed_command": "control-agent-server api POST /api/profiles/qa-f23-draft/validate --timeout 120 --expect 200 --json \"{\\\"llm\\\": {\\\"model\\\": \\\"deepseek/deepseek-flash\\\", \\\"provider_connection_id\\\": \\\"$CONN\\\", \\\"num_retries\\\": 0}}\" --save F23.validate-linked-draft/linked --check valid eq true --check error eq null" }, { "feature_id": "F23.persist-restart", "family": "F23", "result": "pass", "label": "Restart (`F23.persist-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-19-restart-f23-persist-restart.json", "failed_command": null }, { "feature_id": "F23.strict-decrypt", "family": "F23", "result": "pass", "label": "Secret key changed (`F23.strict-decrypt`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-20-secret-key-changed-f23-strict-decrypt.json", "failed_command": null }, { "feature_id": "F23.corrupt-file", "family": "F23", "result": "pass", "label": "Corrupted file (`F23.corrupt-file`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-21-corrupted-file-f23-corrupt-file.json", "failed_command": null }, { "feature_id": "F23.store-busy", "family": "F23", "result": "pass", "label": "Store busy (`F23.store-busy`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-22-store-busy-f23-store-busy.json", "failed_command": null }, { "feature_id": "F23.store-busy-responsive", "family": "F23", "result": "xfail", "label": "Lock wait blocks the server (`F23.store-busy-responsive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-23-lock-wait-blocks-the-server-f23-store-bu.json", "failed_command": "control-agent-server api GET /alive --auth none --expect 200 --expect-max-ms 3000 --timeout 60 --save F23.store-busy-responsive/alive" }, { "feature_id": "F23.delete-guard", "family": "F23", "result": "pass", "label": "Delete guard (`F23.delete-guard`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-24-delete-guard-f23-delete-guard.json", "failed_command": null }, { "feature_id": "F23.conn-delete", "family": "F23", "result": "pass", "label": "Delete (`F23.conn-delete`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-25-delete-f23-conn-delete.json", "failed_command": null }, { "feature_id": "F23.conn-limit", "family": "F23", "result": "pass", "label": "Limit (`F23.conn-limit`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-26-limit-f23-conn-limit.json", "failed_command": null }, { "feature_id": "F23.acp-discovery-errors", "family": "F23", "result": "pass", "label": "ACP discovery errors (`F23.acp-discovery-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "transcript": "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-27-acp-discovery-errors-f23-acp-discovery-e.json", "failed_command": null }, { "feature_id": "F24.models", "family": "F24", "result": "pass", "label": "Subscription models (`F24.models`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-01-subscription-models-f24-models.json", "failed_command": null }, { "feature_id": "F24.status-disconnected", "family": "F24", "result": "pass", "label": "Status with no login (`F24.status-disconnected`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-02-status-with-no-login-f24-status-disconne.json", "failed_command": null }, { "feature_id": "F24.status-connected", "family": "F24", "result": "pass", "label": "Status with a stored login (`F24.status-connected`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-03-status-with-a-stored-login-f24-status-co.json", "failed_command": null }, { "feature_id": "F24.auth-required", "family": "F24", "result": "pass", "label": "Key required (`F24.auth-required`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-04-key-required-f24-auth-required.json", "failed_command": null }, { "feature_id": "F24.status-restart", "family": "F24", "result": "pass", "label": "Login across a restart (`F24.status-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-05-login-across-a-restart-f24-status-restar.json", "failed_command": null }, { "feature_id": "F24.logout", "family": "F24", "result": "pass", "label": "Logout (`F24.logout`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-06-logout-f24-logout.json", "failed_command": null }, { "feature_id": "F24.status-refresh-rejected", "family": "F24", "result": "pass", "label": "Rejected refresh (`F24.status-refresh-rejected`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-07-rejected-refresh-f24-status-refresh-reje.json", "failed_command": null }, { "feature_id": "F24.status-invalid-file", "family": "F24", "result": "pass", "label": "Invalid credentials file (`F24.status-invalid-file`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-08-invalid-credentials-file-f24-status-inva.json", "failed_command": null }, { "feature_id": "F24.device-poll-invalid", "family": "F24", "result": "pass", "label": "Malformed polls (`F24.device-poll-invalid`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-09-malformed-polls-f24-device-poll-invalid.json", "failed_command": null }, { "feature_id": "F24.device-poll-unknown", "family": "F24", "result": "pass", "label": "Unknown token (`F24.device-poll-unknown`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-10-unknown-token-f24-device-poll-unknown.json", "failed_command": null }, { "feature_id": "F24.device-start", "family": "F24", "result": "pass", "label": "Start a login (`F24.device-start`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-11-start-a-login-f24-device-start.json", "failed_command": null }, { "feature_id": "F24.device-poll-pending", "family": "F24", "result": "pass", "label": "Pending login (`F24.device-poll-pending`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-12-pending-login-f24-device-poll-pending.json", "failed_command": null }, { "feature_id": "F24.logout-cancels-pending", "family": "F24", "result": "pass", "label": "Logout cancels pending logins (`F24.logout-cancels-pending`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-13-logout-cancels-pending-logins-f24-logout.json", "failed_command": null }, { "feature_id": "F24.pending-lost-on-restart", "family": "F24", "result": "pass", "label": "Pending logins and restarts (`F24.pending-lost-on-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-14-pending-logins-and-restarts-f24-pending-.json", "failed_command": null }, { "feature_id": "F24.ts-client", "family": "F24", "result": "pass", "label": "TypeScript client (`F24.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-15-typescript-client-f24-ts-client.json", "failed_command": null }, { "feature_id": "F24.offline-safe", "family": "F24", "result": "pass", "label": "Offline-safe routes (`F24.offline-safe`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-16-offline-safe-routes-f24-offline-safe.json", "failed_command": null }, { "feature_id": "F24.device-poll-outage", "family": "F24", "result": "pass", "label": "Outage during a pending login (`F24.device-poll-outage`).", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-17-outage-during-a-pending-login-f24-device.json", "failed_command": null }, { "feature_id": "F24.status-offline", "family": "F24", "result": "xfail", "label": "Status with an expired login offline (`F24.status-offline`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-18-status-with-an-expired-login-offline-f24.json", "failed_command": "control-agent-server api GET /api/llm/subscription/openai/status --run \"$OFF\" --expect 200 --check connected eq false --save F24.status-offline/status" }, { "feature_id": "F24.device-start-offline", "family": "F24", "result": "xfail", "label": "Start and poll a login offline (`F24.device-start-offline`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-19-start-and-poll-a-login-offline-f24-devic.json", "failed_command": "control-agent-server api POST /api/llm/subscription/openai/device/start --run \"$OUTAGE\" --expect 502,503,504 --save F24.device-start-offline/start" }, { "feature_id": "F24.login-complete", "family": "F24", "result": "blocked", "label": "Complete a login (`F24.login-complete`), blocked.", "run": "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "transcript": null, "failed_command": null }, { "feature_id": "F25.auth", "family": "F25", "result": "pass", "label": "Session key required (`F25.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-01-session-key-required-f25-auth.json", "failed_command": null }, { "feature_id": "F25.public-repo", "family": "F25", "result": "pass", "label": "The real public repository (`F25.public-repo`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-02-the-real-public-repository-f25-public-re.json", "failed_command": null }, { "feature_id": "F25.marketplace-public", "family": "F25", "result": "xfail", "label": "Public marketplace catalog (`F25.marketplace-public`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-03-public-marketplace-catalog-f25-marketpla.json", "failed_command": "control-agent-server api GET /api/skills/marketplace --timeout 180 --expect 200 --check skills len-ge 1 --save F25.marketplace-public/catalog" }, { "feature_id": "F25.catalog-project", "family": "F25", "result": "pass", "label": "Project skills (`F25.catalog-project`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-04-project-skills-f25-catalog-project.json", "failed_command": null }, { "feature_id": "F25.catalog-user-precedence", "family": "F25", "result": "pass", "label": "User skills and precedence (`F25.catalog-user-precedence`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-05-user-skills-and-precedence-f25-catalog-u.json", "failed_command": null }, { "feature_id": "F25.catalog-server-sources", "family": "F25", "result": "pass", "label": "Sandbox and organization sources (`F25.catalog-server-sources`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-06-sandbox-and-organization-sources-f25-cat.json", "failed_command": null }, { "feature_id": "F25.catalog-validation", "family": "F25", "result": "pass", "label": "Request validation (`F25.catalog-validation`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-07-request-validation-f25-catalog-validatio.json", "failed_command": null }, { "feature_id": "F25.install-local", "family": "F25", "result": "pass", "label": "Install from a local directory (`F25.install-local`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-08-install-from-a-local-directory-f25-insta.json", "failed_command": null }, { "feature_id": "F25.enable-disable", "family": "F25", "result": "pass", "label": "Disable and enable (`F25.enable-disable`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-09-disable-and-enable-f25-enable-disable.json", "failed_command": null }, { "feature_id": "F25.install-conflict", "family": "F25", "result": "pass", "label": "Already installed (`F25.install-conflict`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-10-already-installed-f25-install-conflict.json", "failed_command": null }, { "feature_id": "F25.install-errors", "family": "F25", "result": "pass", "label": "Install errors (`F25.install-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-11-install-errors-f25-install-errors.json", "failed_command": null }, { "feature_id": "F25.install-bad-frontmatter", "family": "F25", "result": "xfail", "label": "Malformed frontmatter (`F25.install-bad-frontmatter`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-12-malformed-frontmatter-f25-install-bad-fr.json", "failed_command": "control-agent-server api POST /api/skills/install --json '{\"source\": \"'\"$F/qa-f25-badyaml\"'\"}' --expect 422 --save F25.install-bad-frontmatter/install" }, { "feature_id": "F25.install-git", "family": "F25", "result": "pass", "label": "Install from a git source (`F25.install-git`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-13-install-from-a-git-source-f25-install-gi.json", "failed_command": null }, { "feature_id": "F25.sdk-load-skills", "family": "F25", "result": "pass", "label": "SDK consumer (`F25.sdk-load-skills`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-14-sdk-consumer-f25-sdk-load-skills.json", "failed_command": null }, { "feature_id": "F25.install-github", "family": "F25", "result": "pass", "label": "Install from GitHub (`F25.install-github`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-15-install-from-github-f25-install-github.json", "failed_command": null }, { "feature_id": "F25.install-documented-sources", "family": "F25", "result": "xfail", "label": "Documented install sources (`F25.install-documented-sources`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-16-documented-install-sources-f25-install-d.json", "failed_command": "control-agent-server api POST /api/skills/install --json '{\"source\": \"'\"$SRC\"'\"}' --timeout 240 --expect 200 --check name eq github --save F25.install-documented-sources/documented" }, { "feature_id": "F25.refresh", "family": "F25", "result": "pass", "label": "Refresh from the source (`F25.refresh`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-17-refresh-from-the-source-f25-refresh.json", "failed_command": null }, { "feature_id": "F25.install-ref", "family": "F25", "result": "pass", "label": "Install at a ref (`F25.install-ref`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-18-install-at-a-ref-f25-install-ref.json", "failed_command": null }, { "feature_id": "F25.install-unknown-ref", "family": "F25", "result": "xfail", "label": "Unknown ref (`F25.install-unknown-ref`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-19-unknown-ref-f25-install-unknown-ref-know.json", "failed_command": "control-agent-server api POST /api/skills/install --json '{\"source\": \"file://'\"$G\"'\", \"repo_path\": \"skills/qa-f25-src-skill\", \"ref\": \"qa-f25-no-such-ref\", \"force\": true}' --expect 400 --save F25.install-unknown-ref/cached" }, { "feature_id": "F25.refresh-unfetchable", "family": "F25", "result": "xfail", "label": "Refresh of a vanished source (`F25.refresh-unfetchable`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-20-refresh-of-a-vanished-source-f25-refresh.json", "failed_command": "control-agent-server api POST /api/skills/installed/qa-f25-gone/refresh --expect 400,422 --save F25.refresh-unfetchable/refresh" }, { "feature_id": "F25.name-errors", "family": "F25", "result": "pass", "label": "Unknown and invalid names (`F25.name-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-21-unknown-and-invalid-names-f25-name-error.json", "failed_command": null }, { "feature_id": "F25.self-heal", "family": "F25", "result": "pass", "label": "Hand-copied and hand-deleted directories (`F25.self-heal`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-22-hand-copied-and-hand-deleted-directories.json", "failed_command": null }, { "feature_id": "F25.sync", "family": "F25", "result": "pass", "label": "Sync the public repository (`F25.sync`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-23-sync-the-public-repository-f25-sync.json", "failed_command": null }, { "feature_id": "F25.sync-offline", "family": "F25", "result": "xfail", "label": "Sync without a reachable origin (`F25.sync-offline`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-24-sync-without-a-reachable-origin-f25-sync.json", "failed_command": "control-agent-server api POST /api/skills/sync --expect 200 --check status eq error --save F25.sync-offline/sync" }, { "feature_id": "F25.catalog-registered-marketplace", "family": "F25", "result": "pass", "label": "Registered marketplace skills (`F25.catalog-registered-marketplace`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-25-registered-marketplace-skills-f25-catalo.json", "failed_command": null }, { "feature_id": "F25.conversation-activation", "family": "F25", "result": "pass", "label": "Installed skill in a conversation (`F25.conversation-activation`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-26-installed-skill-in-a-conversation-f25-co.json", "failed_command": null }, { "feature_id": "F25.invoke-skill", "family": "F25", "result": "pass", "label": "Skill invoked through `invoke_skill` (`F25.invoke-skill`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-27-skill-invoked-through-invoke-skill-f25-i.json", "failed_command": null }, { "feature_id": "F25.restart-persist", "family": "F25", "result": "pass", "label": "Restart (`F25.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-28-restart-f25-restart-persist.json", "failed_command": null }, { "feature_id": "F25.catalog-server-marketplaces", "family": "F25", "result": "pass", "label": "Server-default marketplaces (`F25.catalog-server-marketplaces`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-29-server-default-marketplaces-f25-catalog-.json", "failed_command": null }, { "feature_id": "F25.marketplace-catalog", "family": "F25", "result": "pass", "label": "Marketplace catalog (`F25.marketplace-catalog`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-30-marketplace-catalog-f25-marketplace-cata.json", "failed_command": null }, { "feature_id": "F25.ts-client", "family": "F25", "result": "pass", "label": "TypeScript client (`F25.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-31-typescript-client-f25-ts-client.json", "failed_command": null }, { "feature_id": "F25.marketplace-stale-cache", "family": "F25", "result": "xfail", "label": "Catalog after a sync (`F25.marketplace-stale-cache`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-32-catalog-after-a-sync-f25-marketplace-sta.json", "failed_command": "control-agent-server api GET /api/skills/marketplace --expect 200 --check skills contains '\"name\": \"qa-f25-pub-extra\"' --save F25.marketplace-stale-cache/after-sync" }, { "feature_id": "F25.uninstall", "family": "F25", "result": "pass", "label": "Uninstall (`F25.uninstall`).", "run": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "transcript": "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-33-uninstall-f25-uninstall.json", "failed_command": null }, { "feature_id": "F26.auth", "family": "F26", "result": "pass", "label": "Session key required (`F26.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-01-session-key-required-f26-auth.json", "failed_command": null }, { "feature_id": "F26.catalog-project", "family": "F26", "result": "pass", "label": "Project plugins (`F26.catalog-project`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-02-project-plugins-f26-catalog-project.json", "failed_command": null }, { "feature_id": "F26.catalog-user-precedence", "family": "F26", "result": "pass", "label": "User plugins and precedence (`F26.catalog-user-precedence`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-03-user-plugins-and-precedence-f26-catalog-.json", "failed_command": null }, { "feature_id": "F26.install-local", "family": "F26", "result": "pass", "label": "Install from a local path (`F26.install-local`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-04-install-from-a-local-path-f26-install-lo.json", "failed_command": null }, { "feature_id": "F26.install-agent-plugins", "family": "F26", "result": "pass", "label": "Agent Plugins layout (`F26.install-agent-plugins`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-05-agent-plugins-layout-f26-install-agent-p.json", "failed_command": null }, { "feature_id": "F26.install-git", "family": "F26", "result": "pass", "label": "Install from a git source (`F26.install-git`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-06-install-from-a-git-source-f26-install-gi.json", "failed_command": null }, { "feature_id": "F26.install-repo-root-name", "family": "F26", "result": "xfail", "label": "Repository root name (`F26.install-repo-root-name`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-07-repository-root-name-f26-install-repo-ro.json", "failed_command": "test \"$N\" = qa-f26-src" }, { "feature_id": "F26.install-conflict", "family": "F26", "result": "pass", "label": "Duplicate install (`F26.install-conflict`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-08-duplicate-install-f26-install-conflict.json", "failed_command": null }, { "feature_id": "F26.install-errors", "family": "F26", "result": "pass", "label": "Install errors (`F26.install-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-09-install-errors-f26-install-errors.json", "failed_command": null }, { "feature_id": "F26.install-github", "family": "F26", "result": "pass", "label": "Install from GitHub (`F26.install-github`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-10-install-from-github-f26-install-github.json", "failed_command": null }, { "feature_id": "F26.install-documented-source", "family": "F26", "result": "xfail", "label": "Documented install source (`F26.install-documented-source`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-11-documented-install-source-f26-install-do.json", "failed_command": "control-agent-server api POST /api/plugins/install --json '{\"source\": \"github:/plugins/city-weather\"}' --timeout 240 --expect 200 --check name eq city-weather --save F26.install-documented-source/documented" }, { "feature_id": "F26.enable-disable", "family": "F26", "result": "pass", "label": "Enable and disable (`F26.enable-disable`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-12-enable-and-disable-f26-enable-disable.json", "failed_command": null }, { "feature_id": "F26.name-errors", "family": "F26", "result": "pass", "label": "Unknown and invalid names (`F26.name-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-13-unknown-and-invalid-names-f26-name-error.json", "failed_command": null }, { "feature_id": "F26.refresh", "family": "F26", "result": "pass", "label": "Refresh (`F26.refresh`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-14-refresh-f26-refresh.json", "failed_command": null }, { "feature_id": "F26.self-heal", "family": "F26", "result": "pass", "label": "Hand-edited store (`F26.self-heal`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-15-hand-edited-store-f26-self-heal.json", "failed_command": null }, { "feature_id": "F26.refresh-unfetchable", "family": "F26", "result": "xfail", "label": "Unfetchable refresh (`F26.refresh-unfetchable`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-16-unfetchable-refresh-f26-refresh-unfetcha.json", "failed_command": "control-agent-server api POST /api/plugins/installed/qa-f26-gone/refresh --expect 400,422 --save F26.refresh-unfetchable/vanished" }, { "feature_id": "F26.uninstall", "family": "F26", "result": "pass", "label": "Uninstall (`F26.uninstall`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-17-uninstall-f26-uninstall.json", "failed_command": null }, { "feature_id": "F26.conversation-ambient", "family": "F26", "result": "pass", "label": "Ambient load into a conversation (`F26.conversation-ambient`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-18-ambient-load-into-a-conversation-f26-con.json", "failed_command": null }, { "feature_id": "F26.conversation-disabled", "family": "F26", "result": "pass", "label": "Disabled plugin in a new conversation (`F26.conversation-disabled`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-19-disabled-plugin-in-a-new-conversation-f2.json", "failed_command": null }, { "feature_id": "F26.conversation-resume-disabled", "family": "F26", "result": "xfail", "label": "Resume after disabling (`F26.conversation-resume-disabled`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-20-resume-after-disabling-f26-conversation-.json", "failed_command": "control-agent-server ws stop qa-f26-resume --kinds MessageEvent --contains '\"activated_skills\": [\"qa-f26-inst:qa-hello\"]' --expect-none --save F26.conversation-resume-disabled/frames" }, { "feature_id": "F26.restart-persist", "family": "F26", "result": "pass", "label": "Persistence across a restart (`F26.restart-persist`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-21-persistence-across-a-restart-f26-restart.json", "failed_command": null }, { "feature_id": "F26.marketplace-local", "family": "F26", "result": "pass", "label": "Marketplace against a local stand-in (`F26.marketplace-local`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-22-marketplace-against-a-local-stand-in-f26.json", "failed_command": null }, { "feature_id": "F26.marketplace-cache", "family": "F26", "result": "pass", "label": "Catalog cache (`F26.marketplace-cache`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-23-catalog-cache-f26-marketplace-cache.json", "failed_command": null }, { "feature_id": "F26.ts-client", "family": "F26", "result": "pass", "label": "TypeScript client (`F26.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-24-typescript-client-f26-ts-client.json", "failed_command": null }, { "feature_id": "F26.marketplace-public", "family": "F26", "result": "pass", "label": "Public marketplace (`F26.marketplace-public`).", "run": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "transcript": "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-25-public-marketplace-f26-marketplace-publi.json", "failed_command": null }, { "feature_id": "F27.auth", "family": "F27", "result": "pass", "label": "Session key required (`F27.auth`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-01-session-key-required-f27-auth.json", "failed_command": null }, { "feature_id": "F27.validation", "family": "F27", "result": "pass", "label": "Bad bodies (`F27.validation`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-02-bad-bodies-f27-validation.json", "failed_command": null }, { "feature_id": "F27.tools-list", "family": "F27", "result": "pass", "label": "Registered tools (`F27.tools-list`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-03-registered-tools-f27-tools-list.json", "failed_command": null }, { "feature_id": "F27.tools-redirect", "family": "F27", "result": "pass", "label": "Slash redirects (`F27.tools-redirect`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-04-slash-redirects-f27-tools-redirect.json", "failed_command": null }, { "feature_id": "F27.catalog", "family": "F27", "result": "pass", "label": "Tool catalog (`F27.catalog`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-05-tool-catalog-f27-catalog.json", "failed_command": null }, { "feature_id": "F27.hooks-project", "family": "F27", "result": "pass", "label": "Project hooks (`F27.hooks-project`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-06-project-hooks-f27-hooks-project.json", "failed_command": null }, { "feature_id": "F27.hooks-formats", "family": "F27", "result": "pass", "label": "Formats (`F27.hooks-formats`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-07-formats-f27-hooks-formats.json", "failed_command": null }, { "feature_id": "F27.hooks-absent", "family": "F27", "result": "pass", "label": "No hooks (`F27.hooks-absent`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-08-no-hooks-f27-hooks-absent.json", "failed_command": null }, { "feature_id": "F27.hooks-invalid", "family": "F27", "result": "pass", "label": "Broken hooks.json (`F27.hooks-invalid`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-09-broken-hooks-json-f27-hooks-invalid.json", "failed_command": null }, { "feature_id": "F27.subagents-builtin", "family": "F27", "result": "pass", "label": "Sub-agent built-ins (`F27.subagents-builtin`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-10-sub-agent-built-ins-f27-subagents-builti.json", "failed_command": null }, { "feature_id": "F27.subagents-project", "family": "F27", "result": "pass", "label": "Project sub-agents (`F27.subagents-project`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-11-project-sub-agents-f27-subagents-project.json", "failed_command": null }, { "feature_id": "F27.subagents-user", "family": "F27", "result": "pass", "label": "User sub-agents (`F27.subagents-user`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-12-user-sub-agents-f27-subagents-user.json", "failed_command": null }, { "feature_id": "F27.subagents-precedence", "family": "F27", "result": "pass", "label": "Precedence (`F27.subagents-precedence`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-13-precedence-f27-subagents-precedence.json", "failed_command": null }, { "feature_id": "F27.subagents-invalid", "family": "F27", "result": "pass", "label": "Invalid definitions (`F27.subagents-invalid`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-14-invalid-definitions-f27-subagents-invali.json", "failed_command": null }, { "feature_id": "F27.subagents-bad-skill", "family": "F27", "result": "xfail", "label": "Agent with a missing skill (`F27.subagents-bad-skill`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-15-agent-with-a-missing-skill-f27-subagents.json", "failed_command": "control-agent-server api POST \"/$BCID/events\" --json '{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"hi\"}], \"run\": false}' --expect 2xx,4xx --save F27.subagents-bad-skill/send" }, { "feature_id": "F27.hooks-conversation", "family": "F27", "result": "pass", "label": "Hooks in a conversation (`F27.hooks-conversation`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-16-hooks-in-a-conversation-f27-hooks-conver.json", "failed_command": null }, { "feature_id": "F27.hooks-not-automatic", "family": "F27", "result": "pass", "label": "Hooks are not loaded implicitly (`F27.hooks-not-automatic`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-17-hooks-are-not-loaded-implicitly-f27-hook.json", "failed_command": null }, { "feature_id": "F27.hooks-block", "family": "F27", "result": "pass", "label": "Blocking hooks (`F27.hooks-block`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-18-blocking-hooks-f27-hooks-block.json", "failed_command": null }, { "feature_id": "F27.catalog-sealed", "family": "F27", "result": "pass", "label": "Client tools stay out of the catalog (`F27.catalog-sealed`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-19-client-tools-stay-out-of-the-catalog-f27.json", "failed_command": null }, { "feature_id": "F27.client-tool-call", "family": "F27", "result": "pass", "label": "Client tool call (`F27.client-tool-call`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-20-client-tool-call-f27-client-tool-call.json", "failed_command": null }, { "feature_id": "F27.agent-definitions", "family": "F27", "result": "pass", "label": "Forwarded sub-agent definitions (`F27.agent-definitions`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-21-forwarded-sub-agent-definitions-f27-agen.json", "failed_command": null }, { "feature_id": "F27.custom-tool-modules", "family": "F27", "result": "pass", "label": "Custom tool module (`F27.custom-tool-modules`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-22-custom-tool-module-f27-custom-tool-modul.json", "failed_command": null }, { "feature_id": "F27.import-modules", "family": "F27", "result": "pass", "label": "Startup imports (`F27.import-modules`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-23-startup-imports-f27-import-modules.json", "failed_command": null }, { "feature_id": "F27.custom-tool-missing-on-resume", "family": "F27", "result": "xfail", "label": "Tool module gone on resume (`F27.custom-tool-missing-on-resume`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-24-tool-module-gone-on-resume-f27-custom-to.json", "failed_command": "control-agent-server api POST \"/$MCID/events\" --json '{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"Reply with the single word ok.\"}], \"run\": false}' --expect 2xx,4xx --save F27.custom-tool-missing-on-resume/send" }, { "feature_id": "F27.browser-playwright-path", "family": "F27", "result": "fail", "label": "Playwright browsers directory (`F27.browser-playwright-path`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-25-playwright-browsers-directory-f27-browse.json", "failed_command": "control-agent-server api GET /api/tools/catalog --expect 200 --quiet --check tools contains '{\"name\": \"browser_tool_set\", \"user_selectable\": true, \"usable\": false'" }, { "feature_id": "F27.browser-probe", "family": "F27", "result": "fail", "label": "Browser probe (`F27.browser-probe`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-26-browser-probe-f27-browser-probe.json", "failed_command": "control-agent-server api GET /api/tools/catalog --expect 200 --quiet --check tools contains '{\"name\": \"browser_tool_set\", \"user_selectable\": true, \"usable\": false' --save F27.browser-probe/catalog-before" }, { "feature_id": "F27.browser-disabled-launch", "family": "F27", "result": "fail", "label": "Default tools follow the browser setting (`F27.browser-disabled-launch`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-27-default-tools-follow-the-browser-setting.json", "failed_command": "test -L \"$H/.cache/ms-playwright\"" }, { "feature_id": "F27.browser-disabled", "family": "F27", "result": "fail", "label": "Browser disabled by config (`F27.browser-disabled`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-28-browser-disabled-by-config-f27-browser-d.json", "failed_command": "test -L \"$H/.cache/ms-playwright\"" }, { "feature_id": "F27.server-info-browser-disabled", "family": "F27", "result": "xfail", "label": "Usable tools with the browser disabled (`F27.server-info-browser-disabled`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-29-usable-tools-with-the-browser-disabled-f.json", "failed_command": "control-agent-server api GET /server_info --auth none --expect 200 --check usable_tools contains terminal --check usable_tools not-contains browser_tool_set --save F27.server-info-browser-disabled/server-info" }, { "feature_id": "F27.subagents-browser-disabled", "family": "F27", "result": "xfail", "label": "Browser-only sub-agent with the browser disabled (`F27.subagents-browser-disabled`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-30-browser-only-sub-agent-with-the-browser-.json", "failed_command": "control-agent-server api POST /api/sub-agents --json '{\"load_user\": false, \"load_project\": false}' --expect 200 --check agents contains '\"name\": \"bash-runner\"' --check agents not-contains '\"name\": \"web-researcher\"' --save F27.subagents-browser-disabled/builtins" }, { "feature_id": "F27.catalog-docker-runtime", "family": "F27", "result": "fail", "label": "Docker conversation runtime (`F27.catalog-docker-runtime`).", "run": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "transcript": "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-31-docker-conversation-runtime-f27-catalog-.json", "failed_command": "rm \"$H/.cache/ms-playwright\"" }, { "feature_id": "F28.install-local", "family": "F28", "result": "pass", "label": "Install from a local directory (`F28.install-local`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-01-install-from-a-local-directory-f28-insta.json", "failed_command": null }, { "feature_id": "F28.install-git", "family": "F28", "result": "pass", "label": "Install from a git URL (`F28.install-git`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-02-install-from-a-git-url-f28-install-git.json", "failed_command": null }, { "feature_id": "F28.install-ref", "family": "F28", "result": "pass", "label": "Pin a git ref (`F28.install-ref`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-03-pin-a-git-ref-f28-install-ref.json", "failed_command": null }, { "feature_id": "F28.install-repo-path", "family": "F28", "result": "pass", "label": "Install a subdirectory (`F28.install-repo-path`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-04-install-a-subdirectory-f28-install-repo-.json", "failed_command": null }, { "feature_id": "F28.install-errors", "family": "F28", "result": "pass", "label": "Install errors (`F28.install-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-05-install-errors-f28-install-errors.json", "failed_command": null }, { "feature_id": "F28.bundle", "family": "F28", "result": "pass", "label": "Bundle (`F28.bundle`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-06-bundle-f28-bundle.json", "failed_command": null }, { "feature_id": "F28.icon", "family": "F28", "result": "pass", "label": "Icon (`F28.icon`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-07-icon-f28-icon.json", "failed_command": null }, { "feature_id": "F28.enable-disable", "family": "F28", "result": "pass", "label": "Enable and disable (`F28.enable-disable`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-08-enable-and-disable-f28-enable-disable.json", "failed_command": null }, { "feature_id": "F28.unknown-name", "family": "F28", "result": "pass", "label": "Unknown and invalid names (`F28.unknown-name`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-09-unknown-and-invalid-names-f28-unknown-na.json", "failed_command": null }, { "feature_id": "F28.list-discovery", "family": "F28", "result": "pass", "label": "Discovery by listing (`F28.list-discovery`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-10-discovery-by-listing-f28-list-discovery.json", "failed_command": null }, { "feature_id": "F28.broken-install", "family": "F28", "result": "pass", "label": "Broken installs (`F28.broken-install`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-11-broken-installs-f28-broken-install.json", "failed_command": null }, { "feature_id": "F28.backend-status", "family": "F28", "result": "pass", "label": "Backend status (`F28.backend-status`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-12-backend-status-f28-backend-status.json", "failed_command": null }, { "feature_id": "F28.backend-approval-gate", "family": "F28", "result": "pass", "label": "Approval gate (`F28.backend-approval-gate`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-13-approval-gate-f28-backend-approval-gate.json", "failed_command": null }, { "feature_id": "F28.backend-prepare", "family": "F28", "result": "pass", "label": "Prepare (`F28.backend-prepare`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-14-prepare-f28-backend-prepare.json", "failed_command": null }, { "feature_id": "F28.backend-start", "family": "F28", "result": "pass", "label": "Start (`F28.backend-start`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-15-start-f28-backend-start.json", "failed_command": null }, { "feature_id": "F28.backend-stop", "family": "F28", "result": "fail", "label": "Stop (`F28.backend-stop`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-16-stop-f28-backend-stop.json", "failed_command": "control-agent-server api POST /api/canvas-extensions/installed/qa-f28-flaky/backend/stop --expect 200 --check state eq stopped --expect-max-ms 7000 --save F28.backend-stop/group" }, { "feature_id": "F28.force-reinstall", "family": "F28", "result": "pass", "label": "Reinstall (`F28.force-reinstall`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-17-reinstall-f28-force-reinstall.json", "failed_command": null }, { "feature_id": "F28.backend-logs", "family": "F28", "result": "pass", "label": "Logs while starting (`F28.backend-logs`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-18-logs-while-starting-f28-backend-logs.json", "failed_command": null }, { "feature_id": "F28.ts-client", "family": "F28", "result": "pass", "label": "TypeScript client (`F28.ts-client`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-19-typescript-client-f28-ts-client.json", "failed_command": null }, { "feature_id": "F28.backend-start-fails", "family": "F28", "result": "pass", "label": "Failed starts (`F28.backend-start-fails`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-20-failed-starts-f28-backend-start-fails.json", "failed_command": null }, { "feature_id": "F28.backend-inherit-environment", "family": "F28", "result": "pass", "label": "Inherited environment (`F28.backend-inherit-environment`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-21-inherited-environment-f28-backend-inheri.json", "failed_command": null }, { "feature_id": "F28.disable-stops-backend", "family": "F28", "result": "pass", "label": "Disable stops the backend (`F28.disable-stops-backend`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-22-disable-stops-the-backend-f28-disable-st.json", "failed_command": null }, { "feature_id": "F28.restart-persistence", "family": "F28", "result": "fail", "label": "Persistence across a restart (`F28.restart-persistence`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-23-persistence-across-a-restart-f28-restart.json", "failed_command": "control-agent-server api GET /api/canvas-extensions/installed --check canvas_extensions len-eq 6 --quiet" }, { "feature_id": "F28.uninstall", "family": "F28", "result": "fail", "label": "Uninstall (`F28.uninstall`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-24-uninstall-f28-uninstall.json", "failed_command": "control-agent-server api GET /api/canvas-extensions/installed --check canvas_extensions len-eq 5 --check canvas_extensions not-contains qa-backend --quiet" }, { "feature_id": "F28.prepare-corrupt-archive", "family": "F28", "result": "xfail", "label": "Non-gzip artifact (`F28.prepare-corrupt-archive`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-25-non-gzip-artifact-f28-prepare-corrupt-ar.json", "failed_command": "control-agent-server api POST /api/canvas-extensions/installed/qa-f28-notgz/backend/prepare --json \"{\\\"revision\\\": \\\"$NREV\\\"}\" --expect 409 --save F28.prepare-corrupt-archive/prepare" }, { "feature_id": "F28.install-ref-cached", "family": "F28", "result": "xfail", "label": "Unknown ref on a cached source (`F28.install-ref-cached`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-26-unknown-ref-on-a-cached-source-f28-insta.json", "failed_command": "control-agent-server api POST /api/canvas-extensions/install --json \"{\\\"source\\\": \\\"file://$GX\\\", \\\"ref\\\": \\\"qa-no-such-ref\\\", \\\"force\\\": true}\" --expect 400 --check detail contains 'Could not read canvas extension source' --save F28.install-ref-cached/unknown-ref" }, { "feature_id": "F28.status-probe-latch", "family": "F28", "result": "xfail", "label": "Health probe latch (`F28.status-probe-latch`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-27-health-probe-latch-f28-status-probe-latc.json", "failed_command": "control-agent-server api GET /api/canvas-extensions/installed/qa-f28-flaky/backend --until-ok 5 --check state eq ready --save F28.status-probe-latch/after-recovery" }, { "feature_id": "F28.status-exit-hang", "family": "F28", "result": "fail", "label": "Status after the leader exits (`F28.status-exit-hang`).", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-28-status-after-the-leader-exits-f28-status.json", "failed_command": "false" }, { "feature_id": "F28.hard-restart-orphan", "family": "F28", "result": "xfail", "label": "Backend outlives a crash (`F28.hard-restart-orphan`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28", "transcript": "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28/evidence/map-run/F28-canvas-extensions-29-backend-outlives-a-crash-f28-hard-restar.json", "failed_command": "false" }, { "feature_id": "F29.ingress-advertised", "family": "F29", "result": "pass", "label": "Ingress advertised (`F29.ingress-advertised`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-01-ingress-advertised-f29-ingress-advertise.json", "failed_command": null }, { "feature_id": "F29.ingress-config-errors", "family": "F29", "result": "pass", "label": "Ingress not configured or unusable (`F29.ingress-config-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-02-ingress-not-configured-or-unusable-f29-i.json", "failed_command": null }, { "feature_id": "F29.session-bootstrap", "family": "F29", "result": "pass", "label": "Session bootstrap (`F29.session-bootstrap`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-03-session-bootstrap-f29-session-bootstrap.json", "failed_command": null }, { "feature_id": "F29.session-key-required", "family": "F29", "result": "pass", "label": "Session key required (`F29.session-key-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-04-session-key-required-f29-session-key-req.json", "failed_command": null }, { "feature_id": "F29.session-ingress-host", "family": "F29", "result": "pass", "label": "Ingress host (`F29.session-ingress-host`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-05-ingress-host-f29-session-ingress-host.json", "failed_command": null }, { "feature_id": "F29.session-forwarded-proto", "family": "F29", "result": "xfail", "label": "Forwarded proto (`F29.session-forwarded-proto`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-06-forwarded-proto-f29-session-forwarded-pr.json", "failed_command": "control-agent-server api POST /session --header 'Origin: http://localhost:3000' --header 'X-Forwarded-Proto: https' --expect 200 --check ingress_url eq \"$S/\" --save F29.session-forwarded-proto/session" }, { "feature_id": "F29.session-origin-rules", "family": "F29", "result": "pass", "label": "Origin rules (`F29.session-origin-rules`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-07-origin-rules-f29-session-origin-rules.json", "failed_command": null }, { "feature_id": "F29.session-backend-not-ready", "family": "F29", "result": "pass", "label": "Backend not ready (`F29.session-backend-not-ready`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-08-backend-not-ready-f29-session-backend-no.json", "failed_command": null }, { "feature_id": "F29.proxy-get", "family": "F29", "result": "pass", "label": "Proxied safe methods (`F29.proxy-get`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-09-proxied-safe-methods-f29-proxy-get.json", "failed_command": null }, { "feature_id": "F29.proxy-cookie-required", "family": "F29", "result": "pass", "label": "Cookie required (`F29.proxy-cookie-required`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-10-cookie-required-f29-proxy-cookie-require.json", "failed_command": null }, { "feature_id": "F29.proxy-unsafe-methods", "family": "F29", "result": "pass", "label": "Unsafe methods (`F29.proxy-unsafe-methods`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-11-unsafe-methods-f29-proxy-unsafe-methods.json", "failed_command": null }, { "feature_id": "F29.proxy-root", "family": "F29", "result": "pass", "label": "App root (`F29.proxy-root`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-12-app-root-f29-proxy-root.json", "failed_command": null }, { "feature_id": "F29.proxy-root-path-query", "family": "F29", "result": "xfail", "label": "Root `path` parameter (`F29.proxy-root-path-query`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-13-root-path-parameter-f29-proxy-root-path-.json", "failed_command": "control-agent-server api GET --query path=echo --auth none --cookie \"oh_app_backend_session=$RP\" --expect 200 --check path eq / --check query.path eq echo --save F29.proxy-root-path-query/get" }, { "feature_id": "F29.proxy-credentials-stripped", "family": "F29", "result": "pass", "label": "Credentials stripped (`F29.proxy-credentials-stripped`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-14-credentials-stripped-f29-proxy-credentia.json", "failed_command": null }, { "feature_id": "F29.proxy-request-framing", "family": "F29", "result": "xfail", "label": "Request framing (`F29.proxy-request-framing`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-15-request-framing-f29-proxy-request-framin.json", "failed_command": "control-agent-server api GET /app-backends/qa-headers/echo --auth none --cookie \"oh_app_backend_session=$FR\" --expect 200 --check method eq GET --check body eq '' --check headers.transfer-encoding missing --save F29.proxy-request-framing/get" }, { "feature_id": "F29.proxy-response-filter", "family": "F29", "result": "pass", "label": "Response filter (`F29.proxy-response-filter`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-16-response-filter-f29-proxy-response-filte.json", "failed_command": null }, { "feature_id": "F29.proxy-response-headers", "family": "F29", "result": "xfail", "label": "One Date and Server header (`F29.proxy-response-headers`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-17-one-date-and-server-header-f29-proxy-res.json", "failed_command": "control-agent-server api GET /echo --auth none --cookie \"oh_app_backend_session=$RH\" --expect 200 --check-header date matches '^[A-Z][a-z]{2}, [0-9]{2} [A-Z][a-z]{2} [0-9]{4} [0-9:]{8} GMT$' --check-header server not-contains ', ' --all-headers --save F29.proxy-response-headers/echo" }, { "feature_id": "F29.proxy-path-traversal", "family": "F29", "result": "pass", "label": "Path traversal (`F29.proxy-path-traversal`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-18-path-traversal-f29-proxy-path-traversal.json", "failed_command": null }, { "feature_id": "F29.ws-bridge", "family": "F29", "result": "pass", "label": "WebSocket bridge (`F29.ws-bridge`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-19-websocket-bridge-f29-ws-bridge.json", "failed_command": null }, { "feature_id": "F29.ws-handshake-rejections", "family": "F29", "result": "pass", "label": "Handshake rejections (`F29.ws-handshake-rejections`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-20-handshake-rejections-f29-ws-handshake-re.json", "failed_command": null }, { "feature_id": "F29.ws-subprotocol", "family": "F29", "result": "xfail", "label": "Subprotocol (`F29.ws-subprotocol`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-21-subprotocol-f29-ws-subprotocol-known-bug.json", "failed_command": "control-agent-server exec --expect-output 'subprotocol=qa.v1' --expect-output '\"protocol\": \"qa.v1\"' --save F29.ws-subprotocol/bridged -- \"$PWD/.venv/bin/python\" -c 'import sys; from websockets.sync.client import connect; ws = connect(sys.argv[1], subprotocols=[\"qa.v1\"], origin=sys.argv[2], additional_headers={\"Cookie\": \"oh_app_backend_session=\" + sys.argv[3]}, proxy=None); print(\"subprotocol=\" + str(ws.subprotocol)); ws.send(\"qa-f29-proto\"); print(ws.recv(timeout=10)); ws.close()' \"ws://127.0.0." }, { "feature_id": "F29.session-revoke", "family": "F29", "result": "pass", "label": "Revoke a session (`F29.session-revoke`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-22-revoke-a-session-f29-session-revoke.json", "failed_command": null }, { "feature_id": "F29.backend-lifecycle-revokes", "family": "F29", "result": "pass", "label": "Backend stop and disable revoke (`F29.backend-lifecycle-revokes`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-23-backend-stop-and-disable-revoke-f29-back.json", "failed_command": null }, { "feature_id": "F29.backend-crash-detected", "family": "F29", "result": "xfail", "label": "Crash seen by the bridge (`F29.backend-crash-detected`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-24-crash-seen-by-the-bridge-f29-backend-cra.json", "failed_command": "control-agent-server api GET /echo --auth none --cookie \"oh_app_backend_session=$CC\" --until-ok 15 --expect 503 --check exception eq '503: Canvas App backend is not ready' --save F29.backend-crash-detected/crashed" }, { "feature_id": "F29.session-configured-control-origin", "family": "F29", "result": "pass", "label": "Configured control origins (`F29.session-configured-control-origin`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-25-configured-control-origins-f29-session-c.json", "failed_command": null }, { "feature_id": "F29.trusted-forwarded-headers", "family": "F29", "result": "pass", "label": "Trusted forwarded headers (`F29.trusted-forwarded-headers`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-26-trusted-forwarded-headers-f29-trusted-fo.json", "failed_command": null }, { "feature_id": "F29.sessions-not-persisted", "family": "F29", "result": "pass", "label": "Server restart (`F29.sessions-not-persisted`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-27-server-restart-f29-sessions-not-persiste.json", "failed_command": null }, { "feature_id": "F29.session-ttl", "family": "F29", "result": "pass", "label": "Session expiry (`F29.session-ttl`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-28-session-expiry-f29-session-ttl.json", "failed_command": null }, { "feature_id": "F29.uninstall-revokes", "family": "F29", "result": "pass", "label": "Uninstall (`F29.uninstall-revokes`).", "run": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29", "transcript": "/workspace/verification-artifacts/runs/20261013-021747-0b62-map-f29/evidence/map-run/F29-app-backend-bridge-29-uninstall-f29-uninstall-revokes.json", "failed_command": null }, { "feature_id": "F30.models-list", "family": "F30", "result": "pass", "label": "Model list (`F30.models-list`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-01-model-list-f30-models-list.json", "failed_command": null }, { "feature_id": "F30.auth-bearer-or-header", "family": "F30", "result": "pass", "label": "Bearer or session header (`F30.auth-bearer-or-header`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-02-bearer-or-session-header-f30-auth-bearer.json", "failed_command": null }, { "feature_id": "F30.auth-open-without-keys", "family": "F30", "result": "pass", "label": "No session keys (`F30.auth-open-without-keys`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-03-no-session-keys-f30-auth-open-without-ke.json", "failed_command": null }, { "feature_id": "F30.chat-basic", "family": "F30", "result": "pass", "label": "Chat completion (`F30.chat-basic`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-04-chat-completion-f30-chat-basic.json", "failed_command": null }, { "feature_id": "F30.chat-system-suffix", "family": "F30", "result": "pass", "label": "System text goes to the system prompt (`F30.chat-system-suffix`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-05-system-text-goes-to-the-system-prompt-f3.json", "failed_command": null }, { "feature_id": "F30.chat-latest-user-only", "family": "F30", "result": "pass", "label": "Only the latest user turn (`F30.chat-latest-user-only`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-06-only-the-latest-user-turn-f30-chat-lates.json", "failed_command": null }, { "feature_id": "F30.chat-stream", "family": "F30", "result": "pass", "label": "Streaming (`F30.chat-stream`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-07-streaming-f30-chat-stream.json", "failed_command": null }, { "feature_id": "F30.chat-reuse-header", "family": "F30", "result": "pass", "label": "Continue with the header (`F30.chat-reuse-header`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-08-continue-with-the-header-f30-chat-reuse-.json", "failed_command": null }, { "feature_id": "F30.chat-reuse-after-restart", "family": "F30", "result": "pass", "label": "Continue after a restart (`F30.chat-reuse-after-restart`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-09-continue-after-a-restart-f30-chat-reuse-.json", "failed_command": null }, { "feature_id": "F30.chat-header-new-id", "family": "F30", "result": "pass", "label": "Caller-chosen id (`F30.chat-header-new-id`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-10-caller-chosen-id-f30-chat-header-new-id.json", "failed_command": null }, { "feature_id": "F30.chat-reuse-keeps-llm", "family": "F30", "result": "pass", "label": "Continuing keeps the conversation's LLM (`F30.chat-reuse-keeps-llm`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-11-continuing-keeps-the-conversation-s-llm-.json", "failed_command": null }, { "feature_id": "F30.chat-usage-per-request", "family": "F30", "result": "xfail", "label": "Usage per request (`F30.chat-usage-per-request`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-12-usage-per-request-f30-chat-usage-per-req.json", "failed_command": "control-agent-server state cat fixtures/qa-f30/usage-2.json --check usage.prompt_tokens eq 3 --check usage.completion_tokens eq 2 --check usage.total_tokens eq 5" }, { "feature_id": "F30.chat-reuse-concurrent", "family": "F30", "result": "xfail", "label": "Overlapping turns on one conversation (`F30.chat-reuse-concurrent`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-13-overlapping-turns-on-one-conversation-f3.json", "failed_command": "control-agent-server state cat fixtures/qa-f30/race-x.json --check choices.0.message.content eq QA_F30_RACE_B" }, { "feature_id": "F30.chat-agent-tools", "family": "F30", "result": "pass", "label": "The agent runs its own tools (`F30.chat-agent-tools`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-14-the-agent-runs-its-own-tools-f30-chat-ag.json", "failed_command": null }, { "feature_id": "F30.chat-input-errors", "family": "F30", "result": "pass", "label": "Chat input errors (`F30.chat-input-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-15-chat-input-errors-f30-chat-input-errors.json", "failed_command": null }, { "feature_id": "F30.observability-headers", "family": "F30", "result": "pass", "label": "Observability headers (`F30.observability-headers`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-16-observability-headers-f30-observability-.json", "failed_command": null }, { "feature_id": "F30.chat-run-error-500", "family": "F30", "result": "pass", "label": "Run ends in error (`F30.chat-run-error-500`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-17-run-ends-in-error-f30-chat-run-error-500.json", "failed_command": null }, { "feature_id": "F30.chat-confirmation-409", "family": "F30", "result": "pass", "label": "Confirmation stops a new conversation (`F30.chat-confirmation-409`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-18-confirmation-stops-a-new-conversation-f3.json", "failed_command": null }, { "feature_id": "F30.chat-reuse-confirmation", "family": "F30", "result": "xfail", "label": "Confirmation on a continued conversation (`F30.chat-reuse-confirmation`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-19-confirmation-on-a-continued-conversation.json", "failed_command": "control-agent-server state cat fixtures/qa-f30/reconf-2.json --check status eq 409 --check response.body.detail eq 'Agent run ended with status: waiting_for_confirmation' --check elapsed_ms lt 30000" }, { "feature_id": "F30.chat-run-limit-429", "family": "F30", "result": "pass", "label": "Run limit (`F30.chat-run-limit-429`).", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-20-run-limit-f30-chat-run-limit-429.json", "failed_command": null }, { "feature_id": "F30.chat-reuse-run-limit-429", "family": "F30", "result": "xfail", "label": "Run limit on a continued conversation (`F30.chat-reuse-run-limit-429`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30", "transcript": "/workspace/verification-artifacts/runs/20261013-021801-0cd5-map-f30/evidence/map-run/F30-openai-gateway-21-run-limit-on-a-continued-conversation-f3.json", "failed_command": "control-agent-server api GET \"/$CID/events/search\" --query source=user --query body=QA_F30_429_REUSE --check items len-eq 0 --save F30.chat-reuse-run-limit-429/user-messages" }, { "feature_id": "F31.lease-claimed", "family": "F31", "result": "pass", "label": "Lease claimed on load (`F31.lease-claimed`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-01-lease-claimed-on-load-f31-lease-claimed.json", "failed_command": null }, { "feature_id": "F31.lease-renewed", "family": "F31", "result": "pass", "label": "Lease renewed (`F31.lease-renewed`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-02-lease-renewed-f31-lease-renewed.json", "failed_command": null }, { "feature_id": "F31.lease-released-graceful", "family": "F31", "result": "pass", "label": "Graceful release (`F31.lease-released-graceful`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-03-graceful-release-f31-lease-released-grac.json", "failed_command": null }, { "feature_id": "F31.lease-dead-owner-takeover", "family": "F31", "result": "fail", "label": "Dead owner takeover (`F31.lease-dead-owner-takeover`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-04-dead-owner-takeover-f31-lease-dead-owner.json", "failed_command": "false" }, { "feature_id": "F31.crash-tool-call", "family": "F31", "result": "pass", "label": "Crash during a tool call (`F31.crash-tool-call`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-05-crash-during-a-tool-call-f31-crash-tool-.json", "failed_command": null }, { "feature_id": "F31.state-survives-restarts", "family": "F31", "result": "pass", "label": "State survives restarts (`F31.state-survives-restarts`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-06-state-survives-restarts-f31-state-surviv.json", "failed_command": null }, { "feature_id": "F31.no-plaintext-at-rest", "family": "F31", "result": "pass", "label": "Nothing secret in plaintext (`F31.no-plaintext-at-rest`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-07-nothing-secret-in-plaintext-f31-no-plain.json", "failed_command": null }, { "feature_id": "F31.second-instance-catalog", "family": "F31", "result": "pass", "label": "Second server reads the catalog (`F31.second-instance-catalog`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-08-second-server-reads-the-catalog-f31-seco.json", "failed_command": null }, { "feature_id": "F31.second-instance-lease-held", "family": "F31", "result": "pass", "label": "Second server refused while the owner lives (`F31.second-instance-lease-held`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-09-second-server-refused-while-the-owner-li.json", "failed_command": null }, { "feature_id": "F31.second-instance-takeover", "family": "F31", "result": "fail", "label": "Takeover after the owner dies (`F31.second-instance-takeover`).", "run": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31", "transcript": "/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/map-run/F31-persistence-and-limits-10-takeover-after-the-owner-dies-f31-second.json", "failed_command": "control-agent-server logs --run \"$B\" --grep 'Taking over conversation' --expect-min $((NB + 1))" }, { "feature_id": "F32.events-post", "family": "F32", "result": "pass", "label": "A run's events reach the webhook (`F32.events-post`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-01-a-run-s-events-reach-the-webhook-f32-eve.json", "failed_command": null }, { "feature_id": "F32.events-session-key", "family": "F32", "result": "pass", "label": "Session key and content type on every request (`F32.events-session-key`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-02-session-key-and-content-type-on-every-re.json", "failed_command": null }, { "feature_id": "F32.conversation-created", "family": "F32", "result": "pass", "label": "Conversation created (`F32.conversation-created`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-03-conversation-created-f32-conversation-cr.json", "failed_command": null }, { "feature_id": "F32.events-full-state", "family": "F32", "result": "pass", "label": "Full-state snapshot on every attach (`F32.events-full-state`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-04-full-state-snapshot-on-every-attach-f32-.json", "failed_command": null }, { "feature_id": "F32.conversation-lifecycle", "family": "F32", "result": "pass", "label": "Conversation lifecycle posts (`F32.conversation-lifecycle`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-05-conversation-lifecycle-posts-f32-convers.json", "failed_command": null }, { "feature_id": "F32.events-no-deltas", "family": "F32", "result": "pass", "label": "No streaming deltas on webhooks (`F32.events-no-deltas`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-06-no-streaming-deltas-on-webhooks-f32-even.json", "failed_command": null }, { "feature_id": "F32.custom-headers", "family": "F32", "result": "pass", "label": "Custom headers, path prefix and buffer size (`F32.custom-headers`, `F32.buffer-size`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-07-custom-headers-path-prefix-and-buffer-si.json", "failed_command": null }, { "feature_id": "F32.buffer-size", "family": "F32", "result": "pass", "label": "Custom headers, path prefix and buffer size (`F32.custom-headers`, `F32.buffer-size`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-07-custom-headers-path-prefix-and-buffer-si.json", "failed_command": null }, { "feature_id": "F32.buffer-stranded", "family": "F32", "result": "xfail", "label": "Events stranded behind an in-flight batch (`F32.buffer-stranded`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-08-events-stranded-behind-an-in-flight-batc.json", "failed_command": "control-agent-server sink read --path \"/qa-f32-b/events/${BID//-/}\" --contains QA_F32_BUFFER --expect-min 1 --wait 10 --save F32.buffer-stranded/after-wait" }, { "feature_id": "F32.flush-on-close", "family": "F32", "result": "pass", "label": "Flush on delete and on shutdown (`F32.flush-on-close`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-09-flush-on-delete-and-on-shutdown-f32-flus.json", "failed_command": null }, { "feature_id": "F32.flush-delay", "family": "F32", "result": "fail", "label": "Flush timer from the first event (`F32.flush-delay`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-10-flush-timer-from-the-first-event-f32-flu.json", "failed_command": "control-agent-server sink read --path \"/qa-f32-d/events/${TID//-/}\" --expect-max 0" }, { "feature_id": "F32.webhook-spec-docs", "family": "F32", "result": "xfail", "label": "Webhook field descriptions (`F32.webhook-spec-docs`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-11-webhook-field-descriptions-f32-webhook-s.json", "failed_command": "control-agent-server api GET /openapi.json --auth none --quiet --check components.schemas.WebhookSpec.properties.flush_delay.description not-contains 'Timer is reset on each new event'" }, { "feature_id": "F32.backpressure", "family": "F32", "result": "pass", "label": "Batch and queue limits (`F32.backpressure`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-12-batch-and-queue-limits-f32-backpressure.json", "failed_command": null }, { "feature_id": "F32.retry", "family": "F32", "result": "pass", "label": "Retries against a failing receiver (`F32.retry`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-13-retries-against-a-failing-receiver-f32-r.json", "failed_command": null }, { "feature_id": "F32.requeue", "family": "F32", "result": "pass", "label": "Failed batches are requeued (`F32.requeue`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-14-failed-batches-are-requeued-f32-requeue.json", "failed_command": null }, { "feature_id": "F32.slow-receiver", "family": "F32", "result": "xfail", "label": "A slow receiver blocks the next run (`F32.slow-receiver`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-15-a-slow-receiver-blocks-the-next-run-f32-.json", "failed_command": "control-agent-server api POST \"/$WCID/run\" --expect 200 --save F32.slow-receiver/run-after-finished" }, { "feature_id": "F32.env-config", "family": "F32", "result": "pass", "label": "Webhooks from the environment (`F32.env-config`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-16-webhooks-from-the-environment-f32-env-co.json", "failed_command": null }, { "feature_id": "F32.telemetry-off-by-default", "family": "F32", "result": "fail", "label": "Telemetry is off by default (`F32.telemetry-off-by-default`).", "run": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32", "transcript": "/workspace/verification-artifacts/runs/20261013-021835-ac36-map-f32/evidence/map-run/F32-webhooks-and-telemetry-17-telemetry-is-off-by-default-f32-telemetr.json", "failed_command": "control-agent-server conversation start --placeholder-agent --no-run --prompt QA_F32_OFF --body-json '{\"user_id\": \"qa-f32-off\"}' --no-autotitle" }, { "feature_id": "F33.workspace-exec", "family": "F33", "result": "pass", "label": "Commands, server info and liveness (`F33.workspace-exec`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-01-commands-server-info-and-liveness-f33-wo.json", "failed_command": null }, { "feature_id": "F33.workspace-bad-key", "family": "F33", "result": "pass", "label": "Wrong session key (`F33.workspace-bad-key`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-02-wrong-session-key-f33-workspace-bad-key.json", "failed_command": null }, { "feature_id": "F33.workspace-background", "family": "F33", "result": "pass", "label": "Background command (`F33.workspace-background`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-03-background-command-f33-workspace-backgro.json", "failed_command": null }, { "feature_id": "F33.workspace-files", "family": "F33", "result": "pass", "label": "File round trip (`F33.workspace-files`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-04-file-round-trip-f33-workspace-files.json", "failed_command": null }, { "feature_id": "F33.workspace-git", "family": "F33", "result": "pass", "label": "Git changes and diff (`F33.workspace-git`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-05-git-changes-and-diff-f33-workspace-git.json", "failed_command": null }, { "feature_id": "F33.workspace-settings", "family": "F33", "result": "pass", "label": "LLM profiles, secrets and MCP from the server (`F33.workspace-settings`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-06-llm-profiles-secrets-and-mcp-from-the-se.json", "failed_command": null }, { "feature_id": "F33.async-workspace", "family": "F33", "result": "pass", "label": "Async workspace (`F33.async-workspace`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-07-async-workspace-f33-async-workspace.json", "failed_command": null }, { "feature_id": "F33.conversation-create", "family": "F33", "result": "pass", "label": "Create a conversation without running it (`F33.conversation-create`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-08-create-a-conversation-without-running-it.json", "failed_command": null }, { "feature_id": "F33.conversation-create-request", "family": "F33", "result": "pass", "label": "Create from a request, twice (`F33.conversation-create-request`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-09-create-from-a-request-twice-f33-conversa.json", "failed_command": null }, { "feature_id": "F33.conversation-create-profile", "family": "F33", "result": "pass", "label": "Create from a saved agent profile (`F33.conversation-create-profile`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-10-create-from-a-saved-agent-profile-f33-co.json", "failed_command": null }, { "feature_id": "F33.workspace-runtime-scope", "family": "F33", "result": "pass", "label": "Conversation-scoped workspace (`F33.workspace-runtime-scope`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-11-conversation-scoped-workspace-f33-worksp.json", "failed_command": null }, { "feature_id": "F33.conversation-run", "family": "F33", "result": "pass", "label": "Send and run to completion (`F33.conversation-run`, `F33.conversation-callbacks`, `F33.conversation-stats`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-13-send-and-run-to-completion-f33-conversat.json", "failed_command": null }, { "feature_id": "F33.conversation-callbacks", "family": "F33", "result": "pass", "label": "Send and run to completion (`F33.conversation-run`, `F33.conversation-callbacks`, `F33.conversation-stats`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-13-send-and-run-to-completion-f33-conversat.json", "failed_command": null }, { "feature_id": "F33.conversation-stats", "family": "F33", "result": "pass", "label": "Send and run to completion (`F33.conversation-run`, `F33.conversation-callbacks`, `F33.conversation-stats`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-13-send-and-run-to-completion-f33-conversat.json", "failed_command": null }, { "feature_id": "F33.conversation-stop-hook", "family": "F33", "result": "pass", "label": "A Stop hook that refuses the first stop (`F33.conversation-stop-hook`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-14-a-stop-hook-that-refuses-the-first-stop-.json", "failed_command": null }, { "feature_id": "F33.client-tools", "family": "F33", "result": "pass", "label": "Client-defined tool (`F33.client-tools`).", "run": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33", "transcript": "/workspace/verification-artifacts/runs/20261013-021957-adb9-map-f33/evidence/map-run/F33-sdk-remote-client-15-client-defined-tool-f33-client-tools.json", "failed_command": null }, { "feature_id": "F34.server-client", "family": "F34", "result": "pass", "label": "Server status and version gate (`F34.server-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-01-server-status-and-version-gate-f34-serve.json", "failed_command": null }, { "feature_id": "F34.http-errors", "family": "F34", "result": "pass", "label": "Errors (`F34.http-errors`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-02-errors-f34-http-errors.json", "failed_command": null }, { "feature_id": "F34.workspace-exec", "family": "F34", "result": "pass", "label": "Workspace commands (`F34.workspace-exec`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-03-workspace-commands-f34-workspace-exec.json", "failed_command": null }, { "feature_id": "F34.workspace-files", "family": "F34", "result": "pass", "label": "Workspace files (`F34.workspace-files`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-04-workspace-files-f34-workspace-files.json", "failed_command": null }, { "feature_id": "F34.workspace-binary-download", "family": "F34", "result": "xfail", "label": "Binary download through RemoteWorkspace (`F34.workspace-binary-download`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-05-binary-download-through-remoteworkspace-.json", "failed_command": "jq -e '.blob == [255, 0, 128, 254, 65] and .file_size == 5' \"$F34/binary.json\"" }, { "feature_id": "F34.workspace-scoped", "family": "F34", "result": "pass", "label": "Conversation-scoped workspace (`F34.workspace-scoped`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-06-conversation-scoped-workspace-f34-worksp.json", "failed_command": null }, { "feature_id": "F34.workspace-git", "family": "F34", "result": "pass", "label": "Git through RemoteWorkspace (`F34.workspace-git`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-07-git-through-remoteworkspace-f34-workspac.json", "failed_command": null }, { "feature_id": "F34.workspace-git-types", "family": "F34", "result": "xfail", "label": "Git status type (`F34.workspace-git-types`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-08-git-status-type-f34-workspace-git-types-.json", "failed_command": "$TSC \"$F34/git-types.mts\"" }, { "feature_id": "F34.bash-client", "family": "F34", "result": "pass", "label": "Bash client (`F34.bash-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-09-bash-client-f34-bash-client.json", "failed_command": null }, { "feature_id": "F34.bash-websocket", "family": "F34", "result": "xfail", "label": "Bash events socket from Node (`F34.bash-websocket`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-10-bash-events-socket-from-node-f34-bash-we.json", "failed_command": "jq -e '.saw_output and .errors == []' \"$F34/bash-ws.json\"" }, { "feature_id": "F34.workspace-session", "family": "F34", "result": "pass", "label": "Workspace session (`F34.workspace-session`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-11-workspace-session-f34-workspace-session.json", "failed_command": null }, { "feature_id": "F34.conversation-create", "family": "F34", "result": "pass", "label": "Create a conversation (`F34.conversation-create`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-12-create-a-conversation-f34-conversation-c.json", "failed_command": null }, { "feature_id": "F34.conversation-run", "family": "F34", "result": "pass", "label": "Send and run (`F34.conversation-run`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-13-send-and-run-f34-conversation-run.json", "failed_command": null }, { "feature_id": "F34.events-list", "family": "F34", "result": "pass", "label": "Events list (`F34.events-list`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-14-events-list-f34-events-list.json", "failed_command": null }, { "feature_id": "F34.events-unknown-id", "family": "F34", "result": "xfail", "label": "Unknown event id (`F34.events-unknown-id`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-15-unknown-event-id-f34-events-unknown-id-k.json", "failed_command": "jq -e '.single.http_error and .single.status == 404' \"$F34/unknown-event.json\"" }, { "feature_id": "F34.events-search-kind", "family": "F34", "result": "xfail", "label": "Short event kinds (`F34.events-search-kind`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-16-short-event-kinds-f34-events-search-kind.json", "failed_command": "jq -e '.short == .full and .short_count == 1' \"$F34/kind.json\"" }, { "feature_id": "F34.event-stream", "family": "F34", "result": "pass", "label": "Event stream on Node's WebSocket (`F34.event-stream`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-17-event-stream-on-node-s-websocket-f34-eve.json", "failed_command": null }, { "feature_id": "F34.ws-callback-client", "family": "F34", "result": "xfail", "label": "Conversation callback socket from Node (`F34.ws-callback-client`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-18-conversation-callback-socket-from-node-f.json", "failed_command": "jq -e '(.kinds | index(\"ConversationStateUpdateEvent\")) != null and .errors == []' \"$F34/callback.json\"" }, { "feature_id": "F34.conversation-controls", "family": "F34", "result": "pass", "label": "Title and confirmation policy (`F34.conversation-controls`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-19-title-and-confirmation-policy-f34-conver.json", "failed_command": null }, { "feature_id": "F34.manager", "family": "F34", "result": "pass", "label": "Conversation manager (`F34.manager`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-20-conversation-manager-f34-manager.json", "failed_command": null }, { "feature_id": "F34.manager-update-result", "family": "F34", "result": "xfail", "label": "Update result (`F34.manager-update-result`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-21-update-result-f34-manager-update-result-.json", "failed_command": "jq -e --arg id \"$SCID\" '.id == $id and .title == \"qa-f34-renamed\"' \"$F34/update.json\"" }, { "feature_id": "F34.manager-create-options", "family": "F34", "result": "xfail", "label": "Create options (`F34.manager-create-options`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-22-create-options-f34-manager-create-option.json", "failed_command": "jq -e '.max_iterations == 7 and .stuck_detection == false' \"$F34/create-options.json\"" }, { "feature_id": "F34.manager-acp", "family": "F34", "result": "pass", "label": "ACP conversations (`F34.manager-acp`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-23-acp-conversations-f34-manager-acp.json", "failed_command": null }, { "feature_id": "F34.conversation-branching", "family": "F34", "result": "pass", "label": "Fork, navigate and ask (`F34.conversation-branching`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-24-fork-navigate-and-ask-f34-conversation-b.json", "failed_command": null }, { "feature_id": "F34.conversation-confirm", "family": "F34", "result": "pass", "label": "Confirmation and secrets (`F34.conversation-confirm`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-25-confirmation-and-secrets-f34-conversatio.json", "failed_command": null }, { "feature_id": "F34.hooks-client", "family": "F34", "result": "pass", "label": "Hooks (`F34.hooks-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-26-hooks-f34-hooks-client.json", "failed_command": null }, { "feature_id": "F34.mcp-client", "family": "F34", "result": "pass", "label": "MCP test (`F34.mcp-client`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-27-mcp-test-f34-mcp-client.json", "failed_command": null }, { "feature_id": "F34.restart-reload", "family": "F34", "result": "pass", "label": "Restart (`F34.restart-reload`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-28-restart-f34-restart-reload.json", "failed_command": null }, { "feature_id": "F34.event-stream-reconnect", "family": "F34", "result": "fail", "label": "Event stream reconnect (`F34.event-stream-reconnect`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-29-event-stream-reconnect-f34-event-stream-.json", "failed_command": "jq -e '.ws | .went_down and .attempts >= 1 and .reconnected and .replayed_after >= 1 and .sent_after >= 1' \"$F34/reconnect-ws.json\"" }, { "feature_id": "F34.event-stream-reconnect-native", "family": "F34", "result": "fail", "label": "Reconnect on Node's own WebSocket (`F34.event-stream-reconnect-native`), known bug.", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-30-reconnect-on-node-s-own-websocket-f34-ev.json", "failed_command": "RNID=$(control-agent-server conversation start --no-run --placeholder-agent --tools none --no-autotitle --workspace \"$F34/scoped\" --prompt 'qa-f34-reconnect: never run' --print-id)" }, { "feature_id": "F34.endpoint-audit", "family": "F34", "result": "fail", "label": "Endpoint audit against this checkout (`F34.endpoint-audit`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-31-endpoint-audit-against-this-checkout-f34.json", "failed_command": "mkdir -p \"$A/home\"" }, { "feature_id": "F34.version-gate-old", "family": "F34", "result": "fail", "label": "Old server version (`F34.version-gate-old`).", "run": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34", "transcript": "/workspace/verification-artifacts/runs/20261013-022019-5e33-map-f34/evidence/map-run/F34-typescript-client-32-old-server-version-f34-version-gate-old.json", "failed_command": "mkdir -p \"$OLD/home\" \"$OLD/persist\"" }, { "feature_id": "F03.auth-before-gate", "family": "F03", "result": "retired", "label": "retired by #5700. It asserted that a dormant pod with boot session keys answered 401 to a missing key before the 503 gate; readiness is now checked first, see `F03.gate-before-auth`.", "run": null, "transcript": null }, { "feature_id": "F12.discovery", "family": "F12", "result": "not-run", "label": "the OpenAPI document lists the two runtime routes but not the pause and credential-binding routes, and `/server_info` advertises `credential_binding_v1`, `credential_binding_readiness_probe_v1` and `credential_binding_activation_guard_v1`.", "run": null, "transcript": null }, { "feature_id": "F12.runtime-info", "family": "F12", "result": "not-run", "label": "`GET .../runtime` returns 200 `{\"runtime_status\": \"available\", \"can_resume\": true, \"runtime_error\": null}` for a local conversation, and the conversation's GET, search and batch-get rows embed the same object as `runtime_info`.", "run": null, "transcript": null }, { "feature_id": "F12.runtime-reprovision", "family": "F12", "result": "not-run", "label": "`POST .../runtime/reprovision` returns the same runtime info, ignores a body, and changes nothing (status, event count and lease generation stay the same).", "run": null, "transcript": null }, { "feature_id": "F12.runtime-errors", "family": "F12", "result": "not-run", "label": "both runtime routes answer 404 for an unknown id, 422 for a non-UUID id and 401 without a session key.", "run": null, "transcript": null }, { "feature_id": "F12.ts-client", "family": "F12", "result": "not-run", "label": "the built TypeScript client's `ConversationClient.getRuntime` and `reprovisionRuntime` return the local runtime info, `getConversation()` embeds it as `runtime_info`, and both methods reject an unknown id with a 404 `HttpError` and a client without a key with 401.", "run": null, "transcript": null }, { "feature_id": "F12.sandbox-pause-releases", "family": "F12", "result": "not-run", "label": "`prepare-for-sandbox-pause` answers 204 with an empty body, deletes every `owner_lease.json`, keeps every conversation listed and readable with its status, is idempotent, and is refused with 401 (leaving the leases) without a key.", "run": null, "transcript": null }, { "feature_id": "F12.runtime-cold", "family": "F12", "result": "not-run", "label": "runtime info answers for a conversation that is not loaded, before and after a restart, without loading it (no lease file appears).", "run": null, "transcript": null }, { "feature_id": "F12.sandbox-pause-rehydrate", "family": "F12", "result": "not-run", "label": "after a pause (no restart in between), the next event access loads the conversation again (a new lease, the same events), and a new message runs to `finished` on a real model, which acts again.", "run": null, "transcript": null }, { "feature_id": "F12.sandbox-pause-running", "family": "F12", "result": "not-run", "label": "a conversation whose model call is in flight is cancelled to `paused` within the server's 10 s bound, with a `PauseEvent` and an `InterruptEvent`, and the status change is pushed on its events socket.", "run": null, "transcript": null }, { "feature_id": "F12.sandbox-pause-websocket", "family": "F12", "result": "not-run", "label": "an events socket or a session socket that was open before the pause is either closed by the server or keeps receiving the conversation's new events; it is never left open and silent.", "run": null, "transcript": null }, { "feature_id": "F12.binding-unsupported-secret", "family": "F12", "result": "not-run", "label": "only file credentials can be bound: any other secret name gets 404 without the source being contacted.", "run": null, "transcript": null }, { "feature_id": "F12.binding-validation", "family": "F12", "result": "not-run", "label": "a body with extra keys, no `url`, an empty or over-long (4097+ characters) `url`, an empty or over-long (8193+ characters) header value or more than 16 headers, or a non-UUID id, gets 422 without the source being contacted, also for an unsupported secret name (the body is validated first); no key gets 401.", "run": null, "transcript": null }, { "feature_id": "F12.binding-pending", "family": "F12", "result": "not-run", "label": "a binding for a conversation id the server does not know answers 204 after exactly one probe `GET` that carries the given headers; the id stays unknown (404).", "run": null, "transcript": null }, { "feature_id": "F12.binding-source-statuses", "family": "F12", "result": "not-run", "label": "the source's answer maps to the activation status: 401/403 to 403, 400/404/422 to 422, 409 to 409, 501 to 501, a 200 that is not `{\"value\": str, \"version\": non-empty str}` to 422, and a URL that is not http(s) to 422, each after a single probe (none for the bad URL).", "run": null, "transcript": null }, { "feature_id": "F12.binding-source-unavailable", "family": "F12", "result": "not-run", "label": "a source answering 5xx is probed three times with backoff (at least 0.75 s), then the activation fails with 502; an unreachable source also gives 502.", "run": null, "transcript": null }, { "feature_id": "F12.binding-native-too-late", "family": "F12", "result": "not-run", "label": "binding a loaded regular (non-ACP) conversation is refused with 409 `Credential binding activation arrived after ACP initialization` and records nothing.", "run": null, "transcript": null }, { "feature_id": "F12.binding-native-cold", "family": "F12", "result": "not-run", "label": "a binding sent while a regular (non-ACP) conversation is not loaded never makes that conversation demand a credential binding after the next pause.", "run": null, "transcript": null }, { "feature_id": "F12.binding-live-acp", "family": "F12", "result": "not-run", "label": "binding a loaded ACP Codex conversation that has not run yet answers 204, records `required_runtime_credential_bindings: [\"CODEX_AUTH_JSON\"]` in `meta.json` and removes the stored `CODEX_AUTH_JSON` secret.", "run": null, "transcript": null }, { "feature_id": "F12.binding-rebind", "family": "F12", "result": "not-run", "label": "a second activation with the same URL and new headers is accepted (the probe carries the new headers, which are never written to disk); a different URL is refused with 409.", "run": null, "transcript": null }, { "feature_id": "F12.binding-required-after-pause", "family": "F12", "result": "not-run", "label": "after a pause, a conversation that requires a binding is still listed and reports runtime `available`, but event access and `/run` answer 409 `{\"detail\": \"credential_binding_activation_required\", \"retryable\": true}` and both conversation sockets close with 1013, until a new activation, after which events load again.", "run": null, "transcript": null }, { "feature_id": "F12.binding-required-after-restart", "family": "F12", "result": "not-run", "label": "the requirement survives a restart (409 until a new activation, then 200), and deleting such a conversation needs no activation.", "run": null, "transcript": null }, { "feature_id": "F12.binding-pending-on-create", "family": "F12", "result": "not-run", "label": "a pending binding is consumed when a conversation is created with that `conversation_id`, which then requires the binding from the start and stores no `CODEX_AUTH_JSON` secret.", "run": null, "transcript": null }, { "feature_id": "F12.binding-pending-cleared", "family": "F12", "result": "not-run", "label": "a pause drops pending bindings, so a conversation created afterwards with that id keeps its own `CODEX_AUTH_JSON` secret and requires no binding.", "run": null, "transcript": null }, { "feature_id": "F12.binding-profile-scope", "family": "F12", "result": "not-run", "label": "a conversation launched from an agent profile whose `secret_refs` exclude `CODEX_AUTH_JSON` refuses the binding with 403 `Credential binding authorization was rejected` and records nothing.", "run": null, "transcript": null }, { "feature_id": "F12.runtime-launched-profile", "family": "F12", "result": "not-run", "label": "on a local server started with `OH_RUNTIME_LAUNCHED_PROFILE` (the launched profile a Docker parent passes into each conversation container), a conversation created without an agent profile records that profile as `launched_agent_profile` and its `secret_refs` scope the credential binding (403), while a conversation started from its own agent profile keeps its own profile (204).", "run": null, "transcript": null }, { "feature_id": "F12.managed-key-refresh", "family": "F12", "result": "not-run", "label": "with `OH_LLM_API_KEY_REFRESH_URL`, `OH_LLM_API_KEY_REFRESH_BASE_URLS` naming the LLM's `base_url` and `OH_LLM_API_KEY_REFRESH_HEADERS` (values expanded from the server environment), a model call rejected with 401 makes exactly one `GET` of the refresh URL with the expanded headers and one retry, and the run finishes; the fetched key is not written to the server's state.", "run": null, "transcript": null }, { "feature_id": "F12.managed-key-scope", "family": "F12", "result": "not-run", "label": "an LLM whose `base_url` is outside the allow-list, or any LLM once the allow-list is empty, surfaces the 401 (`LLMAuthenticationError`, status `error`) after one model call and no refresh request; the empty allow-list is logged as \"stays off\".", "run": null, "transcript": null }, { "feature_id": "F12.managed-key-switch", "family": "F12", "result": "not-run", "label": "an in-scope LLM installed with `switch_llm` (or `switch_profile`) recovers from a 401 through the refresh URL like the LLM the conversation was loaded with.", "run": null, "transcript": null }, { "feature_id": "F12.docker-runtime-contract", "family": "F12", "result": "not-run", "label": "on a server with `OH_CONVERSATION_RUNTIME=docker` and no Docker daemon, fork, `switch_llm`, `switch_profile` and credential bindings answer 501 before any lookup (the binding source is never contacted); the runtime routes, `DELETE .../runtime`, `POST .../runtime/credentials` and `/run` answer 404 `Conversation not found` for an unknown id, and the two Docker-only routes 401 without a key; both conversation sockets close with 1008 for an unknown id; create refuses a missing body (400), a non-object (422), a non-local workspace (422) and a bad `conversation_id` (400) without writing a runtime manifest; a well-formed create answers 502, leaving a manifest whose runtime info is 404, whose reprovision is 502 and whose socket closes with 1011, until `DELETE /{id}` removes it.", "run": null, "transcript": null }, { "feature_id": "F12.docker-runtime", "family": "F12", "result": "not-run", "label": "with a Docker daemon, a create starts a container with the configured `--memory`, `--cpus` and `--pids-limit`, the launched profile in `OH_RUNTIME_LAUNCHED_PROFILE` and a startup bound of `conversation_container_startup_timeout`; `POST .../runtime/credentials` returns the container's session key, `DELETE .../runtime` stops it (204) while the conversation stays readable, reprovision starts it again, a request carrying `X-Expose-Secrets` is refused with 422 and an archived runtime answers 410 (blocked here: no Docker daemon).", "run": null, "transcript": null }, { "feature_id": "F14.auth", "family": "F14", "result": "not-run", "label": "every global and conversation-scoped file route answers 401 without a valid session key, and nothing is written.", "run": null, "transcript": null }, { "feature_id": "F14.upload-download", "family": "F14", "result": "not-run", "label": "a multipart upload creates missing parent directories, a download returns byte-identical bytes as an `application/octet-stream` attachment named after the basename, and a second upload to the same path replaces the content.", "run": null, "transcript": null }, { "feature_id": "F14.upload-errors", "family": "F14", "result": "not-run", "label": "an upload without the `file` part or without `path` is 422, and a relative `path` is 400 `Path must be absolute`.", "run": null, "transcript": null }, { "feature_id": "F14.upload-onto-directory", "family": "F14", "result": "not-run", "label": "uploading onto an existing directory, or to a path whose parent is a file, is a client error (4xx); today it is a masked 500.", "run": null, "transcript": null }, { "feature_id": "F14.upload-size", "family": "F14", "result": "not-run", "label": "the SDK's `RemoteWorkspace.file_upload` and the TypeScript `RemoteWorkspace.fileUpload` report the uploaded `file_size`, as `LocalWorkspace.file_upload` does; today the server sends only `{\"success\": true}` and both report no size.", "run": null, "transcript": null }, { "feature_id": "F14.upload-atomic", "family": "F14", "result": "not-run", "label": "an overwrite that fails mid-write leaves the previous file intact; today the target is truncated before streaming and keeps a partial file.", "run": null, "transcript": null }, { "feature_id": "F14.download-errors", "family": "F14", "result": "not-run", "label": "downloading a missing file is 404 `File not found`, a directory 400 `Path is not a file`, a relative path 400.", "run": null, "transcript": null }, { "feature_id": "F14.mkdir", "family": "F14", "result": "not-run", "label": "`create_directory` creates missing parents, is idempotent and keeps existing contents, the new directory is listed by `search_subdirs`, and a path that is or runs through a file is 400.", "run": null, "transcript": null }, { "feature_id": "F14.home", "family": "F14", "result": "not-run", "label": "`GET /api/file/home` reports the server user's `HOME`, its visible top-level directories as case-insensitively sorted `favorites` (no files, no symlinks), `/` as the only location, and hidden directories only with `include_hidden=true`.", "run": null, "transcript": null }, { "feature_id": "F14.subdirs", "family": "F14", "result": "not-run", "label": "`search_subdirs` lists only the immediate real subdirectories of an absolute path (no files, no symlinks, hidden only with `include_hidden=true`), sorted case-insensitively, with absolute paths.", "run": null, "transcript": null }, { "feature_id": "F14.subdirs-pagination", "family": "F14", "result": "not-run", "label": "`limit` and the `next_page_id` cursor page through the entries; `limit` outside 1..100 is 422.", "run": null, "transcript": null }, { "feature_id": "F14.subdirs-errors", "family": "F14", "result": "not-run", "label": "`search_subdirs` on a missing directory is 404, on a file 400, on a relative path 400.", "run": null, "transcript": null }, { "feature_id": "F14.subdirs-case-pagination", "family": "F14", "result": "not-run", "label": "following `next_page_id` across two directories that differ only by case (`Foo`, `foo`) reaches the second one instead of repeating the first page forever.", "run": null, "transcript": null }, { "feature_id": "F14.archive-tar", "family": "F14", "result": "not-run", "label": "`format=tar.gz` returns an `application/gzip` archive named `.tar.gz`, rooted at `/`, with `archive_manifest.json`, the repository identity headers but no `X-Archive-Base-*`, the default excludes applied (`.git/`, `node_modules/`), symlinks skipped, and no temp file left in the parent directory.", "run": null, "transcript": null }, { "feature_id": "F14.archive-excludes", "family": "F14", "result": "not-run", "label": "`use_default_excludes=false` keeps `node_modules` and `.git` internals but never `.git/config` or reflogs; `exclude=` (repeatable) drops matching files and is recorded in the manifest.", "run": null, "transcript": null }, { "feature_id": "F14.archive-git-delta", "family": "F14", "result": "not-run", "label": "the default `git-delta` format with `base_ref=HEAD` is a `text/x-patch` of the uncommitted and untracked changes that applies to a clone of HEAD, carries `X-Archive-Base-Commit`, `X-Archive-Base-Ref`, `X-Archive-Branch` and `X-Archive-Head-Commit`, and leaves the repository's index untouched; without `base_ref` on a remote-less repo the patch is against the empty tree and has no base headers; a subdirectory of a repository yields only that subtree's delta.", "run": null, "transcript": null }, { "feature_id": "F14.archive-auto-base", "family": "F14", "result": "not-run", "label": "without `base_ref`, a clone with an `origin` remote is diffed against `origin/` (the patch holds the local commit and the uncommitted edit, applies to a clone of the origin tip, and carries `X-Archive-Base-Ref: auto`, `X-Archive-Base-Commit` = the origin tip and `X-Archive-Repo-Remote`); a detached HEAD is diffed against HEAD itself.", "run": null, "transcript": null }, { "feature_id": "F14.archive-nested-repo", "family": "F14", "result": "not-run", "label": "archiving a non-repo directory that holds exactly one git repository archives that repository; with two repositories `git-delta` is 400 and `tar.gz` archives the directory itself.", "run": null, "transcript": null }, { "feature_id": "F14.archive-format-doc", "family": "F14", "result": "not-run", "label": "the OpenAPI description of `format` matches what a non-repository directory holding one repository archives; today it promises a tarball of \"the entire directory\" while only the nested repository is packed.", "run": null, "transcript": null }, { "feature_id": "F14.archive-errors", "family": "F14", "result": "not-run", "label": "a relative path is 400, a missing directory 404, a file 400, a `base_ref` starting with `-` 400, an unknown `base_ref` 400, an unknown `format` 422, and `git-delta` of a non-repo directory 400.", "run": null, "transcript": null }, { "feature_id": "F14.trajectory", "family": "F14", "result": "not-run", "label": "`GET /api/file/download-trajectory/{id}` returns `.zip` with `meta.json`, `base_state.json` and one file per event, the LLM `api_key` redacted to `` and no Fernet ciphertext, for dashed and hex ids, and leaves no temp zip behind.", "run": null, "transcript": null }, { "feature_id": "F14.trajectory-errors", "family": "F14", "result": "not-run", "label": "an unknown conversation id is 404 `Conversation not found`, a malformed id 422.", "run": null, "transcript": null }, { "feature_id": "F14.trajectory-concurrent", "family": "F14", "result": "not-run", "label": "concurrent downloads of the same trajectory each return 200 and a valid zip (five rounds of six parallel downloads, 30 in all).", "run": null, "transcript": null }, { "feature_id": "F14.trajectory-init-path", "family": "F14", "result": "not-run", "label": "after `POST /api/init` sets `conversations_path` on a deferred-init server, trajectories stored there are downloadable.", "run": null, "transcript": null }, { "feature_id": "F14.scoped-transfer", "family": "F14", "result": "not-run", "label": "conversation-scoped upload, download and `create_directory` work inside the conversation's workspace, and the scoped download returns the file the agent wrote.", "run": null, "transcript": null }, { "feature_id": "F14.scoped-path-guard", "family": "F14", "result": "not-run", "label": "scoped routes reject paths outside the workspace (another conversation's workspace, `..` traversal, a symlink escape, a sibling with the same prefix, a relative path) with 422 and write nothing, while the global route serves the same path.", "run": null, "transcript": null }, { "feature_id": "F14.scoped-conversation-errors", "family": "F14", "result": "not-run", "label": "a scoped route for an unknown conversation is 404 and does nothing; a malformed conversation id is 422.", "run": null, "transcript": null }, { "feature_id": "F14.scoped-trajectory-archive", "family": "F14", "result": "not-run", "label": "the scoped trajectory route serves only its own conversation (another or malformed id is 422), and the scoped archive packs the workspace, including the agent's file, but refuses a path outside it with 422.", "run": null, "transcript": null }, { "feature_id": "F14.sdk-workspace", "family": "F14", "result": "not-run", "label": "the SDK's `RemoteWorkspace.file_upload`/`file_download` round-trip bytes through the global routes and, with `runtime_conversation_id`, through the scoped routes; a scoped upload outside the workspace returns `success=False` with the 422 and writes nothing.", "run": null, "transcript": null }, { "feature_id": "F14.restart", "family": "F14", "result": "not-run", "label": "after a restart, uploaded files, the scoped download and the trajectory are still served.", "run": null, "transcript": null }, { "feature_id": "F30.chat-timeout-504", "family": "F30", "result": "not-run", "label": "a run longer than the fixed 120 s answers 504 `Agent run timed out`, and the run keeps going server-side.", "run": null, "transcript": null }, { "feature_id": "F30.responses-basic", "family": "F30", "result": "not-run", "label": "`POST /v1/responses` on a real model returns a completed `response` with one `output_text` message equal to the agent's final answer, echoes `instructions` and `metadata`, and puts the instructions in the system suffix.", "run": null, "transcript": null }, { "feature_id": "F30.responses-stateless-replay", "family": "F30", "result": "not-run", "label": "every responses call is a new conversation (the conversation header is ignored); replayed items become one user message wrapped in `` markers, and developer items go to the system suffix.", "run": null, "transcript": null }, { "feature_id": "F30.responses-errors", "family": "F30", "result": "not-run", "label": "`stream`, `store: true` and `previous_response_id` are 400 before anything else is checked, bad input is 400 or 422, an unknown model 404, and nothing is created.", "run": null, "transcript": null }, { "feature_id": "F30.openai-sdk", "family": "F30", "result": "not-run", "label": "the official OpenAI Python SDK lists models, reads the conversation header through `with_raw_response`, continues the conversation with `extra_headers`, parses a stream and creates a response.", "run": null, "transcript": null }, { "feature_id": "F31.takeover-keeps-metadata", "family": "F31", "result": "not-run", "label": "a title the owner stored after the second server started survives the second server's takeover, in its GET and in `meta.json`.", "run": null, "transcript": null }, { "feature_id": "F31.lease-expiry-fencing", "family": "F31", "result": "not-run", "label": "an owner that stops renewing (frozen, pid alive) loses the lease once it expires, and after it resumes its writes are refused and never reach disk.", "run": null, "transcript": null }, { "feature_id": "F31.fenced-write-no-trace", "family": "F31", "result": "not-run", "label": "a write refused for lost ownership answers a 4xx and leaves no trace: the stale owner's own GET does not report the refused title.", "run": null, "transcript": null }, { "feature_id": "F31.lease-disabled", "family": "F31", "result": "not-run", "label": "with `lease_ttl_seconds` 0 no lease file is written and a second server loads and writes the same conversation concurrently.", "run": null, "transcript": null }, { "feature_id": "F31.idle-eviction", "family": "F31", "result": "not-run", "label": "an idle conversation is unloaded at the first eviction pass after its TTL (lease deleted, `Evicted idle` logged), stays listed with its title and status, and loads again with the same events on the next access.", "run": null, "transcript": null }, { "feature_id": "F31.eviction-skips-busy", "family": "F31", "result": "not-run", "label": "the same pass keeps a conversation with an open events socket and a running one loaded; once the socket closes and the run is interrupted, the next pass unloads both.", "run": null, "transcript": null }, { "feature_id": "F31.create-capacity", "family": "F31", "result": "not-run", "label": "with every run slot taken, `POST ` answers 429 and creates nothing, on disk or in the catalog; the same request succeeds once the slot is free.", "run": null, "transcript": null }, { "feature_id": "F31.config-precedence", "family": "F31", "result": "not-run", "label": "the configuration file is applied and an `OH_*` variable overrides the same field (seen through the lease TTL).", "run": null, "transcript": null }, { "feature_id": "F31.config-null-override", "family": "F31", "result": "not-run", "label": "`OH__IS_NONE=1` clears a value the configuration file sets, as a non-null `OH_*` value replaces it (seen through `app_backend_public_url` in `/server_info`).", "run": null, "transcript": null }, { "feature_id": "F31.config-invalid-fails-fast", "family": "F31", "result": "not-run", "label": "a non-numeric `OH_MAX_CONCURRENT_RUNS` or an out-of-range limit in the configuration file stops the server during startup instead of serving with a default.", "run": null, "transcript": null }, { "feature_id": "F31.config-silent-env", "family": "F31", "result": "not-run", "label": "an `OH_*` boolean other than `1`/`true` is read as false (it overrides a `true` in the configuration file) and a `Literal` variable outside its choices falls back to the default; the server starts either way, while the same invalid choice in the file stops startup.", "run": null, "transcript": null }, { "feature_id": "F31.tmux-dir-cleanup", "family": "F31", "result": "not-run", "label": "a server that defaulted its own per-process `TMUX_TMPDIR` removes that directory when it stops.", "run": null, "transcript": null }, { "feature_id": "F31.storage-retention", "family": "F31", "result": "not-run", "label": "with the Docker runtime, `conversation_storage_retention_days` archives a stopped conversation inactive past the limit (events readable, resume and proxied routes 410, runtime directory gone), and `conversation_storage_disk_budget` sheds a stopped workspace's gitignored directories while keeping its files (blocked here: no Docker daemon).", "run": null, "transcript": null }, { "feature_id": "F32.telemetry-lifecycle", "family": "F32", "result": "not-run", "label": "with the `http` exporter and consent, `server_started` arrives at boot and `server_stopped` at a graceful stop, as `{\"schema_version\": 1, \"events\": [...]}` with the configured Bearer token.", "run": null, "transcript": null }, { "feature_id": "F32.telemetry-conversation", "family": "F32", "result": "not-run", "label": "a run emits `conversation_created` and `conversation_finished` under the request's `user_id`, with a pseudonymous `conversation_ref` and without the prompt or the conversation id.", "run": null, "transcript": null }, { "feature_id": "F32.telemetry-request-failed", "family": "F32", "result": "not-run", "label": "an unhandled 500 emits `request_failed` with the route template (never the concrete path), the exception class and the response's `error_id`, attributed to `X-OpenHands-Telemetry-Distinct-Id`, without the exception message.", "run": null, "transcript": null }, { "feature_id": "F32.telemetry-conversation-failed", "family": "F32", "result": "not-run", "label": "a run that ends in `error` emits `conversation_failed` (`terminal_status` `error`) and `conversation_error` with the error code as `error_class`, without the error's detail text.", "run": null, "transcript": null }, { "feature_id": "F32.telemetry-usage-buckets", "family": "F32", "result": "not-run", "label": "`conversation_finished` reports token and cost buckets from the run's real usage (reproduces a real gap; see Gotchas).", "run": null, "transcript": null }, { "feature_id": "F32.telemetry-consent-settings", "family": "F32", "result": "not-run", "label": "granting or denying consent with `PATCH /api/settings` `misc_settings_diff` starts or stops delivery immediately, and persisted consent enables telemetry after a restart.", "run": null, "transcript": null }, { "feature_id": "F32.telemetry-env-override", "family": "F32", "result": "not-run", "label": "`OH_TELEMETRY_CONSENT=denied` with `OH_TELEMETRY_CONSENT_MODE=override` beats a granted setting.", "run": null, "transcript": null }, { "feature_id": "F32.telemetry-kill-switch", "family": "F32", "result": "not-run", "label": "`DO_NOT_TRACK=1` beats every consent source.", "run": null, "transcript": null }, { "feature_id": "F32.telemetry-salt-and-kind", "family": "F32", "result": "not-run", "label": "without a salt `conversation_ref` is keyed by `OH_SECRET_KEY` (the same ref across restarts); `OH_TELEMETRY_SALT` re-keys it (again the same across restarts), and `OH_TELEMETRY_DEPLOYMENT_KIND` tags every event.", "run": null, "transcript": null }, { "feature_id": "F32.telemetry-salt-docs", "family": "F32", "result": "not-run", "label": "the `TelemetrySpec.salt` description published in the OpenAPI schema states the real fallback key (reproduces a documentation bug; see Gotchas).", "run": null, "transcript": null }, { "feature_id": "F32.deferred-init", "family": "F32", "result": "not-run", "label": "a dormant pod sends nothing until `POST /api/init` delivers `telemetry` and `webhooks`; then `server_started`, conversation telemetry and webhooks flow.", "run": null, "transcript": null }, { "feature_id": "F32.deferred-init-flag", "family": "F32", "result": "not-run", "label": "events from an initialized deferred pod report `deferred_init: true` (reproduces a real mislabel; see Gotchas).", "run": null, "transcript": null }, { "feature_id": "F33.workspace-runtime-lifecycle", "family": "F33", "result": "not-run", "label": "`get_runtime_session_key()` and `release_runtime()` hand out a scoped worker key and stop a conversation's runtime on a server with a Docker conversation runtime.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-streaming", "family": "F33", "result": "not-run", "label": "with a `stream=True` agent LLM, callbacks receive the run's `StreamingDeltaEvent` frames live and the server stores none of them.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-token-callbacks", "family": "F33", "result": "not-run", "label": "`Conversation(token_callbacks=[...])` on a remote workspace calls the token callbacks with the run's streamed deltas.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-streaming-state", "family": "F33", "result": "not-run", "label": "streamed deltas stay out of `state.events`, which keeps matching the server's stored events.", "run": null, "transcript": null }, { "feature_id": "F33.completion-logs", "family": "F33", "result": "not-run", "label": "an agent LLM with `log_completions=True` gets one log file per model call in its `log_completions_folder`, written by the client from the `LLMCompletionLogEvent`s the server streams and stores (the server writes no file).", "run": null, "transcript": null }, { "feature_id": "F33.conversation-attach", "family": "F33", "result": "not-run", "label": "a second program reattaches by id with `RemoteConversation.attach()` or `Conversation(conversation_id=...)` and replays the same history without creating anything; `attach()` on an unknown id raises 404 while `Conversation(conversation_id=)` creates that id.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-attach-large", "family": "F33", "result": "not-run", "label": "attaching to a conversation with more than 100 stored events pages through the whole history: `state.events` holds the server's events in the server's order (plus the one unpersisted socket snapshot), and the first reads of `state.confirmation_policy`, `state.security_analyzer`, `state.hook_config` and `state.agent` equal the server's.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-state-reread", "family": "F33", "result": "not-run", "label": "reading `state.confirmation_policy`, `state.security_analyzer`, `state.agent` and `is_confirmation_mode_active` a second time gives the same answer as the first.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-ask-title", "family": "F33", "result": "not-run", "label": "`ask_agent()` answers without adding events, `generate_title()` returns a title within `max_length` without adding events, and `set_title()` changes the server's title.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-fork", "family": "F33", "result": "not-run", "label": "`fork(title=..., tags=...)` returns a `RemoteConversation` for a new server conversation with the same history, the title and the tags; `fork(agent=...)` is refused client-side.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-fork-branch", "family": "F33", "result": "not-run", "label": "`fork(from_event_id=..., reset_metrics=False)` returns a fork whose history ends at that event (its HEAD) with the source's stats, and `navigate_to(None)` and `navigate_to(event_id)` move the fork's HEAD without changing its events.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-plugins-tag", "family": "F33", "result": "not-run", "label": "`Conversation(plugins=[PluginSource(...)])` with inline URL credentials stores the `plugins` tag with the credentials masked, and the token reaches no server state.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-confirmation", "family": "F33", "result": "not-run", "label": "`set_confirmation_policy(AlwaysConfirm())` makes `run()` return at `waiting_for_confirmation`; `reject_pending_actions()` records a `UserRejectObservation`, never runs the action and leaves the conversation `idle`.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-pause", "family": "F33", "result": "not-run", "label": "`pause()` from another thread during a blocking `run()` lets the tool step in flight finish, makes `run()` return with `paused` and a `PauseEvent`, and a later `run()` resumes to `finished`.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-interrupt", "family": "F33", "result": "not-run", "label": "`interrupt()` from another thread while a blocking `run()` waits on a model call makes `run()` return at once without raising, at `paused` with one `InterruptEvent` and no `PauseEvent`, and a later `run()` reaches `finished`.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-run-timeout", "family": "F33", "result": "not-run", "label": "`run(timeout=...)` raises `ConversationRunError` wrapping a `TimeoutError` on time while the server keeps running, and a second `run()` joins the run in flight (the server's 409 is not an error) and returns at `finished`.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-run-join-long-tool", "family": "F33", "result": "not-run", "label": "a second `run()` also joins a run whose current tool call lasts longer than 30 s, instead of raising.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-stuck", "family": "F33", "result": "not-run", "label": "when the server's stuck detector stops a looping agent, `run()` raises `ConversationRunError` (\"got stuck\") at once and the server reports `stuck`.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-secrets", "family": "F33", "result": "not-run", "label": "`update_secrets()` with a plain value and with a `LookupSecret` from `get_secrets()` makes both available to the agent's commands, masked in observations and not stored in plaintext.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-run-error", "family": "F33", "result": "not-run", "label": "when the model provider rejects the key, `run()` raises `ConversationRunError` carrying the server's `ConversationErrorEvent` code, and the server reports `error`.", "run": null, "transcript": null }, { "feature_id": "F33.ws-unavailable", "family": "F33", "result": "not-run", "label": "when the events socket cannot connect, `Conversation(...)` raises `WebSocketConnectionError` after `OPENHANDS_REMOTE_WS_READY_TIMEOUT` seconds (the conversation it created stays on the server); with `OPENHANDS_REMOTE_WS_READY_REQUIRED=false` it continues, and `run()` returns at `finished` through the REST fallback about 30 s after the server finished, with `state.events` matching the server.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-reconnect", "family": "F33", "result": "not-run", "label": "a live `RemoteConversation` survives a server restart: its events socket reconnects, a new message runs to `finished`, and `state.events` again matches the server.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-reconnect-callbacks", "family": "F33", "result": "not-run", "label": "an event the server records while a live `RemoteConversation`'s events socket is down reaches its callbacks once the socket is back, not only `state.events`.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-restart", "family": "F33", "result": "not-run", "label": "after a restart a new program reattaches to a finished conversation, sees the same history, and continues it to `finished`.", "run": null, "transcript": null }, { "feature_id": "F33.custom-tool-module", "family": "F33", "result": "not-run", "label": "a tool registered in the client process is sent as `tool_module_qualnames`, imported by a server started with `OH_EXTRA_PYTHON_PATH`, and runs on the server when the agent calls it; a client-registered module the server cannot import is only a logged warning; reattaching from a process without the module raises `ImportError` naming the tool.", "run": null, "transcript": null }, { "feature_id": "F33.custom-tool-module-unimportable", "family": "F33", "result": "not-run", "label": "a conversation whose agent names a tool from a module the server cannot import fails with a client error (a 4xx, or a recorded conversation error), never with a 500.", "run": null, "transcript": null }, { "feature_id": "F33.conversation-close", "family": "F33", "result": "not-run", "label": "`close()` keeps the server conversation when `delete_on_close=False` and deletes it when the `Conversation(...)` factory's default `delete_on_close=True` applies.", "run": null, "transcript": null } ], "feature_definitions": [ { "feature_id": "F01.alive", "description": "`GET /alive` returns 200 `{\"status\": \"ok\"}` without a session key, while a protected route refuses the same key-less request.", "file": "F01-server-status.md", "line": 19 }, { "feature_id": "F01.health", "description": "`GET /health` returns 200 `{\"status\": \"ok\"}` without a session key.", "file": "F01-server-status.md", "line": 20 }, { "feature_id": "F01.ready", "description": "`GET /ready` returns 200 `{\"status\": \"ready\"}` once startup finished, without a session key.", "file": "F01-server-status.md", "line": 21 }, { "feature_id": "F01.server-info", "description": "`GET /server_info` reports package versions, `build_git_sha` equal to the served checkout, usable tools and capability flags, without a session key.", "file": "F01-server-status.md", "line": 22 }, { "feature_id": "F01.root-info", "description": "without static files, `GET /` returns the same server info document as `/server_info` (every field but the clocks).", "file": "F01-server-status.md", "line": 23 }, { "feature_id": "F01.uptime-restart", "description": "`uptime` grows while the process lives and starts again from zero after a restart.", "file": "F01-server-status.md", "line": 24 }, { "feature_id": "F01.idle-time", "description": "`idle_time` grows while nothing happens and resets when a file operation runs.", "file": "F01-server-status.md", "line": 25 }, { "feature_id": "F01.openapi", "description": "`GET /openapi.json` serves the OpenAPI document with the conversation routes and without the WebSockets.", "file": "F01-server-status.md", "line": 26 }, { "feature_id": "F01.static-files", "description": "with `static_files_path` set to an existing directory, `GET /` answers 302 to `/static/index.html` (or `/static/` without an `index.html`), `/static/*` serves the directory without a session key and refuses to escape it, `/api/*` still needs the key and `/server_info` is unchanged; a missing directory keeps `/` as server info.", "file": "F01-server-status.md", "line": 27 }, { "feature_id": "F01.debug-tracebacks", "description": "with `DEBUG=true` a 5xx body carries a `traceback` next to `detail` and `exception`; without it, or on a 4xx, it does not.", "file": "F01-server-status.md", "line": 28 }, { "feature_id": "F01.preload-tools", "description": "with `preload_tools` on, startup runs the tool preload before `/ready` answers 200; when the server finds no Chromium the preload fails, is logged and skipped and the server still serves, and with a Chromium it can find the preload succeeds.", "file": "F01-server-status.md", "line": 29 }, { "feature_id": "F02.http-key-required", "description": "with keys configured, `/api/*` routes answer 401 `{\"detail\": \"Unauthorized\"}` to a missing or wrong `X-Session-API-Key` and serve requests that carry the run's key.", "file": "F02-auth-and-sessions.md", "line": 21 }, { "feature_id": "F02.http-header-only", "description": "regular `/api/*` routes accept the key only in the header: the workspace cookie, the `session_api_key` query parameter and `Authorization: Bearer` all get 401.", "file": "F02-auth-and-sessions.md", "line": 22 }, { "feature_id": "F02.ws-first-frame", "description": "a first frame `{\"type\": \"auth\", \"session_api_key\": K}` authenticates `/sockets/events/{id}`, `/sockets/session/{id}` and `/sockets/bash-events`.", "file": "F02-auth-and-sessions.md", "line": 23 }, { "feature_id": "F02.ws-first-frame-reject", "description": "a wrong key, a wrong `type`, a missing key or a non-object first frame closes the socket with 4001 `Authentication failed`, before the conversation is looked up.", "file": "F02-auth-and-sessions.md", "line": 24 }, { "feature_id": "F02.ws-auth-timeout", "description": "a socket that sends no first frame is closed with 4001 after about 10 seconds.", "file": "F02-auth-and-sessions.md", "line": 25 }, { "feature_id": "F02.ws-legacy-key", "description": "the deprecated `session_api_key` query parameter and `X-Session-API-Key` header still open a socket, and a redundant auth frame sent after them is ignored.", "file": "F02-auth-and-sessions.md", "line": 26 }, { "feature_id": "F02.ws-legacy-reject", "description": "a wrong legacy key rejects the handshake itself (HTTP 403, no close frame), even when a correct first frame would follow.", "file": "F02-auth-and-sessions.md", "line": 27 }, { "feature_id": "F02.workspace-session-mint", "description": "`POST /api/auth/workspace-session` answers 204 with an HttpOnly `oh_workspace_session_key` cookie on `Path=`, `SameSite=none; Secure; Partitioned` on a loopback host, and 401 with no cookie without a valid header (the cookie itself cannot mint another).", "file": "F02-auth-and-sessions.md", "line": 28 }, { "feature_id": "F02.workspace-cookie-access", "description": "the minted cookie alone (a cookie jar, no header) loads workspace files, also after a restart; a bogus cookie or no credential gets 401.", "file": "F02-auth-and-sessions.md", "line": 29 }, { "feature_id": "F02.workspace-session-delete", "description": "`DELETE /api/auth/workspace-session` answers 204 with a `Max-Age=0` cookie on the same path, which drops the cookie from the jar so workspace files are 401 again; 401 without a header.", "file": "F02-auth-and-sessions.md", "line": 30 }, { "feature_id": "F02.multiple-keys", "description": "every configured key (`OH_SESSION_API_KEYS_0`, `_1`, ...) authenticates HTTP and WebSocket clients; an unlisted key does not.", "file": "F02-auth-and-sessions.md", "line": 31 }, { "feature_id": "F02.rotate-key", "description": "after a restart with a rotated key, the old key gets 401 on HTTP and 4001 on WebSockets and its workspace cookie stops working, while the new key works and conversations persist.", "file": "F02-auth-and-sessions.md", "line": 32 }, { "feature_id": "F02.keys-hidden-from-commands", "description": "commands the server runs do not see `OH_SESSION_API_KEYS_*`, `SESSION_API_KEY` or `OH_SECRET_KEY` in their environment.", "file": "F02-auth-and-sessions.md", "line": 33 }, { "feature_id": "F02.cors-default-origins", "description": "CORS preflights from `localhost` and `127.0.0.1` origins (any port) are allowed with credentials; other origins and `Origin: null` get 400 `Disallowed CORS origin`, and simple responses carry no `Access-Control-Allow-Origin` for them.", "file": "F02-auth-and-sessions.md", "line": 34 }, { "feature_id": "F02.cors-configured-origins", "description": "`OH_ALLOW_CORS_ORIGINS_*`, `OH_ALLOW_CORS_ORIGIN_REGEX` and `DOCKER_HOST_ADDR` extend the allowed origins after a restart.", "file": "F02-auth-and-sessions.md", "line": 35 }, { "feature_id": "F02.cors-workspace-routes", "description": "the workspace-session and workspace file routes allow any http(s) origin with credentials but refuse `Origin: null`, and that wildcard does not reach other routes.", "file": "F02-auth-and-sessions.md", "line": 36 }, { "feature_id": "F02.no-auth-mode", "description": "a server with no keys serves `/api/*` without a header (ignoring any header), accepts sockets without an auth frame, and mints an empty workspace cookie.", "file": "F02-auth-and-sessions.md", "line": 37 }, { "feature_id": "F02.legacy-env-key", "description": "the legacy `SESSION_API_KEY` variable alone turns authentication on with that key.", "file": "F02-auth-and-sessions.md", "line": 38 }, { "feature_id": "F02.default-bind", "description": "a server started without `--host` listens on `127.0.0.1` when no session key is configured and on `0.0.0.0` when one is; `--host 0.0.0.0` without a key still binds but logs the unauthenticated-bind warning.", "file": "F02-auth-and-sessions.md", "line": 39 }, { "feature_id": "F02.workspace-cookie-root-path", "description": "behind a path prefix (`OH_WEB_URL` with a path), a cookie minted through the prefixed URL is sent back by the browser to the prefixed workspace file URLs.", "file": "F02-auth-and-sessions.md", "line": 40 }, { "feature_id": "F02.workspace-session-ipv6-loopback", "description": "a mint reached through the IPv6 loopback (`Host: [::1]:PORT`) gets `Secure; Partitioned`, like `localhost` and `127.0.0.1`.", "file": "F02-auth-and-sessions.md", "line": 41 }, { "feature_id": "F03.dormant-boot", "description": "a deferred server boots healthy, answers `/alive`, `/ready`, `/server_info`, `GET /` and the OpenAPI document with 200, and reports `{\"state\":\"dormant\",\"error\":null}` on `GET /api/init`.", "file": "F03-deferred-init.md", "line": 39 }, { "feature_id": "F03.dormant-gates-api", "description": "every `/api/*` route except `/api/init` returns 503 while dormant (reads, mutations and the separately mounted workspace file routes), and the body carries the gate's own reason (`exception` = `503: server is in deferred-init state 'dormant'; call POST /api/init first`; `detail` is rewritten to `Internal Server Error` by the 5xx handler).", "file": "F03-deferred-init.md", "line": 40 }, { "feature_id": "F03.dormant-gates-sockets", "description": "while dormant the bash-events, conversation events and session WebSockets refuse the handshake (the gate closes with 1013 before accept, which the client sees as HTTP 403), so no auth frame is ever read.", "file": "F03-deferred-init.md", "line": 41 }, { "feature_id": "F03.dormant-gates-v1", "description": "while dormant the OpenAI-compatible `GET /v1/models` and `POST /v1/chat/completions` answer 503 with the gate's reason, as does the Canvas app-backend bridge (`/app-backends/...`).", "file": "F03-deferred-init.md", "line": 42 }, { "feature_id": "F03.init-key-auth", "description": "`POST /api/init` requires `X-Init-API-Key` equal to the dormant `secret_key`; a missing or wrong key is 401, checked before the body is validated, and leaves the state dormant.", "file": "F03-deferred-init.md", "line": 43 }, { "feature_id": "F03.init-validation", "description": "an unknown field or an out-of-range value is a 422 (`extra_forbidden` / constraint) and the state stays dormant.", "file": "F03-deferred-init.md", "line": 44 }, { "feature_id": "F03.init-redacts-secret-input", "description": "a 422 whose echoed input carries a secret-bearing key (`Authorization` header) redacts it to `` and the raw value appears nowhere in the response.", "file": "F03-deferred-init.md", "line": 45 }, { "feature_id": "F03.init-failure-rollback", "description": "an init that fails mid-build returns 500, rolls the state back to `dormant` with the gate still closed, and records the error on `GET /api/init` so it can be retried.", "file": "F03-deferred-init.md", "line": 46 }, { "feature_id": "F03.init-success", "description": "a valid `POST /api/init` flips the server to `ready`, and new conversations are stored under the delivered `conversations_path`.", "file": "F03-deferred-init.md", "line": 47 }, { "feature_id": "F03.init-delivered-webhooks", "description": "`webhooks` delivered at init (on a pod that booted with none) receive the new conversation's `ConversationInfo` at `{base_url}/conversations`, with the delivered headers and the delivered session key.", "file": "F03-deferred-init.md", "line": 48 }, { "feature_id": "F03.init-enforces-session-keys", "description": "after init, `/api/*` rejects a missing or wrong key with 401 and accepts the delivered session key with 200, on a pod that booted without keys.", "file": "F03-deferred-init.md", "line": 49 }, { "feature_id": "F03.init-applies-env", "description": "`env` entries delivered at init reach `os.environ` and therefore the bash subprocesses, and the delivered `bash_events_dir` receives the events.", "file": "F03-deferred-init.md", "line": 50 }, { "feature_id": "F03.init-keys-gate-bash-socket", "description": "after init, `/sockets/bash-events` closes with 4001 for a client without the delivered key and runs commands for one with it.", "file": "F03-deferred-init.md", "line": 51 }, { "feature_id": "F03.init-opens-events-socket-and-v1", "description": "after init, the conversation events socket that was refused while dormant opens and replays the conversation's state with the delivered key (4001 without it), and `GET /v1/models` answers 401 without a key and 200 with the delivered key as a Bearer token.", "file": "F03-deferred-init.md", "line": 52 }, { "feature_id": "F03.init-web-url-root-path", "description": "a delivered `web_url` with a path becomes the root path: prefixed URLs serve and the OpenAPI document advertises the prefix.", "file": "F03-deferred-init.md", "line": 53 }, { "feature_id": "F03.init-delivered-worktree-root", "description": "a `worktree: true` conversation on a git workspace created after init gets its worktree under the delivered `conversation_worktree_root`, not the boot root.", "file": "F03-deferred-init.md", "line": 54 }, { "feature_id": "F03.init-delivered-max-runs", "description": "a delivered `max_concurrent_runs` of 1 replaces the boot default (10): a second conversation's `/run` is 429 while another run holds the slot, and is accepted once the slot is free.", "file": "F03-deferred-init.md", "line": 55 }, { "feature_id": "F03.init-delivered-cors-origins", "description": "an origin delivered in `allow_cors_origins` must pass a CORS preflight after init, as the field promises (reproduces a real bug; see Gotchas).", "file": "F03-deferred-init.md", "line": 56 }, { "feature_id": "F03.init-second-call-400", "description": "a second `POST /api/init` after `ready` is rejected with 400 and the state is unchanged; the init key is now the delivered `secret_key`, so the dormant bootstrap key is 401.", "file": "F03-deferred-init.md", "line": 57 }, { "feature_id": "F03.init-open-without-secret", "description": "a pod booted with neither a secret key nor session keys accepts `POST /api/init` without `X-Init-API-Key`; afterwards the first delivered session key is the init key (a keyless retry is 401, a retry with it is 400).", "file": "F03-deferred-init.md", "line": 58 }, { "feature_id": "F03.init-state-not-persisted", "description": "the init state lives in memory only, so after a restart a deferred pod is dormant again (the previously delivered key gets the dormant 503 too) and must be re-initialized.", "file": "F03-deferred-init.md", "line": 59 }, { "feature_id": "F03.auth-before-gate", "description": "retired by #5700. It asserted that a dormant pod with boot session keys answered 401 to a missing key before the 503 gate; readiness is now checked first, see `F03.gate-before-auth`.", "file": "F03-deferred-init.md", "line": 60 }, { "feature_id": "F03.gate-before-auth", "description": "when the dormant config already has session keys, readiness is checked before authentication: a missing, wrong or valid key all get the 503 dormant gate (never 401), on HTTP and on WebSockets.", "file": "F03-deferred-init.md", "line": 61 }, { "feature_id": "F03.not-deferred-404", "description": "on a non-deferred server both `/api/init` routes return 404, and `POST` still authenticates the init key before the 404.", "file": "F03-deferred-init.md", "line": 62 }, { "feature_id": "F04.create-settings", "description": "a create from the saved settings (`agent_settings` plus `secrets_encrypted`, the Agent Canvas path) returns 201 `idle`, adds one to the count, and persists the agent only in `base_state.json`, never in `meta.json`.", "file": "F04-conversation-lifecycle.md", "line": 25 }, { "feature_id": "F04.create-inline-agent", "description": "an inline `agent` without `kind` is accepted as `Agent`; the API key is never echoed back and the response trims inline skills.", "file": "F04-conversation-lifecycle.md", "line": 26 }, { "feature_id": "F04.create-agent-profile", "description": "`agent_profile_id` resolves a stored Agent Profile and its LLM profile and records `launched_agent_profile`.", "file": "F04-conversation-lifecycle.md", "line": 27 }, { "feature_id": "F04.create-validation", "description": "a missing agent source, two agent sources, a missing workspace, a bad tag key, `max_iterations` 0 and invalid `agent_settings` are 422; an unknown `agent_profile_id` is 404; a profile with a dangling LLM reference is 422 `unresolved_profile_references`; no key is 401; nothing is created.", "file": "F04-conversation-lifecycle.md", "line": 28 }, { "feature_id": "F04.create-validation-redaction", "description": "a 422 caused by a wrongly typed field next to the LLM key (in an inline `agent.llm` and in `agent_settings.llm`) never contains the posted key; where the error echoes the enclosing object, the key reads ``.", "file": "F04-conversation-lifecycle.md", "line": 29 }, { "feature_id": "F04.create-idempotent", "description": "re-posting an existing client `conversation_id` returns 200 with the same conversation, creates nothing and does not resend the initial message.", "file": "F04-conversation-lifecycle.md", "line": 30 }, { "feature_id": "F04.create-initial-message-run-false", "description": "an `initial_message` with `\"run\": false` is recorded without starting a run (the same body with `\"run\": true` runs).", "file": "F04-conversation-lifecycle.md", "line": 31 }, { "feature_id": "F04.create-parent", "description": "`parent_conversation_id` links a child that the parent lists in `sub_conversation_ids`; an unknown parent, a different workspace and a self-parent are 422.", "file": "F04-conversation-lifecycle.md", "line": 32 }, { "feature_id": "F04.create-worktree", "description": "`worktree: true` inside a git repository moves the conversation into its own worktree on branch `openhands/`.", "file": "F04-conversation-lifecycle.md", "line": 33 }, { "feature_id": "F04.create-observability", "description": "the `X-OpenHands-Observability-*` headers are validated (422 for non-object metadata) and stored in `meta.json`.", "file": "F04-conversation-lifecycle.md", "line": 34 }, { "feature_id": "F04.create-observability-precedence", "description": "body `observability_span_name`, `observability_tags` and `observability_parent_span_context` win over their headers, header metadata merges under the body's (body keys win), the parent-span header applies when the body has none, and an over-long or malformed span-name header, non-scalar header metadata and an empty body tag are 422.", "file": "F04-conversation-lifecycle.md", "line": 35 }, { "feature_id": "F04.create-secrets", "description": "start-time `StaticSecret` values appear masked in the create response and are encrypted at rest.", "file": "F04-conversation-lifecycle.md", "line": 36 }, { "feature_id": "F04.create-secrets-get", "description": "a start-time secret is listed by `GET /{id}` right after the create.", "file": "F04-conversation-lifecycle.md", "line": 37 }, { "feature_id": "F04.create-secrets-plain", "description": "a plain-string secret value in the create body is accepted or rejected with 422, never a 500.", "file": "F04-conversation-lifecycle.md", "line": 38 }, { "feature_id": "F04.run-real", "description": "an initial message runs on DeepSeek flash to `finished`; the events socket replays the run, the final response is non-empty and autotitle names the conversation.", "file": "F04-conversation-lifecycle.md", "line": 39 }, { "feature_id": "F04.final-response", "description": "`agent_final_response` is empty before the agent answers, carries the exact `finish` message afterwards, and is 404 for an unknown id.", "file": "F04-conversation-lifecycle.md", "line": 40 }, { "feature_id": "F04.title-llm-profile", "description": "with `title_llm_profile` naming a saved LLM profile, autotitle calls that profile and not the agent's LLM; an unknown profile name falls back to the agent's LLM and never fails the create.", "file": "F04-conversation-lifecycle.md", "line": 41 }, { "feature_id": "F04.get", "description": "`GET /{id}` returns `runtime_info`, trims skills unless `include_skills=true`, accepts the 32-hex id form, and answers 404, 422 and 401.", "file": "F04-conversation-lifecycle.md", "line": 42 }, { "feature_id": "F04.batch-get", "description": "`GET ?ids=..` returns a list aligned with the ids, `null` for unknown ones, and 422 without or with malformed ids.", "file": "F04-conversation-lifecycle.md", "line": 43 }, { "feature_id": "F04.batch-get-limit", "description": "100 ids in one batch are refused with a client error, not a 500.", "file": "F04-conversation-lifecycle.md", "line": 44 }, { "feature_id": "F04.search-page", "description": "`limit` and `next_page_id` page newest first by default; `sort_order=CREATED_AT` reverses it; skills are trimmed unless `include_skills=true`; bad limits and sort orders are 422.", "file": "F04-conversation-lifecycle.md", "line": 45 }, { "feature_id": "F04.search-filter", "description": "`status` filters search and count consistently; an unknown status is 422.", "file": "F04-conversation-lifecycle.md", "line": 46 }, { "feature_id": "F04.count", "description": "`GET /count` equals the number of conversations search lists.", "file": "F04-conversation-lifecycle.md", "line": 47 }, { "feature_id": "F04.patch-title", "description": "PATCH `title` is stripped, stored in `meta.json`, bumps `updated_at` and moves the conversation to the top of `UPDATED_AT_DESC`.", "file": "F04-conversation-lifecycle.md", "line": 48 }, { "feature_id": "F04.patch-tags", "description": "PATCH `tags` replaces the whole map in `meta.json` and `base_state.json` and pushes a `tags` state update on the events socket.", "file": "F04-conversation-lifecycle.md", "line": 49 }, { "feature_id": "F04.patch-errors", "description": "an empty or 201-character title, a bad tag key and a malformed id are 422; no key is 401; the stored values are untouched.", "file": "F04-conversation-lifecycle.md", "line": 50 }, { "feature_id": "F04.patch-whitespace-title", "description": "a whitespace-only title is rejected instead of erasing the stored title.", "file": "F04-conversation-lifecycle.md", "line": 51 }, { "feature_id": "F04.patch-unknown-id", "description": "PATCH on an unknown id answers the 404 the OpenAPI document declares, not 200 `{\"success\": false}`.", "file": "F04-conversation-lifecycle.md", "line": 52 }, { "feature_id": "F04.delete", "description": "DELETE stops a running conversation (`PauseEvent`, `InterruptEvent`), removes its state, keeps the workspace, removes a worktree conversation's worktree but keeps its branch, orphans children, and later reads are 404 (socket close 4004); a malformed id is 422 and no key is 401.", "file": "F04-conversation-lifecycle.md", "line": 53 }, { "feature_id": "F04.delete-unknown-id", "description": "DELETE on an unknown or already deleted id answers the 404 the OpenAPI document declares, not 400.", "file": "F04-conversation-lifecycle.md", "line": 54 }, { "feature_id": "F04.restart-persist", "description": "after a restart the catalog, statuses, titles, tags, final responses and event replay are unchanged, and deleted conversations stay deleted.", "file": "F04-conversation-lifecycle.md", "line": 55 }, { "feature_id": "F04.read-no-hydrate", "description": "after a restart, GET, the batch get, search and count read a cold conversation without loading it (no `owner_lease.json`), while `agent_final_response` loads it and claims the lease.", "file": "F04-conversation-lifecycle.md", "line": 56 }, { "feature_id": "F05.run-queued", "description": "a message appended with `run: false` leaves the conversation `idle`; `POST /run` returns 200 at once and the agent runs on DeepSeek flash through `running` to `finished`, both transitions pushed on the events socket.", "file": "F05-run-pause-interrupt.md", "line": 26 }, { "feature_id": "F05.run-finished-noop", "description": "`POST /run` on a `finished` conversation without new input returns 200 and the run ends at once: no `execution_status` change on the socket, still `finished`, no new events (no model call).", "file": "F05-run-pause-interrupt.md", "line": 27 }, { "feature_id": "F05.run-new-input", "description": "a new user message moves a `finished` conversation back to `idle`, and `POST /run` runs it to `finished` again.", "file": "F05-run-pause-interrupt.md", "line": 28 }, { "feature_id": "F05.pause-idle", "description": "pausing an idle conversation returns 200, reports `paused` on GET, the status search and count and the events socket, and persists one `PauseEvent`.", "file": "F05-run-pause-interrupt.md", "line": 29 }, { "feature_id": "F05.pause-repeat", "description": "pausing a `paused` conversation again returns 200 and adds no second `PauseEvent`.", "file": "F05-run-pause-interrupt.md", "line": 30 }, { "feature_id": "F05.run-resume-paused", "description": "`POST /run` resumes a `paused` conversation through `running` to `finished`.", "file": "F05-run-pause-interrupt.md", "line": 31 }, { "feature_id": "F05.interrupt-idle", "description": "interrupt with nothing running falls back to pause: `paused`, one `PauseEvent`, no `InterruptEvent`.", "file": "F05-run-pause-interrupt.md", "line": 32 }, { "feature_id": "F05.run-error", "description": "a run whose model provider is unreachable ends in `error` with a `ConversationErrorEvent` (also pushed on the events socket) and is listed by `status=error`.", "file": "F05-run-pause-interrupt.md", "line": 33 }, { "feature_id": "F05.pause-terminal", "description": "pause and interrupt on a `finished` or `error` conversation return 200 and change neither the status nor the event log.", "file": "F05-run-pause-interrupt.md", "line": 34 }, { "feature_id": "F05.run-error-responsive", "description": "a run that fails does not stall the server: other requests keep answering while the error is handled.", "file": "F05-run-pause-interrupt.md", "line": 35 }, { "feature_id": "F05.unknown-id-run", "description": "`POST /run` on an unknown id returns 404.", "file": "F05-run-pause-interrupt.md", "line": 36 }, { "feature_id": "F05.unknown-id-pause", "description": "`POST /pause` and `POST /interrupt` on an unknown id return the documented 404.", "file": "F05-run-pause-interrupt.md", "line": 37 }, { "feature_id": "F05.control-errors", "description": "all three routes answer 401 without or with a wrong session key and 422 for a malformed id, and the idle conversation they targeted is untouched (still `idle`, no `PauseEvent`).", "file": "F05-run-pause-interrupt.md", "line": 38 }, { "feature_id": "F05.run-conflict", "description": "`POST /run` while a run is active returns 409 `Conversation already running...`, and the active run keeps going without a second model call.", "file": "F05-run-pause-interrupt.md", "line": 39 }, { "feature_id": "F05.pause-running", "description": "pause during a long model call returns 200 at once; the events socket, status search and event log say `paused`, and `/run` stays 409 while the step is still in flight.", "file": "F05-run-pause-interrupt.md", "line": 40 }, { "feature_id": "F05.pause-running-get", "description": "after a pause during a long model call, `GET /{id}` and `base_state.json` report `paused` like every other view.", "file": "F05-run-pause-interrupt.md", "line": 41 }, { "feature_id": "F05.pause-running-release", "description": "interrupting a paused conversation whose step is still in flight cancels it (`InterruptEvent`), GET catches up to `paused`, and the run slot is free again (a create that needs the slot succeeds).", "file": "F05-run-pause-interrupt.md", "line": 42 }, { "feature_id": "F05.interrupt-running", "description": "interrupt cancels a hanging model call well within the server's 5 s wait: 200, `paused`, an `InterruptEvent` and no `PauseEvent`, pushed on the events socket.", "file": "F05-run-pause-interrupt.md", "line": 43 }, { "feature_id": "F05.run-after-interrupt", "description": "after an interrupt, `POST /run` is accepted at once and the resumed run reaches `finished`.", "file": "F05-run-pause-interrupt.md", "line": 44 }, { "feature_id": "F05.pause-tool-step", "description": "pause while a tool call runs lets that step finish (its observation is recorded), then stops the loop before the next model call: `paused`, one `PauseEvent`, one model request.", "file": "F05-run-pause-interrupt.md", "line": 45 }, { "feature_id": "F05.interrupt-tool-call", "description": "interrupt while a tool call runs returns at once: `paused`, an `InterruptEvent`, an `AgentErrorEvent` for the unfinished call (same `tool_call_id`) and no observation; the resumed run sends that error to the model as the tool result and finishes.", "file": "F05-run-pause-interrupt.md", "line": 46 }, { "feature_id": "F05.run-capacity", "description": "with `max_concurrent_runs` 1, `POST /run` on a second conversation returns 429 while the first holds the slot, leaves it `idle`, and succeeds once the first is interrupted.", "file": "F05-run-pause-interrupt.md", "line": 47 }, { "feature_id": "F05.stuck-detected", "description": "with `stuck_detection` on (the default), a run whose model repeats the same tool call ends `stuck` after four identical action/observation pairs (four model calls), with no `ConversationErrorEvent`; GET, `status=stuck` search and count and the events socket say `stuck`.", "file": "F05-run-pause-interrupt.md", "line": 48 }, { "feature_id": "F05.stuck-rerun", "description": "`POST /run` on a `stuck` conversation with no new input returns 200, goes `running` and straight back to `stuck` without a model call.", "file": "F05-run-pause-interrupt.md", "line": 49 }, { "feature_id": "F05.stuck-new-input", "description": "a new user message moves a `stuck` conversation to `idle`, and `POST /run` runs it again to `finished`.", "file": "F05-run-pause-interrupt.md", "line": 50 }, { "feature_id": "F05.max-iterations-stop", "description": "a conversation created with `max_iterations: 6` and `stuck_detection: false` whose model never finishes makes exactly six model calls (past the stuck threshold) and ends `error` with exactly one `ConversationErrorEvent` `code: MaxIterationsReached` naming the limit, on REST and the events socket, never `stuck`.", "file": "F05-run-pause-interrupt.md", "line": 51 }, { "feature_id": "F05.max-iterations-resume", "description": "`POST /run` after `MaxIterationsReached` is accepted and gets a fresh budget of six model calls (a second `MaxIterationsReached`), and once the model calls `finish` the next run ends `finished`.", "file": "F05-run-pause-interrupt.md", "line": 52 }, { "feature_id": "F05.restart-running-graceful", "description": "a conversation running at a graceful restart comes back `paused`, with a `PauseEvent` and an `InterruptEvent`.", "file": "F05-run-pause-interrupt.md", "line": 53 }, { "feature_id": "F05.restart-keeps-status", "description": "`finished`, `paused` and `error` statuses survive a restart unchanged.", "file": "F05-run-pause-interrupt.md", "line": 54 }, { "feature_id": "F05.restart-hard-running", "description": "a conversation running when the server is SIGKILLed comes back `error` on the next start, and `/run` resumes it.", "file": "F05-run-pause-interrupt.md", "line": 55 }, { "feature_id": "F05.pause-running-crash", "description": "a pause accepted during a model call survives a SIGKILL: the conversation comes back `paused`, as its persisted `PauseEvent` and status update say, not `error`.", "file": "F05-run-pause-interrupt.md", "line": 56 }, { "feature_id": "F06.send-no-run", "description": "`POST .../events` with `run: false` persists a user `MessageEvent` (preceded by the `SystemPromptEvent` on the first message), returns `{\"success\": true}` and leaves the conversation `idle`.", "file": "F06-conversation-events.md", "line": 29 }, { "feature_id": "F06.search-page", "description": "search returns `{items, next_page_id}` in log order; `limit` plus the inclusive `page_id` cursor pages through it, `TIMESTAMP_DESC` reverses it, and an unknown `page_id` restarts from the first event.", "file": "F06-conversation-events.md", "line": 30 }, { "feature_id": "F06.search-validation", "description": "search rejects `limit` 0 or 101, an unknown `sort_order` and an unparsable timestamp with 422.", "file": "F06-conversation-events.md", "line": 31 }, { "feature_id": "F06.search-filters", "description": "`source`, `body` (case-insensitive, message text only) and fully-qualified `kind` filters narrow search results and combine.", "file": "F06-conversation-events.md", "line": 32 }, { "feature_id": "F06.search-kind-short", "description": "`kind=MessageEvent`, the short name the API docs and the TypeScript client use, filters by the wire `kind`.", "file": "F06-conversation-events.md", "line": 33 }, { "feature_id": "F06.search-time-window", "description": "`timestamp__gte` is inclusive, `timestamp__lt` exclusive, and a value with a UTC offset is converted to server time.", "file": "F06-conversation-events.md", "line": 34 }, { "feature_id": "F06.count", "description": "count returns a bare integer equal to the number of search results for the same filters.", "file": "F06-conversation-events.md", "line": 35 }, { "feature_id": "F06.get-one", "description": "`GET .../events/{event_id}` returns the event that search lists under that id, and the same event is on disk.", "file": "F06-conversation-events.md", "line": 36 }, { "feature_id": "F06.unknown-event", "description": "an unknown event id returns 404 from get-one and `null` in its slot from batch-get.", "file": "F06-conversation-events.md", "line": 37 }, { "feature_id": "F06.batch-get", "description": "`GET .../events` with a JSON array of ids as the body returns those events in request order; no body is a 422.", "file": "F06-conversation-events.md", "line": 38 }, { "feature_id": "F06.not-found-auth", "description": "every event route returns 404 for an unknown conversation, 422 for a malformed conversation id and 401 without a valid session key.", "file": "F06-conversation-events.md", "line": 39 }, { "feature_id": "F06.send-validation", "description": "a missing body, a message with an unknown role, string content or an unknown content type is rejected with 422 and nothing is appended.", "file": "F06-conversation-events.md", "line": 40 }, { "feature_id": "F06.send-image", "description": "a message with a text part and an `image` part (a `data:` URL) is persisted with its `image_urls` intact and returned the same by search, get-one and the socket replay; an `image` part without `image_urls` is a 422 and appends nothing.", "file": "F06-conversation-events.md", "line": 41 }, { "feature_id": "F06.send-non-user-role", "description": "a message with a schema-valid non-user role (`assistant`) is rejected as a client error (4xx), not a 500, and leaves no event behind (not even the system prompt of an empty log).", "file": "F06-conversation-events.md", "line": 42 }, { "feature_id": "F06.send-run", "description": "`POST .../events` with `run: true` drives a real agent turn to `finished`, persisting the kinds of a normal run (user message, `execution_status` running and finished, `ActionEvent`/`ObservationEvent` pairs, `stats`).", "file": "F06-conversation-events.md", "line": 43 }, { "feature_id": "F06.ws-live", "description": "a socket opened before the run receives each event live, every live frame is persisted, and a closing `full_state` snapshot follows the run.", "file": "F06-conversation-events.md", "line": 44 }, { "feature_id": "F06.search-timestamp-order", "description": "after a tool-using run, `sort_order=TIMESTAMP` lists events in timestamp order, so a `timestamp__gte` or `resend_mode=since` cut at a listed event keeps every event listed after it.", "file": "F06-conversation-events.md", "line": 45 }, { "feature_id": "F06.send-resets-finished", "description": "a message appended to a `finished` conversation sets it back to `idle` with a persisted `execution_status` update.", "file": "F06-conversation-events.md", "line": 46 }, { "feature_id": "F06.ws-snapshot", "description": "the first frame after authentication is a transient `full_state` `ConversationStateUpdateEvent` that never appears in search; without `resend_mode` nothing else is replayed.", "file": "F06-conversation-events.md", "line": 47 }, { "feature_id": "F06.ws-handshake", "description": "the socket accepts first-frame, deprecated query and deprecated header keys; a wrong first-frame key, a first frame that is not an auth frame, or no auth frame within 10 s closes with 4001, a wrong deprecated query or header key is refused at the upgrade (HTTP 403), and an unknown conversation closes with 4004.", "file": "F06-conversation-events.md", "line": 48 }, { "feature_id": "F06.ws-delete-close", "description": "deleting a conversation closes the sockets that follow it.", "file": "F06-conversation-events.md", "line": 49 }, { "feature_id": "F06.ws-replay-all", "description": "`resend_mode=all` (and the deprecated `resend_all=true`) replays the persisted log in search order after the snapshot.", "file": "F06-conversation-events.md", "line": 50 }, { "feature_id": "F06.ws-replay-since", "description": "`resend_mode=since` with `after_timestamp` replays exactly the events at or after that timestamp; without `after_timestamp` it replays nothing.", "file": "F06-conversation-events.md", "line": 51 }, { "feature_id": "F06.ws-bad-frame", "description": "an invalid inbound frame returns a `ServerErrorEvent` (`ValidationError`, `AssertionError`, `JSONDecodeError` for text that is not JSON), appends nothing and leaves the socket open; a redundant auth frame is ignored.", "file": "F06-conversation-events.md", "line": 52 }, { "feature_id": "F06.ws-send", "description": "a user message frame sent on the socket is persisted and runs the agent to `finished`, streaming the run on the same socket.", "file": "F06-conversation-events.md", "line": 53 }, { "feature_id": "F06.restart-persist", "description": "event ids, order and count survive a server restart, the socket replays the same log, and new messages append after it.", "file": "F06-conversation-events.md", "line": 54 }, { "feature_id": "F06.send-run-limit", "description": "with every run slot taken, `POST .../events` with `run: true` returns 429 `Message saved...` and the message is persisted; a socket send gets a `ConversationRunLimitExceeded` `ServerErrorEvent`.", "file": "F06-conversation-events.md", "line": 55 }, { "feature_id": "F06.send-while-running", "description": "`POST .../events` with `run: true` on a conversation that is already running returns 200 (not the 409 of `POST /run`), persists the message, keeps the conversation `running` and starts no second run.", "file": "F06-conversation-events.md", "line": 56 }, { "feature_id": "F07.sync-first", "description": "the first server frame is `{\"type\": \"sync\"}`; `from_seq` echoes `after_seq` (absent when live-only) and `through_seq` is the last on-disk `seq` (absent for an empty log).", "file": "F07-session-socket.md", "line": 26 }, { "feature_id": "F07.snapshot-transient", "description": "every connection gets one `transient` frame with a `full_state` `ConversationStateUpdateEvent` and no `seq`, after `sync` (and after the replay when one ran); it is never persisted.", "file": "F07-session-socket.md", "line": 27 }, { "feature_id": "F07.full-replay", "description": "`after_seq=-1` (or any value below it) replays every persisted event as `durable` frames with `seq` 0..`through_seq`, in order, once each, with the same ids REST returns.", "file": "F07-session-socket.md", "line": 28 }, { "feature_id": "F07.seq-on-disk", "description": "a durable frame's `seq` is the event's on-disk index, the `NNNNN` in `events/event-NNNNN-.json`.", "file": "F07-session-socket.md", "line": 29 }, { "feature_id": "F07.resume-cursor", "description": "`after_seq=k` replays exactly the events with `seq > k`; `after_seq=through_seq` replays nothing.", "file": "F07-session-socket.md", "line": 30 }, { "feature_id": "F07.live-durable", "description": "on a live-only connection, an event appended later (a REST message with `run: false`) arrives as a `durable` frame with the next `seq`.", "file": "F07-session-socket.md", "line": 31 }, { "feature_id": "F07.live-order", "description": "live `durable` frames on one connection arrive in ascending `seq` order.", "file": "F07-session-socket.md", "line": 32 }, { "feature_id": "F07.replay-then-live", "description": "a connection opened with `after_seq=-1` while events are being appended receives every persisted event exactly once across the replay and the live tail: its durable `seq` values are exactly 0..N-1 for the N events REST counts.", "file": "F07-session-socket.md", "line": 33 }, { "feature_id": "F07.send-message", "description": "a `Message` JSON sent on the socket is persisted as a user `MessageEvent` (echoed as a `durable` frame) and runs the agent to `finished`.", "file": "F07-session-socket.md", "line": 34 }, { "feature_id": "F07.error-frame", "description": "JSON that is not a valid `Message` gets an `error` frame (`code` `ValidationError`); the connection stays open and nothing is persisted.", "file": "F07-session-socket.md", "line": 35 }, { "feature_id": "F07.error-non-json", "description": "a text frame that is not JSON gets an `error` frame with `code` `JSONDecodeError`, and a binary frame gets an `error` frame too; the connection stays open.", "file": "F07-session-socket.md", "line": 36 }, { "feature_id": "F07.reject-non-user", "description": "a `Message` whose `role` is not `user` gets an `error` frame (`Only user messages are allowed to be sent to the agent.`) and is not persisted.", "file": "F07-session-socket.md", "line": 37 }, { "feature_id": "F07.auth-frame-ignored", "description": "a redundant `{\"type\": \"auth\", ...}` frame after authentication is ignored: no `error` frame, nothing persisted.", "file": "F07-session-socket.md", "line": 38 }, { "feature_id": "F07.auth", "description": "a first-frame key opens the socket; a wrong key or a message sent before authenticating closes it with 4001 (nothing persisted); the run's key in the deprecated query parameter opens it, and a wrong one fails the handshake with HTTP 403.", "file": "F07-session-socket.md", "line": 39 }, { "feature_id": "F07.unknown-conversation", "description": "a well-formed id that names no conversation closes with 4004 `Conversation not found` after authentication.", "file": "F07-session-socket.md", "line": 40 }, { "feature_id": "F07.bad-handshake", "description": "a conversation id that is not a UUID, or an `after_seq` that is not an integer, fails the handshake with HTTP 403.", "file": "F07-session-socket.md", "line": 41 }, { "feature_id": "F07.restart-resume", "description": "`seq` survives a server restart: reconnecting with the same cursor replays the same event ids at the same `seq` values.", "file": "F07-session-socket.md", "line": 42 }, { "feature_id": "F07.progress-frames", "description": "with `llm.stream` on, a run streams `item_started`, then `delta` frames (`order` 0, 1, ...), and each item is retired by exactly one `durable` frame whose `event.id` equals `item_id` (or by `item_aborted`).", "file": "F07-session-socket.md", "line": 43 }, { "feature_id": "F07.progress-not-replayed", "description": "reconnecting with `after_seq=-1` after a streamed run replays the durable message but no `item_started`, `delta` or `item_aborted` frame.", "file": "F07-session-socket.md", "line": 44 }, { "feature_id": "F07.item-aborted", "description": "interrupting a run while its streamed answer is producing `delta` frames retires the item with exactly one `item_aborted` frame (same `item_id` and `attempt`, `reason` `cancelled`) and no further progress for it; no `durable` frame or persisted event carries that id, and a reconnect with `after_seq=-1` replays no progress frame.", "file": "F07-session-socket.md", "line": 45 }, { "feature_id": "F07.frame-too-large", "description": "replaying an event over 4 MiB closes the socket with 1013 `frame_too_large`; the event stays readable over REST and a cursor past it replays the rest.", "file": "F07-session-socket.md", "line": 46 }, { "feature_id": "F07.slow-consumer", "description": "a client that stops reading while more than the 16 MiB write budget queues up is closed with 1013 `slow_consumer`, and the conversation keeps serving other sockets.", "file": "F07-session-socket.md", "line": 47 }, { "feature_id": "F07.subscriber-limit", "description": "the event fan-out admits at most 50 subscribers per conversation; extra sockets close with 1013 `Too many connections for this conversation`, and closing sockets frees their slots.", "file": "F07-session-socket.md", "line": 48 }, { "feature_id": "F07.deleted-close", "description": "deleting a conversation closes the session sockets attached to it.", "file": "F07-session-socket.md", "line": 49 }, { "feature_id": "F08.policy-set", "description": "`POST .../confirmation_policy` switches between `AlwaysConfirm`, `ConfirmRisky` (with `threshold` and `confirm_unknown`) and `NeverConfirm`; GET, `base_state.json` and a `confirmation_policy` state update on the events socket reflect each change at once.", "file": "F08-confirmation-security-secrets.md", "line": 27 }, { "feature_id": "F08.policy-errors", "description": "an unknown policy kind, a `threshold` of `UNKNOWN` or an unknown level, and a missing or empty `policy` are 422; an unknown conversation is 404; no key is 401; the stored policy is unchanged.", "file": "F08-confirmation-security-secrets.md", "line": 28 }, { "feature_id": "F08.always-confirm-wait", "description": "after a runtime switch to `AlwaysConfirm`, the agent's next terminal action parks the conversation in `waiting_for_confirmation` (pushed on the socket) with the `ActionEvent` recorded, no observation, and the command not run.", "file": "F08-confirmation-security-secrets.md", "line": 29 }, { "feature_id": "F08.confirm-accept", "description": "`accept: true` runs the pending command (observation and file on disk) and the run continues to `finished`.", "file": "F08-confirmation-security-secrets.md", "line": 30 }, { "feature_id": "F08.confirm-reject", "description": "`accept: false` records one `UserRejectObservation` with the given reason (REST and socket) and returns to `idle` without running the command; a second reject is a no-op, and a later accept only resumes the agent.", "file": "F08-confirmation-security-secrets.md", "line": 31 }, { "feature_id": "F08.confirm-errors", "description": "a missing or non-boolean `accept` and a malformed id are 422; an unknown conversation is 404; no key is 401.", "file": "F08-confirmation-security-secrets.md", "line": 32 }, { "feature_id": "F08.confirm-capacity", "description": "with the server's run capacity full, accept is 429 and leaves the action pending and unrun; reject still works; accept on a running conversation is a 200 no-op.", "file": "F08-confirmation-security-secrets.md", "line": 33 }, { "feature_id": "F08.confirm-real", "description": "DeepSeek flash under `AlwaysConfirm` stops before its terminal command and finishes the task once its actions are accepted.", "file": "F08-confirmation-security-secrets.md", "line": 34 }, { "feature_id": "F08.analyzer-set-clear", "description": "`POST .../security_analyzer` sets `PatternSecurityAnalyzer` or `LLMSecurityAnalyzer` and clears it with `null`; GET, `base_state.json` and the socket reflect each change.", "file": "F08-confirmation-security-secrets.md", "line": 35 }, { "feature_id": "F08.analyzer-errors", "description": "a missing `security_analyzer` field and an unknown kind are 422; an unknown conversation is 404; no key is 401; the stored analyzer is unchanged.", "file": "F08-confirmation-security-secrets.md", "line": 36 }, { "feature_id": "F08.confirm-risky", "description": "under `ConfirmRisky` with `PatternSecurityAnalyzer`, a low-risk command runs unasked and `rm -rf` waits for confirmation; once the analyzer is cleared, an unrated command waits too.", "file": "F08-confirmation-security-secrets.md", "line": 37 }, { "feature_id": "F08.secrets-update", "description": "`POST .../secrets` accepts plain strings, `{\"value\": ...}` and `StaticSecret` objects (200); once the state is saved, GET lists the names with masked values and `base_state.json` holds them encrypted, never in plaintext.", "file": "F08-confirmation-security-secrets.md", "line": 38 }, { "feature_id": "F08.secrets-visible", "description": "a secret posted at runtime is listed by GET right away, without waiting for another state change.", "file": "F08-confirmation-security-secrets.md", "line": 39 }, { "feature_id": "F08.secrets-errors", "description": "a missing or non-object `secrets`, an unknown secret kind and a `LookupSecret` without `url` are 422; an unknown conversation is 404; no key is 401; nothing is stored.", "file": "F08-confirmation-security-secrets.md", "line": 40 }, { "feature_id": "F08.secrets-bad-value", "description": "a number, `null` or a boolean as a secret value is refused with 422, not a 500.", "file": "F08-confirmation-security-secrets.md", "line": 41 }, { "feature_id": "F08.secrets-terminal", "description": "a terminal command that names a runtime secret receives its value, while the observation and the next request to the model show `` and the plaintext is stored nowhere.", "file": "F08-confirmation-security-secrets.md", "line": 42 }, { "feature_id": "F08.secrets-override", "description": "a runtime secret replaces a start-time secret of the same name for the next command.", "file": "F08-confirmation-security-secrets.md", "line": 43 }, { "feature_id": "F08.secrets-lookup", "description": "a runtime `LookupSecret` is fetched from its URL with its headers only when a command names it; the command gets the fetched text, the observation and the model see ``, and GET hides the `Authorization` header that `base_state.json` keeps encrypted.", "file": "F08-confirmation-security-secrets.md", "line": 44 }, { "feature_id": "F08.secrets-lookup-self", "description": "a runtime `LookupSecret` whose URL is the server's own `/api/settings/secrets/` (hostless, made absolute from `OH_INTERNAL_SERVER_URL`, or absolute on the run's address) is answered in process from the secrets store with no HTTP request and no session key; a name the store lacks falls through to HTTP (401 without a key) and the command gets no value.", "file": "F08-confirmation-security-secrets.md", "line": 45 }, { "feature_id": "F08.secrets-profile-scope", "description": "for a conversation launched from an Agent Profile with `secret_refs`, names outside the list are silently dropped (200).", "file": "F08-confirmation-security-secrets.md", "line": 46 }, { "feature_id": "F08.secrets-no-cipher", "description": "on a server with no cipher (no `OH_SECRET_KEY`, no session key) a runtime secret reaches commands while the conversation lives, is persisted only as the redaction placeholder (the server warns at startup), and is empty after a restart although GET still lists the name.", "file": "F08-confirmation-security-secrets.md", "line": 47 }, { "feature_id": "F08.secrets-restart", "description": "a runtime secret that reached `base_state.json` is still injected into commands after a server restart.", "file": "F08-confirmation-security-secrets.md", "line": 48 }, { "feature_id": "F08.secrets-unsaved-restart", "description": "a runtime secret survives a restart even when nothing else changed in the conversation after it was posted.", "file": "F08-confirmation-security-secrets.md", "line": 49 }, { "feature_id": "F09.auth-required", "description": "all four routes answer 401 without a valid session key and change nothing.", "file": "F09-model-switching-plugins.md", "line": 26 }, { "feature_id": "F09.switch-llm", "description": "`switch_llm` with a fresh `usage_id` answers 200 `{\"success\": true}`; `GET` and `base_state.json` show the new model and `usage_id`, the condenser LLM that mirrored the old one follows, and the key is never echoed or stored in plaintext.", "file": "F09-model-switching-plugins.md", "line": 27 }, { "feature_id": "F09.switch-llm-ws", "description": "a switch pushes a `ConversationStateUpdateEvent` with key `agent` and the new model on the events socket, with the key masked.", "file": "F09-model-switching-plugins.md", "line": 28 }, { "feature_id": "F09.switch-llm-errors", "description": "an unknown conversation is 404, a body without `llm` or without `llm.model` is 422, and a dangling `provider_connection_id` is 422; the installed LLM is unchanged.", "file": "F09-model-switching-plugins.md", "line": 29 }, { "feature_id": "F09.switch-llm-encrypted-key", "description": "mid-conversation, a switch whose `api_key` is the `gAAAAA` token from an `encrypted` profile read is decrypted before install; the next turn runs on DeepSeek and its usage is recorded under the new `usage_id`.", "file": "F09-model-switching-plugins.md", "line": 30 }, { "feature_id": "F09.switch-llm-provider-connection", "description": "a switch that references a saved provider connection instead of a key runs the next turn with the connection's key; `GET` shows the `provider_connection_id` and no key.", "file": "F09-model-switching-plugins.md", "line": 31 }, { "feature_id": "F09.switch-llm-registered-usage-id", "description": "after the agent initialized, a switch whose `usage_id` is already registered (the `default` a saved profile's config carries) either installs the posted LLM or is refused, never a silent 200 no-op.", "file": "F09-model-switching-plugins.md", "line": 32 }, { "feature_id": "F09.switch-llm-before-first-run", "description": "a switch before the conversation's first run is used by that run, and the run's token usage is recorded under the switched LLM's `usage_id`.", "file": "F09-model-switching-plugins.md", "line": 33 }, { "feature_id": "F09.switch-llm-mid-run", "description": "a `switch_llm` while a turn is running (the agent is inside a 20 s tool call) answers 200, the socket pushes the new `agent`, and the same run's next LLM call already uses the switched LLM (usage under both `usage_id`s) and the run finishes.", "file": "F09-model-switching-plugins.md", "line": 34 }, { "feature_id": "F09.switch-llm-mid-run-get", "description": "right after a mid-run switch answers 200, `GET` and `base_state.json` show the switched LLM, as the socket already does.", "file": "F09-model-switching-plugins.md", "line": 35 }, { "feature_id": "F09.switch-profile", "description": "`switch_profile` to the saved `deepseek-pro` profile answers 200, installs it under `usage_id` `profile:deepseek-pro`, and the next turn runs on `deepseek/deepseek-v4-pro` with its usage recorded under that id.", "file": "F09-model-switching-plugins.md", "line": 36 }, { "feature_id": "F09.switch-profile-errors", "description": "an unknown profile is 404 `Profile '' not found`, an invalid name and a corrupted profile file are 400, a missing `profile_name` is 422 and an unknown conversation 404; the installed LLM is unchanged.", "file": "F09-model-switching-plugins.md", "line": 37 }, { "feature_id": "F09.switch-profile-edited", "description": "switching to a profile again after it was edited installs the edited config, not the copy cached at the first switch.", "file": "F09-model-switching-plugins.md", "line": 38 }, { "feature_id": "F09.acp-model-native", "description": "`switch_acp_model` on a regular (non-ACP) conversation is 400 `switch_acp_model is only supported for ACP conversations.`; a missing `model` is 422 and an unknown conversation 404.", "file": "F09-model-switching-plugins.md", "line": 39 }, { "feature_id": "F09.acp-model-deferred", "description": "on an ACP conversation that has not run yet, `switch_acp_model` answers 200 without starting the subprocess and persists the model as `agent.acp_model`, `agent_state.acp_current_model_id` and `current_model_id`.", "file": "F09-model-switching-plugins.md", "line": 40 }, { "feature_id": "F09.acp-model-blank", "description": "a blank or whitespace-only ACP model is refused before the first run, as it is on a live session, and the stored model is kept.", "file": "F09-model-switching-plugins.md", "line": 41 }, { "feature_id": "F09.switch-llm-acp-refused", "description": "`switch_llm` and `switch_profile` on an ACP conversation are refused, as `switch_acp_model` is on a regular one, and leave the ACP agent's placeholder LLM (`usage_id` `acp-managed`) alone.", "file": "F09-model-switching-plugins.md", "line": 42 }, { "feature_id": "F09.acp-model-live", "description": "on an ACP conversation with a live session, `switch_acp_model` switches the subprocess's model in place (same ACP session) and `current_model_id`, `acp_model` and `base_state.json` follow; a blank model is 400, a model the provider does not offer is refused and changes nothing, and the next turn runs on the switched model in the same session.", "file": "F09-model-switching-plugins.md", "line": 43 }, { "feature_id": "F09.acp-model-live-unknown", "description": "a live switch to a model id the provider does not offer (not in `available_models`) is a client error (400, as for a blank model), not a 500.", "file": "F09-model-switching-plugins.md", "line": 44 }, { "feature_id": "F09.acp-model-live-timeout", "description": "a live switch whose `session/set_model` round-trip outlasts `acp_prompt_timeout` is 504 with `detail` `Internal Server Error` and leaves the model unchanged.", "file": "F09-model-switching-plugins.md", "line": 45 }, { "feature_id": "F09.ts-client", "description": "the TypeScript client's `ConversationClient.switchLLM`, `switchProfile` and `switchAcpModel` and `ConversationManager.switchProfile` reach the routes: each switch resolves and reads back through `getConversation`, an unknown profile rejects with 404, `switchAcpModel` on a regular conversation with 400, a call without a key with 401, and `switchAcpModel` on an ACP conversation that has not run resolves and stores the model.", "file": "F09-model-switching-plugins.md", "line": 46 }, { "feature_id": "F09.ts-client-acp-docs", "description": "the TypeScript client's `switchAcpModel` docstrings describe what the server does before an ACP conversation's first run (200, the model is stored for the first session), not a 409.", "file": "F09-model-switching-plugins.md", "line": 47 }, { "feature_id": "F09.load-plugin", "description": "`load_plugin` with `@` from a local marketplace registered on the agent's `agent_context` answers 200 and merges the plugin's skill, its command and its hooks into the live agent, visible in `GET`, `base_state.json` and on the socket.", "file": "F09-model-switching-plugins.md", "line": 48 }, { "feature_id": "F09.load-plugin-hook-runs", "description": "after a runtime load, the next user turn runs the plugin's `UserPromptSubmit` hook (a `HookExecutionEvent` with its output) and finishes.", "file": "F09-model-switching-plugins.md", "line": 49 }, { "feature_id": "F09.load-plugin-errors", "description": "no registered marketplaces and no agent context are 400, a registered marketplace whose source is missing is 400, an unknown marketplace or plugin is 404, empty and malformed refs are 400, a missing `plugin_ref` is 422 and an unknown conversation 404; a bare plugin name resolves across the registered marketplaces.", "file": "F09-model-switching-plugins.md", "line": 50 }, { "feature_id": "F09.load-plugin-twice", "description": "loading the same plugin twice leaves one copy of its hooks, so they still run once per message.", "file": "F09-model-switching-plugins.md", "line": 51 }, { "feature_id": "F09.load-plugin-sdk", "description": "the Python SDK's `RemoteConversation.load_plugin(ref)`, on a conversation reattached with `RemoteConversation.attach`, raises `HTTPStatusError` 404 for an unknown plugin and then loads the plugin, whose skill and hook `GET` shows.", "file": "F09-model-switching-plugins.md", "line": 52 }, { "feature_id": "F09.restart-persist", "description": "after a restart the switched LLM, the switched profile (which the next turn runs on), the deferred ACP model and the loaded plugin's skills and hooks are still in place.", "file": "F09-model-switching-plugins.md", "line": 53 }, { "feature_id": "F10.fork-whole", "description": "`POST /fork` with no body answers 201 with a new `idle` conversation that names its source in `forked_from_conversation_id`, keeps the source's HEAD and working directory, and holds a copy of every event (same ids) in its own persistence directory; the source is unchanged.", "file": "F10-fork-navigate-condense-ask.md", "line": 25 }, { "feature_id": "F10.fork-named", "description": "a fork request's `id`, `title` and `tags` become the fork's id, title and tags (plus a `title` tag), on the response, on GET and in `meta.json`.", "file": "F10-fork-navigate-condense-ask.md", "line": 26 }, { "feature_id": "F10.fork-tags-consistent", "description": "when a fork request gives no tags, the fork's tags on GET and the tags stored in its `meta.json` are one consistent set (both inherit the source's tags, or neither does).", "file": "F10-fork-navigate-condense-ask.md", "line": 27 }, { "feature_id": "F10.fork-from-event", "description": "`from_event_id` copies only the branch up to and including that event, sets the fork's HEAD and `forked_from_event_id` to it, and leaves later events out.", "file": "F10-fork-navigate-condense-ask.md", "line": 28 }, { "feature_id": "F10.fork-metrics", "description": "by default the fork's `stats` start empty; `reset_metrics: false` copies the source's accumulated cost and token usage.", "file": "F10-fork-navigate-condense-ask.md", "line": 29 }, { "feature_id": "F10.fork-independent", "description": "messages appended to the fork never reach the source and vice versa; the fork's events socket replays the copied history plus its own events.", "file": "F10-fork-navigate-condense-ask.md", "line": 30 }, { "feature_id": "F10.fork-errors", "description": "an unknown source is 404, an unknown `from_event_id` is 404, a taken `id` is 409, a bad tag key or a title over 200 characters is 422, a malformed id is 422 and a missing key is 401; none of them creates a conversation.", "file": "F10-fork-navigate-condense-ask.md", "line": 31 }, { "feature_id": "F10.fork-keeps-credentials", "description": "the fork's `base_state.json` keeps the agent's (encrypted) model keys, like the source's.", "file": "F10-fork-navigate-condense-ask.md", "line": 32 }, { "feature_id": "F10.fork-run-memory", "description": "a fork of a finished DeepSeek conversation runs to `finished` with the source's history in context (it recalls a codename only the source was told).", "file": "F10-fork-navigate-condense-ask.md", "line": 33 }, { "feature_id": "F10.navigate-event", "description": "`POST /navigate` with an earlier `event_id` answers 200 with `leaf_event_id` set to it; GET and `base_state.json` agree and no event is deleted.", "file": "F10-fork-navigate-condense-ask.md", "line": 34 }, { "feature_id": "F10.navigate-branch", "description": "after navigating, the next appended event's `parent_id` is the new HEAD (a sibling of the abandoned branch), pushed live on the events socket; the HEAD move itself is not broadcast.", "file": "F10-fork-navigate-condense-ask.md", "line": 35 }, { "feature_id": "F10.navigate-empty", "description": "`{}` or `{\"event_id\": null}` selects the empty tree (`leaf_event_id` null, `head_is_empty` true on disk); the next appended event becomes a new root (`parent_id` `__root__`).", "file": "F10-fork-navigate-condense-ask.md", "line": 36 }, { "feature_id": "F10.navigate-errors", "description": "an unknown `event_id` is 404 `Unknown event_id: ...` and an unknown conversation 404, a missing body 422, a missing key 401; none of them moves HEAD.", "file": "F10-fork-navigate-condense-ask.md", "line": 37 }, { "feature_id": "F10.navigate-final-response", "description": "after navigating back before the agent's answer, `agent_final_response` reports the active branch (empty), not the abandoned answer.", "file": "F10-fork-navigate-condense-ask.md", "line": 38 }, { "feature_id": "F10.condense-llm", "description": "on a finished DeepSeek conversation `POST /condense` answers 200 `{\"success\": true}` after recording one `CondensationRequest` and one `Condensation` with a summary, pushed live on the events socket; the status stays `finished`.", "file": "F10-fork-navigate-condense-ask.md", "line": 39 }, { "feature_id": "F10.condense-no-condenser", "description": "an agent without a request-handling condenser (condenser disabled, or `no_op`) gets an error naming the cause, and no `CondensationRequest` or other event is recorded.", "file": "F10-fork-navigate-condense-ask.md", "line": 40 }, { "feature_id": "F10.condense-status", "description": "that refusal is a client error (4xx), not a 500.", "file": "F10-fork-navigate-condense-ask.md", "line": 41 }, { "feature_id": "F10.condense-llm-failure", "description": "when the condenser's model is unreachable, condense answers 500 with an `error_id`, leaves the `CondensationRequest` pending and records no `Condensation`.", "file": "F10-fork-navigate-condense-ask.md", "line": 42 }, { "feature_id": "F10.condense-errors", "description": "an unknown id is 404 `Conversation not found`, a malformed id 422, a missing key 401 (and records nothing).", "file": "F10-fork-navigate-condense-ask.md", "line": 43 }, { "feature_id": "F10.ask-agent", "description": "`POST /ask_agent` answers a question about the conversation with DeepSeek (200 `{\"response\": ...}`), records no message and pushes no message on the socket, and leaves the status unchanged.", "file": "F10-fork-navigate-condense-ask.md", "line": 44 }, { "feature_id": "F10.ask-agent-errors", "description": "a missing `question` is 422, a missing key 401, and an unreachable model 500 with an `error_id`.", "file": "F10-fork-navigate-condense-ask.md", "line": 45 }, { "feature_id": "F10.ask-agent-running", "description": "while a run waits on its model, `ask_agent` answers at once (it does not wait for the step), records nothing and leaves the status `running`.", "file": "F10-fork-navigate-condense-ask.md", "line": 46 }, { "feature_id": "F10.ask-agent-unknown", "description": "an unknown conversation id is 404, as the route documents.", "file": "F10-fork-navigate-condense-ask.md", "line": 47 }, { "feature_id": "F10.restart-persist", "description": "after a restart, forks keep their lineage, title, tags, HEAD and events, a fork id that names a persisted but not yet loaded conversation is still 409, and a navigated empty HEAD stays empty (the next message is still a new root).", "file": "F10-fork-navigate-condense-ask.md", "line": 48 }, { "feature_id": "F10.restart-cold-calls", "description": "after a restart, condense and ask_agent work on a conversation whose agent has not been initialized yet, and ask_agent still sees the history (the codename).", "file": "F10-fork-navigate-condense-ask.md", "line": 49 }, { "feature_id": "F11.goal-validation", "description": "a blank or whitespace objective returns 400 `Goal objective must not be empty.`, `max_iterations` below 1 or a missing objective returns 422, and nothing is recorded.", "file": "F11-goals.md", "line": 29 }, { "feature_id": "F11.goal-unknown-id", "description": "all three routes return 404 for an unknown conversation id and 422 for a malformed one.", "file": "F11-goals.md", "line": 30 }, { "feature_id": "F11.goal-auth", "description": "all three routes return 401 without a session key or with a wrong one, while the right key is accepted.", "file": "F11-goals.md", "line": 31 }, { "feature_id": "F11.goal-stop-noop", "description": "`/goal/stop` with no active loop returns 200 and records nothing.", "file": "F11-goals.md", "line": 32 }, { "feature_id": "F11.goal-resume-none", "description": "`/goal/resume` on a conversation that never had a goal returns 400 `no_resumable_goal` and records nothing.", "file": "F11-goals.md", "line": 33 }, { "feature_id": "F11.goal-complete", "description": "on DeepSeek flash a small objective runs to a goal status `complete` (`active: false`, a verdict with `complete: true`), the agent's file is in the workspace and the conversation is `finished`.", "file": "F11-goals.md", "line": 34 }, { "feature_id": "F11.goal-ws", "description": "the events socket pushes the goal status as `ConversationStateUpdateEvent` frames with `key: \"goal\"`: `running` at the start, then the terminal status.", "file": "F11-goals.md", "line": 35 }, { "feature_id": "F11.goal-in-place", "description": "the loop creates no conversation; the objective is a user message and the agent's tool work is in the same conversation's event log.", "file": "F11-goals.md", "line": 36 }, { "feature_id": "F11.goal-followup", "description": "an incomplete verdict records a `running` status with `iteration: 1` and that verdict, then posts a follow-up user message carrying the judge's `missing` text and runs another round.", "file": "F11-goals.md", "line": 37 }, { "feature_id": "F11.goal-capped", "description": "when no verdict says complete, the loop stops after `max_iterations` rounds with `status: capped`, `iteration` equal to the cap and the last verdict, after exactly one agent and one judge call per round.", "file": "F11-goals.md", "line": 38 }, { "feature_id": "F11.goal-resume-finished", "description": "`/goal/resume` after a `complete` or `capped` goal returns 400 `no_resumable_goal`.", "file": "F11-goals.md", "line": 39 }, { "feature_id": "F11.goal-busy", "description": "while a goal round is in flight, a second `/goal`, `/goal/resume` and `/run` return 409 and add no goal status; an omitted `max_iterations` defaults to 10.", "file": "F11-goals.md", "line": 40 }, { "feature_id": "F11.goal-stop", "description": "`/goal/stop` on an active loop returns 200 at once and records `interrupted` (`active: false`) with the same objective and iteration, but cancels only the loop: the in-flight round keeps the conversation `running` until it is interrupted.", "file": "F11-goals.md", "line": 41 }, { "feature_id": "F11.goal-superseded", "description": "a user message posted through `POST /events` while a loop is active cancels the loop (`interrupted`) and is recorded.", "file": "F11-goals.md", "line": 42 }, { "feature_id": "F11.goal-superseded-socket", "description": "a user message sent over the events socket cancels an active loop the same way.", "file": "F11-goals.md", "line": 43 }, { "feature_id": "F11.goal-interrupt-run", "description": "interrupting the conversation during a goal round leaves it `paused` and the goal `interrupted`.", "file": "F11-goals.md", "line": 44 }, { "feature_id": "F11.goal-busy-run", "description": "`/goal` and `/goal/resume` while an ordinary run is active return 409 and add no goal status.", "file": "F11-goals.md", "line": 45 }, { "feature_id": "F11.goal-resume", "description": "`/goal/resume` after a stop continues from the persisted objective, cap and iteration with a resume prompt (not the objective again) and runs to `complete`.", "file": "F11-goals.md", "line": 46 }, { "feature_id": "F11.goal-restart-resume", "description": "an interrupted goal's status survives a restart and `/goal/resume` continues it afterwards.", "file": "F11-goals.md", "line": 47 }, { "feature_id": "F11.goal-run-error", "description": "a round whose model provider is unreachable ends the conversation in `error` (with a `ConversationErrorEvent`) and the goal `interrupted`, which stays resumable.", "file": "F11-goals.md", "line": 48 }, { "feature_id": "F11.goal-judge-error", "description": "a judge call that fails after a successful round records `interrupted` and leaves the conversation `finished`.", "file": "F11-goals.md", "line": 49 }, { "feature_id": "F11.goal-capacity", "description": "between rounds (conversation `finished`, judge call in flight) the loop stays busy (409 on its own conversation) and holds its run slot, so with the capacity full `/goal` and `/goal/resume` elsewhere return 429 and record nothing; stopping the loop frees the slot.", "file": "F11-goals.md", "line": 50 }, { "feature_id": "F11.goal-delete-active", "description": "deleting a conversation whose loop is active answers 200 promptly, the conversation is gone, and the loop's run slot is returned (the next `/goal` elsewhere is admitted, not 429).", "file": "F11-goals.md", "line": 51 }, { "feature_id": "F11.goal-cap-overrun", "description": "a goal resumed after its last round's judge call failed never reports more rounds than `max_iterations` (no goal status has `iteration` above the cap).", "file": "F11-goals.md", "line": 52 }, { "feature_id": "F11.goal-restart-active", "description": "a loop that was active when the server restarted is reported `interrupted` (`active: false`) after the restart instead of `running`.", "file": "F11-goals.md", "line": 53 }, { "feature_id": "F12.discovery", "description": "the OpenAPI document lists the two runtime routes but not the pause and credential-binding routes, and `/server_info` advertises `credential_binding_v1`, `credential_binding_readiness_probe_v1` and `credential_binding_activation_guard_v1`.", "file": "F12-runtime-and-credentials.md", "line": 41 }, { "feature_id": "F12.runtime-info", "description": "`GET .../runtime` returns 200 `{\"runtime_status\": \"available\", \"can_resume\": true, \"runtime_error\": null}` for a local conversation, and the conversation's GET, search and batch-get rows embed the same object as `runtime_info`.", "file": "F12-runtime-and-credentials.md", "line": 42 }, { "feature_id": "F12.runtime-reprovision", "description": "`POST .../runtime/reprovision` returns the same runtime info, ignores a body, and changes nothing (status, event count and lease generation stay the same).", "file": "F12-runtime-and-credentials.md", "line": 43 }, { "feature_id": "F12.runtime-errors", "description": "both runtime routes answer 404 for an unknown id, 422 for a non-UUID id and 401 without a session key.", "file": "F12-runtime-and-credentials.md", "line": 44 }, { "feature_id": "F12.ts-client", "description": "the built TypeScript client's `ConversationClient.getRuntime` and `reprovisionRuntime` return the local runtime info, `getConversation()` embeds it as `runtime_info`, and both methods reject an unknown id with a 404 `HttpError` and a client without a key with 401.", "file": "F12-runtime-and-credentials.md", "line": 45 }, { "feature_id": "F12.sandbox-pause-releases", "description": "`prepare-for-sandbox-pause` answers 204 with an empty body, deletes every `owner_lease.json`, keeps every conversation listed and readable with its status, is idempotent, and is refused with 401 (leaving the leases) without a key.", "file": "F12-runtime-and-credentials.md", "line": 46 }, { "feature_id": "F12.runtime-cold", "description": "runtime info answers for a conversation that is not loaded, before and after a restart, without loading it (no lease file appears).", "file": "F12-runtime-and-credentials.md", "line": 47 }, { "feature_id": "F12.sandbox-pause-rehydrate", "description": "after a pause (no restart in between), the next event access loads the conversation again (a new lease, the same events), and a new message runs to `finished` on a real model, which acts again.", "file": "F12-runtime-and-credentials.md", "line": 48 }, { "feature_id": "F12.sandbox-pause-running", "description": "a conversation whose model call is in flight is cancelled to `paused` within the server's 10 s bound, with a `PauseEvent` and an `InterruptEvent`, and the status change is pushed on its events socket.", "file": "F12-runtime-and-credentials.md", "line": 49 }, { "feature_id": "F12.sandbox-pause-websocket", "description": "an events socket or a session socket that was open before the pause is either closed by the server or keeps receiving the conversation's new events; it is never left open and silent.", "file": "F12-runtime-and-credentials.md", "line": 50 }, { "feature_id": "F12.binding-unsupported-secret", "description": "only file credentials can be bound: any other secret name gets 404 without the source being contacted.", "file": "F12-runtime-and-credentials.md", "line": 51 }, { "feature_id": "F12.binding-validation", "description": "a body with extra keys, no `url`, an empty or over-long (4097+ characters) `url`, an empty or over-long (8193+ characters) header value or more than 16 headers, or a non-UUID id, gets 422 without the source being contacted, also for an unsupported secret name (the body is validated first); no key gets 401.", "file": "F12-runtime-and-credentials.md", "line": 52 }, { "feature_id": "F12.binding-pending", "description": "a binding for a conversation id the server does not know answers 204 after exactly one probe `GET` that carries the given headers; the id stays unknown (404).", "file": "F12-runtime-and-credentials.md", "line": 53 }, { "feature_id": "F12.binding-source-statuses", "description": "the source's answer maps to the activation status: 401/403 to 403, 400/404/422 to 422, 409 to 409, 501 to 501, a 200 that is not `{\"value\": str, \"version\": non-empty str}` to 422, and a URL that is not http(s) to 422, each after a single probe (none for the bad URL).", "file": "F12-runtime-and-credentials.md", "line": 54 }, { "feature_id": "F12.binding-source-unavailable", "description": "a source answering 5xx is probed three times with backoff (at least 0.75 s), then the activation fails with 502; an unreachable source also gives 502.", "file": "F12-runtime-and-credentials.md", "line": 55 }, { "feature_id": "F12.binding-native-too-late", "description": "binding a loaded regular (non-ACP) conversation is refused with 409 `Credential binding activation arrived after ACP initialization` and records nothing.", "file": "F12-runtime-and-credentials.md", "line": 56 }, { "feature_id": "F12.binding-native-cold", "description": "a binding sent while a regular (non-ACP) conversation is not loaded never makes that conversation demand a credential binding after the next pause.", "file": "F12-runtime-and-credentials.md", "line": 57 }, { "feature_id": "F12.binding-live-acp", "description": "binding a loaded ACP Codex conversation that has not run yet answers 204, records `required_runtime_credential_bindings: [\"CODEX_AUTH_JSON\"]` in `meta.json` and removes the stored `CODEX_AUTH_JSON` secret.", "file": "F12-runtime-and-credentials.md", "line": 58 }, { "feature_id": "F12.binding-rebind", "description": "a second activation with the same URL and new headers is accepted (the probe carries the new headers, which are never written to disk); a different URL is refused with 409.", "file": "F12-runtime-and-credentials.md", "line": 59 }, { "feature_id": "F12.binding-required-after-pause", "description": "after a pause, a conversation that requires a binding is still listed and reports runtime `available`, but event access and `/run` answer 409 `{\"detail\": \"credential_binding_activation_required\", \"retryable\": true}` and both conversation sockets close with 1013, until a new activation, after which events load again.", "file": "F12-runtime-and-credentials.md", "line": 60 }, { "feature_id": "F12.binding-required-after-restart", "description": "the requirement survives a restart (409 until a new activation, then 200), and deleting such a conversation needs no activation.", "file": "F12-runtime-and-credentials.md", "line": 61 }, { "feature_id": "F12.binding-pending-on-create", "description": "a pending binding is consumed when a conversation is created with that `conversation_id`, which then requires the binding from the start and stores no `CODEX_AUTH_JSON` secret.", "file": "F12-runtime-and-credentials.md", "line": 62 }, { "feature_id": "F12.binding-pending-cleared", "description": "a pause drops pending bindings, so a conversation created afterwards with that id keeps its own `CODEX_AUTH_JSON` secret and requires no binding.", "file": "F12-runtime-and-credentials.md", "line": 63 }, { "feature_id": "F12.binding-profile-scope", "description": "a conversation launched from an agent profile whose `secret_refs` exclude `CODEX_AUTH_JSON` refuses the binding with 403 `Credential binding authorization was rejected` and records nothing.", "file": "F12-runtime-and-credentials.md", "line": 64 }, { "feature_id": "F12.runtime-launched-profile", "description": "on a local server started with `OH_RUNTIME_LAUNCHED_PROFILE` (the launched profile a Docker parent passes into each conversation container), a conversation created without an agent profile records that profile as `launched_agent_profile` and its `secret_refs` scope the credential binding (403), while a conversation started from its own agent profile keeps its own profile (204).", "file": "F12-runtime-and-credentials.md", "line": 65 }, { "feature_id": "F12.managed-key-refresh", "description": "with `OH_LLM_API_KEY_REFRESH_URL`, `OH_LLM_API_KEY_REFRESH_BASE_URLS` naming the LLM's `base_url` and `OH_LLM_API_KEY_REFRESH_HEADERS` (values expanded from the server environment), a model call rejected with 401 makes exactly one `GET` of the refresh URL with the expanded headers and one retry, and the run finishes; the fetched key is not written to the server's state.", "file": "F12-runtime-and-credentials.md", "line": 66 }, { "feature_id": "F12.managed-key-scope", "description": "an LLM whose `base_url` is outside the allow-list, or any LLM once the allow-list is empty, surfaces the 401 (`LLMAuthenticationError`, status `error`) after one model call and no refresh request; the empty allow-list is logged as \"stays off\".", "file": "F12-runtime-and-credentials.md", "line": 67 }, { "feature_id": "F12.managed-key-switch", "description": "an in-scope LLM installed with `switch_llm` (or `switch_profile`) recovers from a 401 through the refresh URL like the LLM the conversation was loaded with.", "file": "F12-runtime-and-credentials.md", "line": 68 }, { "feature_id": "F12.docker-runtime-contract", "description": "on a server with `OH_CONVERSATION_RUNTIME=docker` and no Docker daemon, fork, `switch_llm`, `switch_profile` and credential bindings answer 501 before any lookup (the binding source is never contacted); the runtime routes, `DELETE .../runtime`, `POST .../runtime/credentials` and `/run` answer 404 `Conversation not found` for an unknown id, and the two Docker-only routes 401 without a key; both conversation sockets close with 1008 for an unknown id; create refuses a missing body (400), a non-object (422), a non-local workspace (422) and a bad `conversation_id` (400) without writing a runtime manifest; a well-formed create answers 502, leaving a manifest whose runtime info is 404, whose reprovision is 502 and whose socket closes with 1011, until `DELETE /{id}` removes it.", "file": "F12-runtime-and-credentials.md", "line": 69 }, { "feature_id": "F12.docker-runtime", "description": "with a Docker daemon, a create starts a container with the configured `--memory`, `--cpus` and `--pids-limit`, the launched profile in `OH_RUNTIME_LAUNCHED_PROFILE` and a startup bound of `conversation_container_startup_timeout`; `POST .../runtime/credentials` returns the container's session key, `DELETE .../runtime` stops it (204) while the conversation stays readable, reprovision starts it again, a request carrying `X-Expose-Secrets` is refused with 422 and an archived runtime answers 410 (blocked here: no Docker daemon).", "file": "F12-runtime-and-credentials.md", "line": 70 }, { "feature_id": "F13.auth", "description": "every bash route answers 401 without or with a wrong session key (and runs nothing); `WS /sockets/bash-events` closes with 4001 on a bad key.", "file": "F13-bash.md", "line": 32 }, { "feature_id": "F13.execute-sync", "description": "`POST /api/bash/execute_bash_command` blocks until the command exits and returns its final `BashOutput` with the real exit code and separate stdout and stderr; a body without `command` or with a non-integer `timeout` is 422.", "file": "F13-bash.md", "line": 33 }, { "feature_id": "F13.global-cwd", "description": "global commands run in the server process cwd by default, accept any explicit cwd, and a cwd that does not exist yields 200 with `exit_code` -1 and an `Error executing command` stderr.", "file": "F13-bash.md", "line": 34 }, { "feature_id": "F13.env-sanitized", "description": "commands run without `OH_SESSION_API_KEYS_*`, `SESSION_API_KEY` and `OH_SECRET_KEY` in their environment and with `AI_AGENT=openhands`.", "file": "F13-bash.md", "line": 35 }, { "feature_id": "F13.env-config-credentials", "description": "only those three names are stripped: other credential-bearing configuration variables (`OH_TELEMETRY_HTTP_TOKEN`, `OH_WEBHOOKS_0_HEADERS`) reach commands unchanged.", "file": "F13-bash.md", "line": 36 }, { "feature_id": "F13.start-async", "description": "`POST /api/bash/start_bash_command` returns the `BashCommand` before the command finishes; its `BashOutput` with `exit_code` appears later in search.", "file": "F13-bash.md", "line": 37 }, { "feature_id": "F13.ws-live", "description": "`WS /sockets/bash-events` streams `BashCommand` and `BashOutput` frames for commands started over REST, and `resend_mode=all` replays stored events on connect.", "file": "F13-bash.md", "line": 38 }, { "feature_id": "F13.ws-command", "description": "a client frame shaped like `ExecuteBashRequest` starts a command on the socket; an invalid frame gets a `BashError` frame and the socket keeps working.", "file": "F13-bash.md", "line": 39 }, { "feature_id": "F13.ws-resend-all", "description": "the deprecated `resend_all=true` query parameter replays stored events like `resend_mode=all` (and logs a deprecation warning), while a connection without either replays nothing.", "file": "F13-bash.md", "line": 40 }, { "feature_id": "F13.ws-subscriber-cap", "description": "`WS /sockets/bash-events` holds at most 50 subscribers: with 50 open, the 51st is closed with 1013 `Too many bash event connections`, and closing one frees the slot.", "file": "F13-bash.md", "line": 41 }, { "feature_id": "F13.get-events", "description": "`GET /api/bash/bash_events/{event_id}` returns one event by its hex id (404 when unknown), and `GET /api/bash/bash_events/` with a JSON array body returns events in input order with `null` for missing ids (422 without a body).", "file": "F13-bash.md", "line": 42 }, { "feature_id": "F13.search", "description": "search filters by `kind__eq`, `command_id__eq` (outputs only), a UTC `timestamp__gte`/`timestamp__lt` window, sorts with `sort_order`, pages with `limit`/`next_page_id`, and rejects bad parameters with 422.", "file": "F13-bash.md", "line": 43 }, { "feature_id": "F13.timestamp-offset", "description": "a `timestamp__gte` with a non-UTC offset selects the same instant as its UTC spelling.", "file": "F13-bash.md", "line": 44 }, { "feature_id": "F13.output-chunks", "description": "output over 1 MiB is stored as ordered `BashOutput` chunks of 1048576 characters plus a final chunk with the exit code; `order__gt` skips seen chunks, and the SDK's `RemoteWorkspace.execute_command` reassembles them.", "file": "F13-bash.md", "line": 45 }, { "feature_id": "F13.order-gt-paging", "description": "`order__gt` filters before `limit` applies, so a poller asking for outputs after order N with a small page gets them.", "file": "F13-bash.md", "line": 46 }, { "feature_id": "F13.execute-large-output", "description": "`execute_bash_command` returns the complete stdout of a command whose output is over 1 MiB.", "file": "F13-bash.md", "line": 47 }, { "feature_id": "F13.timeout", "description": "a command that outlives its `timeout` is killed with its process group and reports `exit_code` -1.", "file": "F13-bash.md", "line": 48 }, { "feature_id": "F13.stop", "description": "stopping a running command by id terminates only that command (final `exit_code` -15, process gone) while another command finishes normally.", "file": "F13-bash.md", "line": 49 }, { "feature_id": "F13.stop-idempotent", "description": "stopping an already finished command (hex or dashed id) returns 200, an unknown id 404 and a non-UUID id 422.", "file": "F13-bash.md", "line": 50 }, { "feature_id": "F13.scoped-cwd", "description": "conversation-scoped commands run in the conversation workspace by default, accept an absolute cwd inside it (also through the SDK's `runtime_conversation_id`), and reject a cwd outside it, an escaping `..` path or a relative path with 422.", "file": "F13-bash.md", "line": 51 }, { "feature_id": "F13.scoped-routes", "description": "the conversation-scoped execute, start, stop, get, batch get and search routes work on the conversation's own history, and the command's files land in the workspace.", "file": "F13-bash.md", "line": 52 }, { "feature_id": "F13.scoped-isolation", "description": "scoped events are invisible to the global search, the global socket, the global stop route and other conversations, and are stored under the conversation's directory.", "file": "F13-bash.md", "line": 53 }, { "feature_id": "F13.scoped-unknown", "description": "every conversation-scoped bash route answers 404 `Conversation not found` for an unknown conversation id.", "file": "F13-bash.md", "line": 54 }, { "feature_id": "F13.persist-restart", "description": "global and conversation-scoped bash history survives a server restart.", "file": "F13-bash.md", "line": 55 }, { "feature_id": "F13.clear", "description": "`DELETE .../bash_events` deletes one scope's stored events and reports `cleared_count`; the other scope is untouched, and a cleared finished command can no longer be stopped (404).", "file": "F13-bash.md", "line": 56 }, { "feature_id": "F13.retention", "description": "with `bash_events_retention_seconds` set, global events older than the window are purged at startup while conversation-scoped history stays.", "file": "F13-bash.md", "line": 57 }, { "feature_id": "F13.retention-rolling", "description": "the retention loop keeps purging while the server runs: a global event written after startup is readable at once and gone after the next pass (about 60 s) with no restart, while a conversation-scoped event stays.", "file": "F13-bash.md", "line": 58 }, { "feature_id": "F13.scoped-bad-id", "description": "a conversation-scoped bash route with a non-UUID conversation id answers 422.", "file": "F13-bash.md", "line": 59 }, { "feature_id": "F13.scoped-idle-eviction", "description": "a conversation with a running scoped command is not evicted as idle, so the command keeps running.", "file": "F13-bash.md", "line": 60 }, { "feature_id": "F14.auth", "description": "every global and conversation-scoped file route answers 401 without a valid session key, and nothing is written.", "file": "F14-files.md", "line": 31 }, { "feature_id": "F14.upload-download", "description": "a multipart upload creates missing parent directories, a download returns byte-identical bytes as an `application/octet-stream` attachment named after the basename, and a second upload to the same path replaces the content.", "file": "F14-files.md", "line": 32 }, { "feature_id": "F14.upload-errors", "description": "an upload without the `file` part or without `path` is 422, and a relative `path` is 400 `Path must be absolute`.", "file": "F14-files.md", "line": 33 }, { "feature_id": "F14.upload-onto-directory", "description": "uploading onto an existing directory, or to a path whose parent is a file, is a client error (4xx); today it is a masked 500.", "file": "F14-files.md", "line": 34 }, { "feature_id": "F14.upload-size", "description": "the SDK's `RemoteWorkspace.file_upload` and the TypeScript `RemoteWorkspace.fileUpload` report the uploaded `file_size`, as `LocalWorkspace.file_upload` does; today the server sends only `{\"success\": true}` and both report no size.", "file": "F14-files.md", "line": 35 }, { "feature_id": "F14.upload-atomic", "description": "an overwrite that fails mid-write leaves the previous file intact; today the target is truncated before streaming and keeps a partial file.", "file": "F14-files.md", "line": 36 }, { "feature_id": "F14.download-errors", "description": "downloading a missing file is 404 `File not found`, a directory 400 `Path is not a file`, a relative path 400.", "file": "F14-files.md", "line": 37 }, { "feature_id": "F14.mkdir", "description": "`create_directory` creates missing parents, is idempotent and keeps existing contents, the new directory is listed by `search_subdirs`, and a path that is or runs through a file is 400.", "file": "F14-files.md", "line": 38 }, { "feature_id": "F14.home", "description": "`GET /api/file/home` reports the server user's `HOME`, its visible top-level directories as case-insensitively sorted `favorites` (no files, no symlinks), `/` as the only location, and hidden directories only with `include_hidden=true`.", "file": "F14-files.md", "line": 39 }, { "feature_id": "F14.subdirs", "description": "`search_subdirs` lists only the immediate real subdirectories of an absolute path (no files, no symlinks, hidden only with `include_hidden=true`), sorted case-insensitively, with absolute paths.", "file": "F14-files.md", "line": 40 }, { "feature_id": "F14.subdirs-pagination", "description": "`limit` and the `next_page_id` cursor page through the entries; `limit` outside 1..100 is 422.", "file": "F14-files.md", "line": 41 }, { "feature_id": "F14.subdirs-errors", "description": "`search_subdirs` on a missing directory is 404, on a file 400, on a relative path 400.", "file": "F14-files.md", "line": 42 }, { "feature_id": "F14.subdirs-case-pagination", "description": "following `next_page_id` across two directories that differ only by case (`Foo`, `foo`) reaches the second one instead of repeating the first page forever.", "file": "F14-files.md", "line": 43 }, { "feature_id": "F14.archive-tar", "description": "`format=tar.gz` returns an `application/gzip` archive named `.tar.gz`, rooted at `/`, with `archive_manifest.json`, the repository identity headers but no `X-Archive-Base-*`, the default excludes applied (`.git/`, `node_modules/`), symlinks skipped, and no temp file left in the parent directory.", "file": "F14-files.md", "line": 44 }, { "feature_id": "F14.archive-excludes", "description": "`use_default_excludes=false` keeps `node_modules` and `.git` internals but never `.git/config` or reflogs; `exclude=` (repeatable) drops matching files and is recorded in the manifest.", "file": "F14-files.md", "line": 45 }, { "feature_id": "F14.archive-git-delta", "description": "the default `git-delta` format with `base_ref=HEAD` is a `text/x-patch` of the uncommitted and untracked changes that applies to a clone of HEAD, carries `X-Archive-Base-Commit`, `X-Archive-Base-Ref`, `X-Archive-Branch` and `X-Archive-Head-Commit`, and leaves the repository's index untouched; without `base_ref` on a remote-less repo the patch is against the empty tree and has no base headers; a subdirectory of a repository yields only that subtree's delta.", "file": "F14-files.md", "line": 46 }, { "feature_id": "F14.archive-auto-base", "description": "without `base_ref`, a clone with an `origin` remote is diffed against `origin/` (the patch holds the local commit and the uncommitted edit, applies to a clone of the origin tip, and carries `X-Archive-Base-Ref: auto`, `X-Archive-Base-Commit` = the origin tip and `X-Archive-Repo-Remote`); a detached HEAD is diffed against HEAD itself.", "file": "F14-files.md", "line": 47 }, { "feature_id": "F14.archive-nested-repo", "description": "archiving a non-repo directory that holds exactly one git repository archives that repository; with two repositories `git-delta` is 400 and `tar.gz` archives the directory itself.", "file": "F14-files.md", "line": 48 }, { "feature_id": "F14.archive-format-doc", "description": "the OpenAPI description of `format` matches what a non-repository directory holding one repository archives; today it promises a tarball of \"the entire directory\" while only the nested repository is packed.", "file": "F14-files.md", "line": 49 }, { "feature_id": "F14.archive-errors", "description": "a relative path is 400, a missing directory 404, a file 400, a `base_ref` starting with `-` 400, an unknown `base_ref` 400, an unknown `format` 422, and `git-delta` of a non-repo directory 400.", "file": "F14-files.md", "line": 50 }, { "feature_id": "F14.trajectory", "description": "`GET /api/file/download-trajectory/{id}` returns `.zip` with `meta.json`, `base_state.json` and one file per event, the LLM `api_key` redacted to `` and no Fernet ciphertext, for dashed and hex ids, and leaves no temp zip behind.", "file": "F14-files.md", "line": 51 }, { "feature_id": "F14.trajectory-errors", "description": "an unknown conversation id is 404 `Conversation not found`, a malformed id 422.", "file": "F14-files.md", "line": 52 }, { "feature_id": "F14.trajectory-concurrent", "description": "concurrent downloads of the same trajectory each return 200 and a valid zip (five rounds of six parallel downloads, 30 in all).", "file": "F14-files.md", "line": 53 }, { "feature_id": "F14.trajectory-init-path", "description": "after `POST /api/init` sets `conversations_path` on a deferred-init server, trajectories stored there are downloadable.", "file": "F14-files.md", "line": 54 }, { "feature_id": "F14.scoped-transfer", "description": "conversation-scoped upload, download and `create_directory` work inside the conversation's workspace, and the scoped download returns the file the agent wrote.", "file": "F14-files.md", "line": 55 }, { "feature_id": "F14.scoped-path-guard", "description": "scoped routes reject paths outside the workspace (another conversation's workspace, `..` traversal, a symlink escape, a sibling with the same prefix, a relative path) with 422 and write nothing, while the global route serves the same path.", "file": "F14-files.md", "line": 56 }, { "feature_id": "F14.scoped-conversation-errors", "description": "a scoped route for an unknown conversation is 404 and does nothing; a malformed conversation id is 422.", "file": "F14-files.md", "line": 57 }, { "feature_id": "F14.scoped-trajectory-archive", "description": "the scoped trajectory route serves only its own conversation (another or malformed id is 422), and the scoped archive packs the workspace, including the agent's file, but refuses a path outside it with 422.", "file": "F14-files.md", "line": 58 }, { "feature_id": "F14.sdk-workspace", "description": "the SDK's `RemoteWorkspace.file_upload`/`file_download` round-trip bytes through the global routes and, with `runtime_conversation_id`, through the scoped routes; a scoped upload outside the workspace returns `success=False` with the 422 and writes nothing.", "file": "F14-files.md", "line": 59 }, { "feature_id": "F14.restart", "description": "after a restart, uploaded files, the scoped download and the trajectory are still served.", "file": "F14-files.md", "line": 60 }, { "feature_id": "F15.auth", "description": "every global and conversation-scoped git route answers 401 without a valid session key.", "file": "F15-git.md", "line": 32 }, { "feature_id": "F15.changes", "description": "`changes` on the fixture repository lists the modified tracked file as `UPDATED` and the untracked file as `ADDED`, sorted by path, as repository-relative POSIX paths, matching `git status`.", "file": "F15-git.md", "line": 33 }, { "feature_id": "F15.changes-statuses", "description": "a staged deletion is `DELETED`, a staged new file `ADDED`, and a staged rename a `DELETED` old path plus an `ADDED` new path.", "file": "F15-git.md", "line": 34 }, { "feature_id": "F15.changes-base", "description": "without `ref` the display base is `HEAD` on a remote-less default branch, the fork point from local `main` on a feature branch (committed work stays visible), the fork point from `origin`'s default branch on a fully pushed clean branch, and the branch's own upstream once the tree is dirty.", "file": "F15-git.md", "line": 35 }, { "feature_id": "F15.changes-ref", "description": "`ref=HEAD` gives `git status`-style changes and an older commit adds its later commits' files; an unknown or option-shaped `ref` is 400 and never runs as a git option; no `path` is 422.", "file": "F15-git.md", "line": 36 }, { "feature_id": "F15.changes-nested-repo", "description": "changes inside a repository nested one level below the requested repository are listed with the nested directory as prefix, the nested directory itself is not listed, and `diff` of such an entry reads the nested repository.", "file": "F15-git.md", "line": 37 }, { "feature_id": "F15.changes-repo-below-path", "description": "a non-repository directory that holds one repository (the layout of a repository-backed conversation) lists that repository's changes with its directory as prefix; today it lists nothing.", "file": "F15-git.md", "line": 38 }, { "feature_id": "F15.changes-special-names", "description": "a modified tracked file whose name has a space or a non-ASCII character is listed under its real name; today a space fails the whole list with 400 and a non-ASCII name comes back as a mangled git-quoted string.", "file": "F15-git.md", "line": 39 }, { "feature_id": "F15.changes-subdir-paths", "description": "`changes` for a subdirectory of a repository returns paths on one base; today tracked paths are repository-relative (and include files outside the subdirectory) while untracked paths are subdirectory-relative.", "file": "F15-git.md", "line": 40 }, { "feature_id": "F15.non-repo", "description": "on a plain directory, a missing directory and a non-repository file every operation answers 200 with an empty result (`[]`, `{\"original\": null, \"modified\": null}`, `{\"commits\": [], \"has_more\": false}`); a repository without commits lists its files as `ADDED` and has no commits.", "file": "F15-git.md", "line": 41 }, { "feature_id": "F15.diff", "description": "`diff` for a modified file returns the base text as `original` and the disk text as `modified` (final newline dropped on both sides), an untracked file has `original` `\"\"`, and an explicit `ref` moves the base.", "file": "F15-git.md", "line": 42 }, { "feature_id": "F15.diff-commit", "description": "`commit=` returns the file as that commit changed it, read from git objects: `original` from the first parent (empty for the root commit or an added file), `modified` from the commit (empty for a file it deleted), also for files no longer on disk and for abbreviated SHAs.", "file": "F15-git.md", "line": 43 }, { "feature_id": "F15.diff-errors", "description": "`ref` together with `commit` is 400, a non-hex `commit` 422, an unknown commit or ref 400, a missing file 400, a file over 1 MiB 400, and no `path` 422.", "file": "F15-git.md", "line": 44 }, { "feature_id": "F15.diff-whitespace", "description": "an unchanged file with leading indentation or trailing blank lines diffs to identical `original` and `modified`; today `original` loses them, so a file that `changes` does not list shows a phantom diff.", "file": "F15-git.md", "line": 45 }, { "feature_id": "F15.diff-deleted-file", "description": "the working-tree diff of a file that `changes` lists as `DELETED` returns its base text as `original` and `\"\"` as `modified`; today it is 400 `File does not exist`.", "file": "F15-git.md", "line": 46 }, { "feature_id": "F15.commits", "description": "`commits` lists the history newest first with `sha`, `short_sha`, `subject`, `author` and an ISO 8601 `timestamp`, `has_more` tells whether `limit` cut it, and `limit` outside 1..200 is 422.", "file": "F15-git.md", "line": 47 }, { "feature_id": "F15.commit-changes", "description": "`commits/{sha}/changes` lists one commit's files against its first parent (the root commit as all `ADDED`, a rename as `DELETED` plus `ADDED`), accepts abbreviated SHAs, answers 400 for an unknown SHA and 422 for a non-hex or shorter-than-4 SHA.", "file": "F15-git.md", "line": 48 }, { "feature_id": "F15.scoped", "description": "the four conversation-scoped routes on a conversation whose workspace is the fixture repository return exactly what the global routes return.", "file": "F15-git.md", "line": 49 }, { "feature_id": "F15.scoped-guard", "description": "scoped routes refuse a `path` outside the workspace (another directory, a same-prefix sibling, `..`, a symlink escape, a relative path, a second `path` parameter pointing outside) with 422 while the global route serves it; an unknown conversation is 404, a malformed id 422, and there is no scoped repository search.", "file": "F15-git.md", "line": 50 }, { "feature_id": "F15.scoped-restart", "description": "after a restart the scoped routes still resolve the conversation's workspace and return the same changes.", "file": "F15-git.md", "line": 51 }, { "feature_id": "F15.sdk-workspace", "description": "the Python SDK's `RemoteWorkspace.git_changes` and `git_diff` join relative paths to `working_dir` and return the same results plain and scoped (`runtime_conversation_id`), and the scoped workspace refuses a path outside it with 422.", "file": "F15-git.md", "line": 52 }, { "feature_id": "F15.ts-client", "description": "the TypeScript client's `RemoteWorkspace.gitChanges` (with `ref`) and `gitDiff` return the same results plain and with `conversationId`, the scoped one rejects a path outside the workspace, and `GitClient.searchRepositories` returns the `missing_token` page.", "file": "F15-git.md", "line": 53 }, { "feature_id": "F15.agent-changes", "description": "after a real model run writes a file in a repository workspace, the scoped `changes` lists it as `ADDED` and the scoped `diff` shows the written text, and the event stream has the file editor's observation.", "file": "F15-git.md", "line": 54 }, { "feature_id": "F15.repo-search-token", "description": "without a stored token, GitHub search is 200 `{\"items\": [], \"next_page_id\": null, \"missing_token\": true}`; a secret named `github_token` or `GH_TOKEN` is picked up (a malformed `page_id` is then validated and refused with 400), and deleting it restores `missing_token`.", "file": "F15-git.md", "line": 55 }, { "feature_id": "F15.repo-search-errors", "description": "`gitlab` and `bitbucket` are 400 (not supported), an unknown or missing `provider` is 422, and `limit` outside 1..100 is 422.", "file": "F15-git.md", "line": 56 }, { "feature_id": "F15.repo-search-unreachable", "description": "when GitHub cannot be reached the search is 502 with the generic masked body and no token in it.", "file": "F15-git.md", "line": 57 }, { "feature_id": "F15.repo-search-timeout", "description": "when GitHub does not answer within 15 s the search is 504 with the masked body `GitHub repository search timed out` and no token in it.", "file": "F15-git.md", "line": 58 }, { "feature_id": "F15.repo-search-github", "description": "with a real GitHub token, search lists the token's repositories with a `page:offset` cursor and a case-insensitive `query` filter, and an invalid token is passed through as 401.", "file": "F15-git.md", "line": 59 }, { "feature_id": "F16.root-index", "description": "`GET .../workspace` and `.../workspace/` serve the workspace root's `index.html` as `text/html; charset=utf-8`, inline (no `Content-Disposition`).", "file": "F16-workspace-serving-and-vscode.md", "line": 29 }, { "feature_id": "F16.file-bytes", "description": "a nested file comes back byte for byte with `Content-Length`, `ETag`, `Last-Modified` and `Accept-Ranges: bytes`; percent-encoded names (spaces, non-ASCII) resolve.", "file": "F16-workspace-serving-and-vscode.md", "line": 30 }, { "feature_id": "F16.content-types", "description": "the `Content-Type` follows the extension (`.css`, `.json`, `.svg`, `.png`, `.txt`), and an unknown extension is `application/octet-stream`.", "file": "F16-workspace-serving-and-vscode.md", "line": 31 }, { "feature_id": "F16.dir-index", "description": "a subdirectory serves its own `index.html` with or without the trailing slash; a directory without one is 404 `No index.html in directory` (the workspace root too), and a missing file is 404 `File not found`.", "file": "F16-workspace-serving-and-vscode.md", "line": 32 }, { "feature_id": "F16.range", "description": "a `Range: bytes=0-5` request answers 206 with `Content-Range` and only those bytes.", "file": "F16-workspace-serving-and-vscode.md", "line": 33 }, { "feature_id": "F16.live-content", "description": "a file changed on disk is served fresh at once, with a new `ETag` (no server-side cache).", "file": "F16-workspace-serving-and-vscode.md", "line": 34 }, { "feature_id": "F16.traversal-rejected", "description": "encoded `..` segments and absolute paths (`//etc/hostname`, `%2Fetc%2Fhostname`) answer 400 `Path is outside the workspace`; a `..` that stays inside is served.", "file": "F16-workspace-serving-and-vscode.md", "line": 35 }, { "feature_id": "F16.symlink-escape", "description": "a symlinked file or directory that resolves outside the workspace is 400; a symlink to a file inside it is served.", "file": "F16-workspace-serving-and-vscode.md", "line": 36 }, { "feature_id": "F16.unknown-conversation", "description": "an unknown conversation id is 404 `Conversation not found: ` on both routes, a non-UUID id is 422, and the dashless hex form of a real id works.", "file": "F16-workspace-serving-and-vscode.md", "line": 37 }, { "feature_id": "F16.workspace-dir-missing", "description": "a conversation whose workspace directory was removed answers 404 `Workspace directory does not exist`, and serves again once the directory is back.", "file": "F16-workspace-serving-and-vscode.md", "line": 38 }, { "feature_id": "F16.deleted-conversation", "description": "after `DELETE /{id}` its workspace is no longer served (404 `Conversation not found`), although the files stay on disk.", "file": "F16-workspace-serving-and-vscode.md", "line": 39 }, { "feature_id": "F16.worktree-served", "description": "a `worktree: true` conversation is served from its own git worktree: files written there and the committed content, not the source checkout's uncommitted edit or untracked file.", "file": "F16-workspace-serving-and-vscode.md", "line": 40 }, { "feature_id": "F16.auth", "description": "without a key or with a wrong one both routes are 401 (also for an unknown conversation); the header works, and so does the workspace cookie alone, but not a cookie with a wrong value.", "file": "F16-workspace-serving-and-vscode.md", "line": 41 }, { "feature_id": "F16.agent-artifact", "description": "an HTML file the agent writes with the file editor is served at the conversation's workspace URL, and the events socket shows the observation that created it.", "file": "F16-workspace-serving-and-vscode.md", "line": 42 }, { "feature_id": "F16.restart", "description": "after a server restart the same conversation's workspace is served again, with the header and with the cookie minted before the restart.", "file": "F16-workspace-serving-and-vscode.md", "line": 43 }, { "feature_id": "F16.vscode-disabled", "description": "with VS Code off (the launcher's default), both status routes answer `{\"running\": false, \"enabled\": false, \"message\": \"VSCode is disabled in configuration\"}` and both URL routes 503.", "file": "F16-workspace-serving-and-vscode.md", "line": 44 }, { "feature_id": "F16.vscode-scoped-guards", "description": "the conversation-scoped VS Code routes answer 404 for an unknown conversation, 422 for a non-UUID id, and 422 `workspace_dir must be inside the conversation workspace` for a relative or outside folder, before looking at VS Code.", "file": "F16-workspace-serving-and-vscode.md", "line": 45 }, { "feature_id": "F16.vscode-auth", "description": "all four VS Code routes need the header (401 otherwise), and the workspace cookie, whose path covers the scoped routes, does not open them.", "file": "F16-workspace-serving-and-vscode.md", "line": 46 }, { "feature_id": "F16.vscode-enabled-status", "description": "with `enable_vscode` on and VS Code unable to start (its port taken, or no binary), both status routes answer `{\"running\": false, \"enabled\": true}` and the server keeps serving.", "file": "F16-workspace-serving-and-vscode.md", "line": 47 }, { "feature_id": "F16.vscode-url-shape", "description": "the URL is `http://localhost:/?tkn=&folder=workspace`; `base_url` replaces the origin, `workspace_dir` sets the folder, the scoped route defaults the folder to the conversation's `workspace.working_dir`, and `vscode_base_path` is inserted after the origin.", "file": "F16-workspace-serving-and-vscode.md", "line": 48 }, { "feature_id": "F16.vscode-url-not-running", "description": "a VS Code that is not running must not be advertised: `url` is null while `running` is false (known bug: a keyed server returns a URL carrying the session key).", "file": "F16-workspace-serving-and-vscode.md", "line": 49 }, { "feature_id": "F16.vscode-url-no-key", "description": "a server without session keys, with VS Code enabled but not running (no binary), answers `{\"url\": null}` on both URL routes.", "file": "F16-workspace-serving-and-vscode.md", "line": 50 }, { "feature_id": "F16.vscode-url-after-init", "description": "a deferred-init server that receives its session key through `POST /api/init`, with VS Code enabled but not running (no binary), answers `{\"url\": null}`.", "file": "F16-workspace-serving-and-vscode.md", "line": 51 }, { "feature_id": "F16.vscode-folder-encoding", "description": "the URL's `folder` parameter decodes back to the workspace path even when the path contains `&` or `#` (known bug: the value is not URL-encoded).", "file": "F16-workspace-serving-and-vscode.md", "line": 52 }, { "feature_id": "F16.docker-root-cookie", "description": "in Docker runtime mode the slashless workspace root accepts the workspace cookie like the slashed form does (known bug: it falls into the header-only proxy and answers 401).", "file": "F16-workspace-serving-and-vscode.md", "line": 53 }, { "feature_id": "F16.vscode-running", "description": "with the OpenVSCode Server binary installed and its port free, status reports `running: true` and the URL opens the editor (blocked on this machine: no binary).", "file": "F16-workspace-serving-and-vscode.md", "line": 54 }, { "feature_id": "F17.list-empty", "description": "on a fresh persistence directory `GET /api/workspaces` returns `{\"workspaces\": [], \"workspaceParents\": []}` and creates no `workspaces.json`.", "file": "F17-workspaces-registry.md", "line": 23 }, { "feature_id": "F17.noop-delete-no-file", "description": "on a fresh persistence directory, deleting a path from either list answers 404 and creates no `workspaces.json`, like a read (known bug: the refused delete writes an empty registry file).", "file": "F17-workspaces-registry.md", "line": 24 }, { "feature_id": "F17.add-workspaces", "description": "`POST /api/workspaces` registers a git repository and a plain directory in request order and answers the whole document; `GET` and `workspaces.json` (mode `0600`, `schema_version` 1) show the same entries.", "file": "F17-workspaces-registry.md", "line": 25 }, { "feature_id": "F17.wire-format", "description": "requests take `parentPath` or `parent_path`; responses and the file use `parentPath` and `workspaceParents`, an unset `parentPath` is omitted rather than `null`, and unknown item fields are dropped.", "file": "F17-workspaces-registry.md", "line": 26 }, { "feature_id": "F17.idempotent-add", "description": "re-posting a registered path, or the same path twice in one body, answers 200 and keeps exactly one entry, the first, with its original `id` and `name`.", "file": "F17-workspaces-registry.md", "line": 27 }, { "feature_id": "F17.delete-workspace", "description": "`DELETE /api/workspaces?path=` answers `{\"deleted\": true}` and removes only that entry, from the list and from the file, keeping the others in order.", "file": "F17-workspaces-registry.md", "line": 28 }, { "feature_id": "F17.delete-not-found", "description": "deleting a path that is not a saved workspace (a deleted one, or a near miss with a trailing slash) is 404 `Workspace not found` and changes nothing; a delete without `path` is 422.", "file": "F17-workspaces-registry.md", "line": 29 }, { "feature_id": "F17.noop-write-untouched", "description": "a write that changes nothing (re-posting saved workspaces, deleting an unknown workspace or parent) leaves `workspaces.json` as it was, same inode and modification time (known bug: every write route rewrites the file).", "file": "F17-workspaces-registry.md", "line": 30 }, { "feature_id": "F17.paths-opaque", "description": "paths are stored as given, without existence checks or normalization: a missing directory and a relative path are accepted, `/x` and `/x/` are two entries that delete separately, and a path with spaces, `&`, `?`, `#` or non-ASCII letters round-trips through an encoded delete query.", "file": "F17-workspaces-registry.md", "line": 31 }, { "feature_id": "F17.validation", "description": "a body without `workspaces` or `parents`, a non-list, an item with a missing or empty `id`, `name` or `path`, or a field over its limit (name 256, id, path and `parentPath` 4096) is 422 and stores nothing, not even the valid items of the same body; values at the limits are accepted.", "file": "F17-workspaces-registry.md", "line": 32 }, { "feature_id": "F17.add-parents", "description": "`POST /api/workspaces/parents` registers parents, deduplicated by `path` like workspaces, answers the whole document and leaves `workspaces` untouched.", "file": "F17-workspaces-registry.md", "line": 33 }, { "feature_id": "F17.parent-children", "description": "registering a parent registers none of its subdirectories; the picker lists them with `GET /api/file/search_subdirs`, and a child appears in `workspaces` only once it is posted with `parentPath` set to the parent.", "file": "F17-workspaces-registry.md", "line": 34 }, { "feature_id": "F17.delete-parent", "description": "`DELETE /api/workspaces/parents?path=` answers `{\"deleted\": true}` and removes only the parent: its child workspaces stay listed with their `parentPath`; a second delete is 404 `Workspace parent not found`, a missing `path` 422.", "file": "F17-workspaces-registry.md", "line": 35 }, { "feature_id": "F17.lists-independent", "description": "one path can be both a workspace and a parent; each delete route looks only at its own list (404 for a path that is only in the other) and deleting from one leaves the other.", "file": "F17-workspaces-registry.md", "line": 36 }, { "feature_id": "F17.persist-restart", "description": "both lists, in order and with every field, read back identically after a server restart.", "file": "F17-workspaces-registry.md", "line": 37 }, { "feature_id": "F17.auth-required", "description": "with a session key configured, all five routes answer 401 to a missing or wrong `X-Session-API-Key`, before validating the request, and change nothing.", "file": "F17-workspaces-registry.md", "line": 38 }, { "feature_id": "F17.concurrent-adds", "description": "parallel `POST`s of different workspaces and parents to one server all land, and parallel `DELETE`s of all of them all apply.", "file": "F17-workspaces-registry.md", "line": 39 }, { "feature_id": "F17.shared-dir", "description": "two servers on one persistence directory share the registry: each sees the other's writes on its next request (nothing is cached), and parallel adds and deletes split across both servers all land (the `.workspaces.lock` file lock serializes them across processes).", "file": "F17-workspaces-registry.md", "line": 40 }, { "feature_id": "F17.ts-client", "description": "the TypeScript `WorkspacesClient` lists, adds and deletes workspaces and parents through these routes, and its delete of an unknown path rejects with the server's 404.", "file": "F17-workspaces-registry.md", "line": 41 }, { "feature_id": "F17.corrupted-file", "description": "with an unparsable or empty `workspaces.json`, all five routes answer 409 `Workspaces file is corrupted` and leave the file byte for byte; restoring the file restores the lists.", "file": "F17-workspaces-registry.md", "line": 42 }, { "feature_id": "F17.future-schema", "description": "a valid document with a `schema_version` newer than 1, or one that is not an integer, or with a wrongly typed list, is 409 on reads and writes and is not overwritten; the same kind of hand-written document without `schema_version` is read, and the next write stamps `schema_version` 1.", "file": "F17-workspaces-registry.md", "line": 43 }, { "feature_id": "F17.io-error", "description": "when the file cannot be read (a directory in its place), every route answers 500 with `exception` naming the failed step (`Failed to read workspaces`, `Failed to save workspaces`, `Failed to update workspaces`), writes nothing, and the registry works again once the file is back.", "file": "F17-workspaces-registry.md", "line": 44 }, { "feature_id": "F17.noop-write-readonly", "description": "on a persistence directory the server cannot write, `GET` answers 200 and a real add 500 `Failed to save workspaces`, while a write that changes nothing keeps its normal answer: 404 for an unknown path, 200 for a re-post (known bug: they are 500 too).", "file": "F17-workspaces-registry.md", "line": 45 }, { "feature_id": "F17.persistence-location", "description": "without `OH_PERSISTENCE_DIR` the registry lives in `/../.openhands/workspaces.json`, separate from the one under `OH_PERSISTENCE_DIR`.", "file": "F17-workspaces-registry.md", "line": 46 }, { "feature_id": "F18.get-defaults", "description": "on a fresh persistence directory `GET /api/settings` returns OpenHands defaults (no API key, `llm_api_key_is_set: false`, every pointer null, empty `misc_settings`) without creating `settings.json`.", "file": "F18-settings-and-secrets.md", "line": 29 }, { "feature_id": "F18.agent-schema", "description": "`GET /api/settings/agent-schema` returns the `AgentSettings` form schema with shared, `openhands` and `acp` sections, secret fields flagged, and every OpenHands field key present in `agent_settings`.", "file": "F18-settings-and-secrets.md", "line": 30 }, { "feature_id": "F18.conversation-schema", "description": "`GET /api/settings/conversation-schema` returns the `ConversationSettings` schema whose three field keys and defaults match `conversation_settings` on a fresh server.", "file": "F18-settings-and-secrets.md", "line": 31 }, { "feature_id": "F18.auth-required", "description": "with a session key configured, every settings and secrets route answers 401 to a missing or wrong `X-Session-API-Key`, whatever `X-Expose-Secrets` says.", "file": "F18-settings-and-secrets.md", "line": 32 }, { "feature_id": "F18.patch-llm", "description": "a PATCH of `agent_settings_diff.llm` stores the model and key, flips `llm_api_key_is_set`, creates `settings.json`, and answers redacted even when `X-Expose-Secrets: plaintext` is sent.", "file": "F18-settings-and-secrets.md", "line": 33 }, { "feature_id": "F18.expose-modes", "description": "GET redacts secrets as `` by default, returns the raw value with `plaintext`, a Fernet token with `encrypted` (and the legacy `true`), and 400 for any other value.", "file": "F18-settings-and-secrets.md", "line": 34 }, { "feature_id": "F18.roundtrip", "description": "posting back the encrypted token keeps the real key; posting back the redacted placeholder clears it.", "file": "F18-settings-and-secrets.md", "line": 35 }, { "feature_id": "F18.agent-diff-merge", "description": "an `agent_settings_diff` merges sparsely (siblings such as the key survive) and a `null` unsets a field (RFC 7386).", "file": "F18-settings-and-secrets.md", "line": 36 }, { "feature_id": "F18.conversation-diff", "description": "a `conversation_settings_diff` updates only the named fields; a top-level `null` or an out-of-range value is 422 and changes nothing.", "file": "F18-settings-and-secrets.md", "line": 37 }, { "feature_id": "F18.misc-settings-merge", "description": "`misc_settings_diff` deep-merges nested objects, replaces lists, removes a nested key on `null`, and stores a top-level `null` as `null`.", "file": "F18-settings-and-secrets.md", "line": 38 }, { "feature_id": "F18.patch-validation", "description": "an empty or unknown-keys-only PATCH is 400, an invalid value is 422 `Settings validation failed` without echoing the posted secret, malformed pointers are 422, and nothing is stored.", "file": "F18-settings-and-secrets.md", "line": 39 }, { "feature_id": "F18.active-profile-pointer", "description": "PATCH `active_profile` applies that LLM profile to `agent_settings.llm`, an unknown profile is 404, an explicit `llm` diff wins over the profile, and `null` clears only the pointer.", "file": "F18-settings-and-secrets.md", "line": 40 }, { "feature_id": "F18.agent-kind-switch", "description": "switching `agent_kind` to `acp` rebuilds from a fresh ACP base (no `llm`, no key); switching back gives fresh OpenHands defaults.", "file": "F18-settings-and-secrets.md", "line": 41 }, { "feature_id": "F18.secret-put-list-get", "description": "`PUT /api/settings/secrets` answers `{name, description}`, the list shows names and descriptions but never values, GET by name returns the raw value as `text/plain`, and a second PUT overwrites value and description.", "file": "F18-settings-and-secrets.md", "line": 42 }, { "feature_id": "F18.secret-delete", "description": "DELETE answers `{\"deleted\": true}`; afterwards GET and a second DELETE are 404 `Secret not found` and the list omits the name.", "file": "F18-settings-and-secrets.md", "line": 43 }, { "feature_id": "F18.secret-name-validation", "description": "names that do not match `^[a-zA-Z][a-zA-Z0-9_]{0,63}$` are 422 on GET, PUT and DELETE (64 characters are accepted, 65 are not); a PUT without `value` is 422; an unknown valid name is 404.", "file": "F18-settings-and-secrets.md", "line": 44 }, { "feature_id": "F18.encrypted-at-rest", "description": "`settings.json` and `secrets.json` hold Fernet tokens, never the raw values, with mode `0600`.", "file": "F18-settings-and-secrets.md", "line": 45 }, { "feature_id": "F18.secret-scope-agent-profile", "description": "`?agent_profile_id=` limits the list to that agent profile's `secret_refs` (`[]` lists nothing, no `secret_refs` lists everything); an unknown id is 404 `Agent profile not found`.", "file": "F18-settings-and-secrets.md", "line": 46 }, { "feature_id": "F18.sdk-remote-workspace", "description": "the SDK's `RemoteWorkspace.get_secrets()` returns `LookupSecret`s whose URLs resolve to the stored values, and `get_llm()` returns the stored model and key.", "file": "F18-settings-and-secrets.md", "line": 47 }, { "feature_id": "F18.sdk-clone-repos", "description": "with the server secret `github_token` stored, the SDK's `RemoteWorkspace.clone_repos()` reads it from `GET /api/settings/secrets/github_token` and clones with it (a local git remote reachable only through the token-bearing URL), a provider whose token secret is missing (`gitlab_token`) clones anonymously, and `get_repos_context()` names both clones.", "file": "F18-settings-and-secrets.md", "line": 48 }, { "feature_id": "F18.ts-client", "description": "the TypeScript `SettingsClient` reads both schemas, reads settings redacted, `plaintext` and `encrypted`, applies a sparse `updateSettings` that a second read shows, upserts, lists (also with `agentProfileId`), reads as text and deletes a secret, and rejects an unknown secret (404), a bad name (422) and a wrong key (401) with an `HttpError`.", "file": "F18-settings-and-secrets.md", "line": 49 }, { "feature_id": "F18.persist-restart", "description": "settings (plaintext key, model, conversation settings) and secrets read back unchanged after a restart with the same cipher key.", "file": "F18-settings-and-secrets.md", "line": 50 }, { "feature_id": "F18.corrupted-files", "description": "with an unparsable `settings.json`, GET silently returns defaults and PATCH is 409 without touching the file; with an unparsable `secrets.json` the list is empty, GET by name 404, PUT and DELETE 500; restoring the files restores the values.", "file": "F18-settings-and-secrets.md", "line": 51 }, { "feature_id": "F18.no-cipher", "description": "without any cipher key, `X-Expose-Secrets: encrypted` is 503, `plaintext` still works, both files store plaintext, and the secrets routes are open without a key.", "file": "F18-settings-and-secrets.md", "line": 52 }, { "feature_id": "F18.secret-empty-value", "description": "a PUT whose value is empty, blank or the redaction placeholder is refused with 422 and neither wipes an existing value nor leaves a listed but unreadable secret (fixed by #5682).", "file": "F18-settings-and-secrets.md", "line": 53 }, { "feature_id": "F18.dangling-pointers", "description": "PATCH must refuse an `active_agent_profile_id` or `active_meta_profile` that names nothing, as the dedicated activate routes do (404).", "file": "F18-settings-and-secrets.md", "line": 54 }, { "feature_id": "F18.secret-scope-profile-errors", "description": "`GET /api/settings/secrets?agent_profile_id=` naming an agent profile whose file no longer validates, or arriving while the agent-profile store's lock is held elsewhere, must answer as the agent-profile routes do (400 and 503), not with an unhandled 500.", "file": "F18-settings-and-secrets.md", "line": 55 }, { "feature_id": "F18.secret-name-contract", "description": "the published OpenAPI description of the secret routes must name the status an invalid secret name actually gets (it says 400; the code answers 422).", "file": "F18-settings-and-secrets.md", "line": 56 }, { "feature_id": "F18.secret-key-rotation", "description": "after a restart with a new cipher key, a write must not destroy values stored under the old key, so rotating back recovers them.", "file": "F18-settings-and-secrets.md", "line": 57 }, { "feature_id": "F19.auth", "description": "every F19 route answers 401 without or with a wrong session key and stores or spawns nothing.", "file": "F19-mcp.md", "line": 30 }, { "feature_id": "F19.settings-create", "description": "`POST /api/settings/mcp/{settings_key}` stores one server (201, the whole redacted `SettingsResponse`), `GET /api/settings` shows it redacted, `X-Expose-Secrets: plaintext` returns the credentials, the file on disk holds them encrypted, and creating an existing key is 409 without changing it.", "file": "F19-mcp.md", "line": 31 }, { "feature_id": "F19.settings-validation", "description": "creating a stdio server without `command`, a remote server without `url`, an unknown field or transport, or `auth` next to an `Authorization` header is 422 and stores nothing.", "file": "F19-mcp.md", "line": 32 }, { "feature_id": "F19.settings-empty-server", "description": "a server body with neither `command` nor `url` (`{}`) is rejected with 422 instead of being stored.", "file": "F19-mcp.md", "line": 33 }, { "feature_id": "F19.settings-patch", "description": "`PATCH` merges sparsely (`enabled: false` keeps url and headers, a `null` header entry deletes only that entry, `enabled: null` restores `true`); an unknown key is 404, an unknown field or a merge that leaves an http server without `url` is 422.", "file": "F19-mcp.md", "line": 34 }, { "feature_id": "F19.settings-delete", "description": "`DELETE` removes one server and leaves the others and their credentials intact; deleting a missing key is 404.", "file": "F19-mcp.md", "line": 35 }, { "feature_id": "F19.test-stdio", "description": "`POST /api/mcp/test` spawns a stdio server, lists its tools, runs a requested `tool_call` and returns its text, and no stdio child survives the response.", "file": "F19-mcp.md", "line": 36 }, { "feature_id": "F19.test-tool-errors", "description": "a tool that raises, an unadvertised tool name and a tool that outlives the timeout are reported in `tool_result` with `is_error: true` while `ok` stays `true`.", "file": "F19-mcp.md", "line": 37 }, { "feature_id": "F19.test-connect-failures", "description": "a missing executable, a closed port and a command that exits at once answer 200 with `ok: false`, an `error` and an `error_kind`.", "file": "F19-mcp.md", "line": 38 }, { "feature_id": "F19.test-timeout", "description": "a stdio server that never completes the handshake yields `ok: false`, `error_kind: timeout` close to the requested `timeout`, and its process is killed.", "file": "F19-mcp.md", "line": 39 }, { "feature_id": "F19.test-validation", "description": "a missing server, an empty command, an unknown transport, a `timeout` outside (0, 120], top-level `authentication` and `auth` next to an `Authorization` header are 422.", "file": "F19-mcp.md", "line": 40 }, { "feature_id": "F19.test-encrypted-env", "description": "`env` values encrypted by the settings API are decrypted before the stdio server starts, and the child does not inherit the agent server's own keys.", "file": "F19-mcp.md", "line": 41 }, { "feature_id": "F19.test-remote-bearer", "description": "a streamable-HTTP server behind a bearer token connects with `auth.strategy: bearer`, an `Authorization` header (also as `type: shttp`) or `auth.strategy: header`, and a wrong token answers `ok: false`.", "file": "F19-mcp.md", "line": 42 }, { "feature_id": "F19.oauth-start-errors", "description": "OAuth start is 400 for a stdio or non-oauth2 server, reports `ok: false` with a failed job for an unreachable server (whose callback is then 409), and finishes without consent for a server that needs no OAuth.", "file": "F19-mcp.md", "line": 43 }, { "feature_id": "F19.oauth-unknown-job", "description": "status and callback for an unknown job id are 404, and a callback without `callback_url` is 422.", "file": "F19-mcp.md", "line": 44 }, { "feature_id": "F19.oauth-flow", "description": "against a local OAuth-protected MCP server, start returns an `authorization_url`, status reports `authorizing` and `callback_ready`, and submitting the browser's redirect URL completes the job with tools, the tool result and an encrypted `oauth_state`.", "file": "F19-mcp.md", "line": 45 }, { "feature_id": "F19.oauth-callback-replay", "description": "submitting a callback again after the job finished answers the job's status or a client error (4xx), not 500, and the job stays `succeeded`.", "file": "F19-mcp.md", "line": 46 }, { "feature_id": "F19.oauth-callback-bad-port", "description": "a callback URL with a port outside 0-65535 is rejected with 400, not 500.", "file": "F19-mcp.md", "line": 47 }, { "feature_id": "F19.oauth-callback-validation", "description": "on a live job, an https URL, a foreign host, another port or another path is 400 and the job keeps waiting; the redirect URL with `127.0.0.1` instead of `localhost` completes it.", "file": "F19-mcp.md", "line": 48 }, { "feature_id": "F19.oauth-consent-denied", "description": "submitting the redirect a provider sends when the user denies consent (`error=access_denied`) answers `ok: false` and fails the job without tokens.", "file": "F19-mcp.md", "line": 49 }, { "feature_id": "F19.oauth-consent-timeout", "description": "a consent that is not completed within the job's `timeout` fails the job with `error_kind: timeout`.", "file": "F19-mcp.md", "line": 50 }, { "feature_id": "F19.oauth-inline-writeback", "description": "an OAuth server passed inline on a conversation's agent (not in settings) whose access token has expired is refreshed when the conversation connects, keeps its tools, and the rotated tokens are posted with the run's session key to the webhook's `/mcp-oauth-state`, not stored in settings.", "file": "F19-mcp.md", "line": 51 }, { "feature_id": "F19.oauth-stored-state", "description": "the returned `oauth_state` used as `auth.state` connects without consent (test, and start without `authorization_url`), and stored through the settings route it is redacted, encrypted on disk and decrypted to the real token.", "file": "F19-mcp.md", "line": 52 }, { "feature_id": "F19.disabled-server", "description": "a conversation started from the settings exposes the tools of enabled servers (including the OAuth server from its stored tokens) and neither exposes nor spawns a disabled server.", "file": "F19-mcp.md", "line": 53 }, { "feature_id": "F19.conversation-tool", "description": "a model-driven conversation calls the configured stdio server's `qa_echo` tool, the `ObservationEvent` (REST and WebSocket) carries `QA_MCP_ECHO:hi`, and deleting the conversation stops the stdio child.", "file": "F19-mcp.md", "line": 54 }, { "feature_id": "F19.oauth-refresh-writeback", "description": "when a stored OAuth server's access token has expired, the next conversation refreshes it, keeps the server's tools, and writes the rotated token and new expiry back into the settings (encrypted on disk) without posting them to the webhook.", "file": "F19-mcp.md", "line": 55 }, { "feature_id": "F19.conversation-secrets-at-rest", "description": "a conversation started from the settings persists its agent's MCP credentials (a disabled server's `env`, OAuth tokens) encrypted, and no MCP credential this family used appears in plaintext under the conversations directory or in the server log.", "file": "F19-mcp.md", "line": 56 }, { "feature_id": "F19.restart", "description": "stored MCP servers and their credentials survive a restart (plaintext GET and the SDK's `RemoteWorkspace.get_mcp_config()`), while in-memory OAuth jobs are gone (404).", "file": "F19-mcp.md", "line": 57 }, { "feature_id": "F20.auth-required", "description": "every profile route answers 401 without a valid `X-Session-API-Key`, and nothing changes.", "file": "F20-llm-profiles.md", "line": 27 }, { "feature_id": "F20.save", "description": "`POST /api/profiles/{name}` with a DeepSeek model and key answers 201 `Profile '' saved`; the list shows the summary `{name, model, base_url, provider_connection_id, provider_connection_broken, api_key_set: true}` and the active profile, and `GET` by name returns the config.", "file": "F20-llm-profiles.md", "line": 28 }, { "feature_id": "F20.key-encrypted-at-rest", "description": "the profile file `home/.openhands/profiles/.json` is mode `600` with `schema_version` 1 and a Fernet `gAAAAA` token as `api_key`; the plaintext key appears nowhere under the server's home.", "file": "F20-llm-profiles.md", "line": 29 }, { "feature_id": "F20.get-secrets", "description": "`GET /api/profiles/{name}` returns `config.api_key: null` with `api_key_set: true` by default, the raw key with `X-Expose-Secrets: plaintext`, a `gAAAAA` token with `encrypted` (or the legacy `true`), and 400 for any other header value.", "file": "F20-llm-profiles.md", "line": 30 }, { "feature_id": "F20.save-encrypted-roundtrip", "description": "posting back the `gAAAAA` token from an `encrypted` read stores the original key (the server decrypts it before re-encrypting), not the token.", "file": "F20-llm-profiles.md", "line": 31 }, { "feature_id": "F20.save-without-secrets", "description": "`include_secrets: false` stores the placeholder `` instead of the key; both the list and `GET` report `api_key_set: false` and plaintext reads return `null`.", "file": "F20-llm-profiles.md", "line": 32 }, { "feature_id": "F20.not-found", "description": "`GET`, `activate` and `rename` of an unknown name answer 404 `Profile '' not found`, also for a same-name rename.", "file": "F20-llm-profiles.md", "line": 33 }, { "feature_id": "F20.name-rules", "description": "names must match `^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`: a leading `-` is 422 on all six named routes, 64 characters are accepted and 65 are 422; a save or validate body without `llm`, or a save without `llm.model`, is 422 and saves nothing.", "file": "F20-llm-profiles.md", "line": 34 }, { "feature_id": "F20.corrupt-files", "description": "a profile file that is not valid JSON (or not an object) is left out of the list and answers 400 `Failed to load profile ...` on `GET` and `activate`, saving over it repairs it, and an unreadable `settings.json` makes activation 409 without touching the file.", "file": "F20-llm-profiles.md", "line": 35 }, { "feature_id": "F20.store-busy", "description": "while another process holds the profile store's lock, a save waits out the 30-second lock timeout and answers 503 `Profile store is busy. Please retry.` without saving; once the lock is free the same save is 201.", "file": "F20-llm-profiles.md", "line": 36 }, { "feature_id": "F20.store-busy-responsive", "description": "while a profile request waits for the store lock, the server keeps answering other requests (`GET /alive` within 3 seconds).", "file": "F20-llm-profiles.md", "line": 37 }, { "feature_id": "F20.profile-limit", "description": "with 50 profiles stored, creating another is 409 `Profile limit reached (50)` while overwriting an existing one still answers 201.", "file": "F20-llm-profiles.md", "line": 38 }, { "feature_id": "F20.validate-live", "description": "`validate` with a working DeepSeek flash config answers `{valid: true, error: null}`, also when the key is the `gAAAAA` token from an `encrypted` read; the draft is not saved.", "file": "F20-llm-profiles.md", "line": 39 }, { "feature_id": "F20.validate-rejects", "description": "`validate` answers 200 `valid: false` with a typed error for a model the provider does not serve (`LLMBadRequestError`) and for an invalid DeepSeek key, without echoing the key.", "file": "F20-llm-profiles.md", "line": 40 }, { "feature_id": "F20.validate-bad-key-type", "description": "an invalid DeepSeek key (provider HTTP 401) is reported as `LLMAuthenticationError`, like any other provider's 401.", "file": "F20-llm-profiles.md", "line": 41 }, { "feature_id": "F20.validate-provider-errors", "description": "with retries off, a provider 401 is `valid: false` `LLMAuthenticationError`, a 500 or an unreachable `base_url` is `valid: false` `LLMServiceUnavailableError`, and a 429 or a per-attempt timeout is `valid: true` (transient; the timeout verdict arrives after one attempt of the posted `timeout`, long before the hung provider would answer); the provider receives a system message `Reply with one token.`, a user `ping` and `max_tokens: 1`.", "file": "F20-llm-profiles.md", "line": 42 }, { "feature_id": "F20.validate-subscription-login", "description": "`validate` of an `auth_type: subscription` config on a server with no stored OpenAI subscription login answers 200 `valid: false` (`ValueError`, `OpenAI subscription login is required`) at once, not a 500.", "file": "F20-llm-profiles.md", "line": 43 }, { "feature_id": "F20.validate-unreachable-bounded", "description": "`validate` of a config whose `base_url` refuses connections answers `valid: false` within 30 seconds with the default retry policy.", "file": "F20-llm-profiles.md", "line": 44 }, { "feature_id": "F20.validate-hang-bounded", "description": "`validate` of a config whose provider never answers returns a verdict within 30 seconds with the default timeout and retry policy.", "file": "F20-llm-profiles.md", "line": 45 }, { "feature_id": "F20.activate", "description": "`POST /api/profiles/{name}/activate` answers 200 `llm_applied: true`; `GET /api/settings` then has the profile's LLM (model, key, options) in `agent_settings.llm` and `active_profile` set, the list reports the same `active_profile`, and `settings.json` holds the key encrypted.", "file": "F20-llm-profiles.md", "line": 46 }, { "feature_id": "F20.activate-conversation", "description": "a conversation started from the saved settings after an activation runs on the activated profile's model and options and finishes.", "file": "F20-llm-profiles.md", "line": 47 }, { "feature_id": "F20.activate-acp", "description": "with ACP agent settings, activation only moves the pointer: 200 `llm_applied: false`, `active_profile` set, no `agent_settings.llm`.", "file": "F20-llm-profiles.md", "line": 48 }, { "feature_id": "F20.sdk-get-llm", "description": "the SDK's `RemoteWorkspace.get_llm(profile_name=...)` returns the stored model, options and plaintext key with `usage_id` `profile:`, `get_llm()` follows `active_profile`, keyword overrides win, and an unknown name raises `FileNotFoundError`.", "file": "F20-llm-profiles.md", "line": 49 }, { "feature_id": "F20.ts-client", "description": "the TypeScript `ProfilesClient` lists, saves, reads (hidden, `plaintext`, `encrypted`), validates, renames, activates and deletes a profile, and surfaces 404 and 422 as `HttpError`s.", "file": "F20-llm-profiles.md", "line": 50 }, { "feature_id": "F20.activate-dangling-provider", "description": "a profile linked to a provider connection that does not exist is saved without its inline key and `base_url`, is listed as `provider_connection_broken: true`, reads fine, and its activation is 422 without moving the active pointer.", "file": "F20-llm-profiles.md", "line": 51 }, { "feature_id": "F20.activate-linked-provider", "description": "a profile linked to an existing provider connection is saved without its inline key, is listed and read with `api_key_set: true` (the connection's key) and a `null` key, and its activation writes the connection's key into `agent_settings.llm`.", "file": "F20-llm-profiles.md", "line": 52 }, { "feature_id": "F20.sdk-get-llm-linked", "description": "an SDK conversation whose LLM comes from `RemoteWorkspace.get_llm(profile_name=)` runs on the provider connection's key and finishes.", "file": "F20-llm-profiles.md", "line": 53 }, { "feature_id": "F20.no-cipher", "description": "on a server with neither `OH_SECRET_KEY` nor a session key (no cipher), a saved key is stored in plaintext in a `600` file, plaintext reads and activation work, and `X-Expose-Secrets: encrypted` answers 503.", "file": "F20-llm-profiles.md", "line": 54 }, { "feature_id": "F20.persist-restart", "description": "profiles, their encrypted keys, options and the active pointer survive a restart with the same secret key.", "file": "F20-llm-profiles.md", "line": 55 }, { "feature_id": "F20.rename", "description": "`POST /api/profiles/{name}/rename` moves the file and keeps the key; the old name is 404, `active_profile` follows the rename in the list and in settings, and agent profiles whose `llm_profile_ref` cited the old name now cite the new one.", "file": "F20-llm-profiles.md", "line": 56 }, { "feature_id": "F20.rename-errors", "description": "renaming onto an existing name is 409 and changes nothing, a same-name rename is 200 `unchanged (same name)`, and an invalid or missing `new_name` is 422.", "file": "F20-llm-profiles.md", "line": 57 }, { "feature_id": "F20.delete-guarded", "description": "deleting a profile that an agent profile references is 409 naming the referrer, and the profile stays.", "file": "F20-llm-profiles.md", "line": 58 }, { "feature_id": "F20.delete", "description": "`DELETE /api/profiles/{name}` removes the file and the list entry and clears `active_profile` in both views, leaves the applied `agent_settings.llm` in place, and answers 200 again for a name that is already gone.", "file": "F20-llm-profiles.md", "line": 59 }, { "feature_id": "F20.json-alias-guard", "description": "a name with a `.json` suffix addresses the same profile (`GET /api/profiles/.json` reads ``), and deleting through that alias is refused with 409 while an agent profile references ``, as it is for the plain name.", "file": "F20-llm-profiles.md", "line": 60 }, { "feature_id": "F20.secret-key-rotation", "description": "after the server's secret key changes, activating a profile whose stored key can no longer be decrypted is refused instead of silently applying a keyless LLM.", "file": "F20-llm-profiles.md", "line": 61 }, { "feature_id": "F21.auth-required", "description": "every agent-profile route answers 401 without a valid `X-Session-API-Key`, and nothing is written (not even the lazy seed).", "file": "F21-agent-profiles.md", "line": 30 }, { "feature_id": "F21.lazy-seed", "description": "the first list on an empty store seeds one `default` OpenHands profile (revision 0, `llm_profile_ref` = the active LLM profile), points `active_agent_profile_id` at it, and later lists do not seed again.", "file": "F21-agent-profiles.md", "line": 31 }, { "feature_id": "F21.seed-no-ghost-llm", "description": "on a never-configured server the seeded `default` keeps a soft `llm_profile_ref` `default` without writing a keyless `default` LLM profile (materialize reports it unresolved); once the live LLM has a real key, the next seed mirrors it into a keyed `default` LLM profile.", "file": "F21-agent-profiles.md", "line": 32 }, { "feature_id": "F21.seed-acp", "description": "when the live `agent_settings` are ACP, the first list seeds an ACP `default` profile with the live backend and model, no LLM reference, and no LLM profile written.", "file": "F21-agent-profiles.md", "line": 33 }, { "feature_id": "F21.save-create", "description": "`POST /api/agent-profiles/{name}` answers 201 and stores the profile under the path name with a server-minted id (a client `id` and body `name` are ignored) at revision 0; GET, the list summary and the file agree.", "file": "F21-agent-profiles.md", "line": 34 }, { "feature_id": "F21.save-overwrite", "description": "saving an existing name replaces the whole profile (no merge), keeps its `id` and sets `revision` to the previous one plus one, ignoring a client `revision`.", "file": "F21-agent-profiles.md", "line": 35 }, { "feature_id": "F21.acp-kind", "description": "an ACP profile (`agent_kind` `acp`, `acp_server`) saves with its ACP defaults and lists with `llm_profile_ref` null.", "file": "F21-agent-profiles.md", "line": 36 }, { "feature_id": "F21.save-validation", "description": "a missing `llm_profile_ref`, a field of the other kind or an unknown field, an invalid `acp_server` or `agent_kind`, a too-new `schema_version` and a non-object body are 422 with a reason, and nothing is stored.", "file": "F21-agent-profiles.md", "line": 37 }, { "feature_id": "F21.name-rules", "description": "names must match `^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$` on every name route and in `new_name`; 64 characters are accepted, 65, a leading dot or dash and a space are 422; activate's id is at most 128 characters (422).", "file": "F21-agent-profiles.md", "line": 38 }, { "feature_id": "F21.not-found", "description": "GET, rename and stored materialize of an unknown name are 404 `Agent profile '' not found`, activate of an unknown id or of a name is 404, and DELETE of an unknown name is 200.", "file": "F21-agent-profiles.md", "line": 39 }, { "feature_id": "F21.activate", "description": "`POST /api/agent-profiles/{profile_id}/activate` sets only `active_agent_profile_id` (`agent_settings_applied` false): the list and settings show the id and `agent_settings` is unchanged.", "file": "F21-agent-profiles.md", "line": 40 }, { "feature_id": "F21.activate-corrupt-settings", "description": "with an unparsable `settings.json`, activate answers 500 `Failed to activate agent profile` (where LLM and meta-profile activation answer 409) and leaves the file as it was; the same activation works again once the file is restored.", "file": "F21-agent-profiles.md", "line": 41 }, { "feature_id": "F21.rename", "description": "rename moves the profile to the new name with the same `id` and `revision`, keeps the active pointer, frees the old name (404), answers 200 \"unchanged\" for the same name and 409 when the target exists.", "file": "F21-agent-profiles.md", "line": 42 }, { "feature_id": "F21.delete", "description": "DELETE removes the profile file; deleting another profile leaves the pointer alone, deleting the active one clears it (and a non-empty store is not reseeded).", "file": "F21-agent-profiles.md", "line": 43 }, { "feature_id": "F21.delete-active-acp-reset", "description": "deleting the active profile while `agent_settings` are ACP resets them to OpenHands defaults and keeps the server's `mcp_config`.", "file": "F21-agent-profiles.md", "line": 44 }, { "feature_id": "F21.materialize-valid", "description": "materializing a stored profile whose references resolve answers `valid` true with `llm_profile_resolved`, `llm_api_key_set`, the resolved MCP keys and skills, the secret allow-list and `resolved_settings` with the key masked (the raw key appears nowhere), and persists nothing.", "file": "F21-agent-profiles.md", "line": 45 }, { "feature_id": "F21.materialize-dangling", "description": "dangling LLM, MCP and meta-profile references and a tool the server cannot run give 200 `valid` false with every problem in `errors`, the dangling names in their fields, and no `resolved_settings`.", "file": "F21-agent-profiles.md", "line": 46 }, { "feature_id": "F21.materialize-draft", "description": "a draft body is previewed under the path name without being saved: an ACP draft reports its credential secret names, `disabled_skills` removes a skill from `resolved_skills`, `mcp_server_refs` null exposes every configured MCP server and `[]` none, and an invalid draft or an extra top-level key is 422.", "file": "F21-agent-profiles.md", "line": 47 }, { "feature_id": "F21.limit", "description": "the store holds at most 50 profiles: creating a 51st is 409 and writes nothing, overwriting one at the cap still works.", "file": "F21-agent-profiles.md", "line": 48 }, { "feature_id": "F21.corrupt-file", "description": "a profile file that is not valid JSON is skipped by the list, GET, stored materialize and rename of it are 400 with the parse error, and DELETE still removes it.", "file": "F21-agent-profiles.md", "line": 49 }, { "feature_id": "F21.conversation-start", "description": "a conversation started with `agent_profile_id` runs the profile's LLM profile and tool selection (not the live settings'), records `launched_agent_profile` with the id and the revision it launched with, and its stats and events name the profile's model after a run.", "file": "F21-agent-profiles.md", "line": 50 }, { "feature_id": "F21.secret-scope", "description": "a profile's `secret_refs` keep only the allowed names of the secrets a conversation is started with, without an error, and the dropped name is not persisted with the conversation; a profile without `secret_refs` keeps them all.", "file": "F21-agent-profiles.md", "line": 51 }, { "feature_id": "F21.launch-additions", "description": "a conversation started with `agent_profile_id` and `agent_launch_additions.system_message_suffix_append` gets the text appended to the profile's `system_message_suffix` in its agent and its `SystemPromptEvent`, the additions are not stored in `meta.json`, and the stored profile keeps its revision; an unknown key or a value over 32768 characters is 422 and creates nothing.", "file": "F21-agent-profiles.md", "line": 52 }, { "feature_id": "F21.ts-client", "description": "the TypeScript `AgentProfilesClient` lists the seed, saves, reads, renames, activates (by id) and deletes a profile, previews a draft and the stored profile with `materializeAgentProfile`, `AgentServerClient.agentProfiles` reads the same store, and an unknown or invalid name rejects with a 404 or 422 `HttpError`.", "file": "F21-agent-profiles.md", "line": 53 }, { "feature_id": "F21.restart", "description": "profiles, their ids and revisions, and the active pointer survive a restart.", "file": "F21-agent-profiles.md", "line": 54 }, { "feature_id": "F21.acp-skill-sourcing", "description": "materializing an ACP profile injects no skills under the default `acp_skill_sourcing` `native`, and the discovered catalog (with `qa-f21-skill`) once the server runs with `OH_ACP_SKILL_SOURCING=openhands_managed` (what the shipped image sets); an OpenHands profile gets the catalog either way.", "file": "F21-agent-profiles.md", "line": 55 }, { "feature_id": "F21.store-busy", "description": "while another process holds the profile store lock, a profile request answers 503 `Profile store is busy` after the 30-second lock timeout, and works again once the lock is free.", "file": "F21-agent-profiles.md", "line": 56 }, { "feature_id": "F21.launch-store-busy", "description": "while another process holds the profile store lock, `POST ` with `agent_profile_id` answers 503 `Agent profile store is busy` (retryable) after the 30-second lock timeout and creates nothing, and `/alive` keeps answering meanwhile (the launch resolves the profile off the event loop).", "file": "F21-agent-profiles.md", "line": 57 }, { "feature_id": "F21.busy-blocks-server", "description": "waiting for the profile store lock must not stall the rest of the server; `/alive` keeps answering promptly.", "file": "F21-agent-profiles.md", "line": 58 }, { "feature_id": "F21.dangling-llm-report", "description": "how materialize reports a dangling LLM reference must match what the server's seed code documents: the documented `dangling_llm_profile_ref` field (the one the launch's 422 carries) is in the materialize body, or the documentation stops promising it.", "file": "F21-agent-profiles.md", "line": 59 }, { "feature_id": "F21.ts-payload-types", "description": "the TypeScript client's exported `AgentProfileSummary`, `AgentProfileDiagnostics`, `OpenHandsAgentProfile` and `ACPAgentProfile` types accept the payloads the server sends, as its generated contract does: every field the server sends is declared and every required field is sent.", "file": "F21-agent-profiles.md", "line": 60 }, { "feature_id": "F21.json-suffix-identity", "description": "saving a name that ends in `.json` twice must keep one `id` and bump `revision`, like any other name, and deleting it while active must clear the pointer (or the name must be refused with 422 and nothing stored).", "file": "F21-agent-profiles.md", "line": 61 }, { "feature_id": "F22.auth-required", "description": "every meta-profile route answers 401 without a valid `X-Session-API-Key`, and nothing is stored.", "file": "F22-meta-profiles.md", "line": 29 }, { "feature_id": "F22.save-classes", "description": "`POST /api/meta-profiles/{name}` with `classifier_model` and `classes` answers 201 `Meta-profile '' saved`; the list shows `{name, classifier_model, num_classes}`, `GET` returns the full config, and the file `meta-profiles/.json` is mode `600`.", "file": "F22-meta-profiles.md", "line": 30 }, { "feature_id": "F22.save-direct", "description": "a direct-routing body (`prompt_template` with `{{ instance_text }}`, `model_table`, no `classes`) is saved; the list reports `num_classes: 0` and `GET` returns the template and table.", "file": "F22-meta-profiles.md", "line": 31 }, { "feature_id": "F22.validation", "description": "a template without `{{ instance_text }}`, a template together with `classes`, a missing `classifier_model` and a class without `model` are 422 and save nothing.", "file": "F22-meta-profiles.md", "line": 32 }, { "feature_id": "F22.empty-router", "description": "a body with neither `classes` nor `prompt_template` (a router that can never route) is refused with 422.", "file": "F22-meta-profiles.md", "line": 33 }, { "feature_id": "F22.overwrite", "description": "saving an existing name replaces the whole config (here classes mode becomes direct mode) without adding a list entry.", "file": "F22-meta-profiles.md", "line": 34 }, { "feature_id": "F22.name-rules", "description": "names must match `^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`: a leading `-` is 422 on all four named routes, 64 characters are accepted and 65 are 422.", "file": "F22-meta-profiles.md", "line": 35 }, { "feature_id": "F22.not-found", "description": "`GET` and `activate` of an unknown name answer 404 `Meta-profile '' not found` and the active pointer stays; `DELETE` of an unknown name is 200 (idempotent).", "file": "F22-meta-profiles.md", "line": 36 }, { "feature_id": "F22.activate", "description": "`activate` answers 200 `Meta-profile '' activated`; the list's `active_meta_profile` and `GET /api/settings` (`active_meta_profile`, `agent_settings.active_meta_profile`, `enable_classify_and_switch_llm_tool: true`, inline `meta_profile`) follow, the agent's LLM is unchanged, and `settings.json` stores it.", "file": "F22-meta-profiles.md", "line": 37 }, { "feature_id": "F22.route-classes", "description": "a conversation started from the settings carries the `ClassifyAndSwitchLLMTool` spec; when the agent calls `route_task_to_model`, the classifier profile picks the class, the observation names the class, profile and model, the conversation switches to `deepseek-pro`, both LLMs' usage is metered, and the events socket shows the switch live.", "file": "F22-meta-profiles.md", "line": 38 }, { "feature_id": "F22.route-direct", "description": "with a direct-routing meta-profile active, the classifier's JSON answer switches the conversation to the named profile (`chosen_class` `model: deepseek-pro`).", "file": "F22-meta-profiles.md", "line": 39 }, { "feature_id": "F22.route-missing-profile", "description": "a meta-profile whose `classifier_model` names no saved LLM profile is accepted at save time; overwriting the active one takes effect without re-activation (the store wins over the inline copy), and the tool then answers an error observation naming the missing profile while the conversation keeps its LLM.", "file": "F22-meta-profiles.md", "line": 40 }, { "feature_id": "F22.persist-restart", "description": "meta-profiles, the active pointer, the settings' routing flag and a routed conversation's switched LLM survive a restart.", "file": "F22-meta-profiles.md", "line": 41 }, { "feature_id": "F22.delete", "description": "deleting an inactive meta-profile removes the file and the list entry, keeps the active pointer, and a repeated delete is 200.", "file": "F22-meta-profiles.md", "line": 42 }, { "feature_id": "F22.delete-active-clears", "description": "deleting the active meta-profile clears `active_meta_profile` in the list and settings, disables the routing tool and drops the inline copy; new conversations no longer get the tool.", "file": "F22-meta-profiles.md", "line": 43 }, { "feature_id": "F22.acp-clears-pointer", "description": "switching the settings' agent kind to ACP clears the active pointer; back on OpenHands settings the meta-profile can be activated again.", "file": "F22-meta-profiles.md", "line": 44 }, { "feature_id": "F22.activate-acp", "description": "on ACP agent settings, where no routing tool can be attached, activation is refused with a 4xx instead of answering `activated` while nothing is active.", "file": "F22-meta-profiles.md", "line": 45 }, { "feature_id": "F22.corrupt-files", "description": "a meta-profile file that is not valid JSON or not an object is left out of the list and answers 400 `Failed to load meta-profile ...` on `GET` and `activate`; saving over it repairs it.", "file": "F22-meta-profiles.md", "line": 46 }, { "feature_id": "F22.corrupt-settings", "description": "with an unreadable `settings.json`, activation is refused with 409 (as LLM-profile activation does) and the file is left untouched.", "file": "F22-meta-profiles.md", "line": 47 }, { "feature_id": "F22.store-busy", "description": "while another process holds the meta-profile store's lock, a save and a delete each wait out the 30-second lock timeout and answer 503 `Meta-profile store is busy. Please retry.` without changing anything; once the lock is free the same save is 201.", "file": "F22-meta-profiles.md", "line": 48 }, { "feature_id": "F22.store-busy-responsive", "description": "while a meta-profile save waits for the store lock, the server keeps answering other requests (`GET /alive` within 3 seconds).", "file": "F22-meta-profiles.md", "line": 49 }, { "feature_id": "F22.limit", "description": "with 50 meta-profiles stored, creating another is 409 `Meta-profile limit reached (50)` while overwriting an existing one still answers 201.", "file": "F22-meta-profiles.md", "line": 50 }, { "feature_id": "F22.ts-client", "description": "the TypeScript `MetaProfilesClient` saves, reads, lists, activates and deletes a meta-profile and surfaces 404 and 422 as `HttpError`s.", "file": "F22-meta-profiles.md", "line": 51 }, { "feature_id": "F22.persistence-dir", "description": "with `OH_PERSISTENCE_DIR=~/`, meta-profiles are stored under the expanded directory next to `settings.json`, like every other store.", "file": "F22-meta-profiles.md", "line": 52 }, { "feature_id": "F23.acp-discovery-errors", "description": "`POST /api/acp/models` requires a session key, validates `agent_settings`, and reports a missing local executable as HTTP 200 with `error.code: ACPSpawnError`, no models and no new conversation; `refresh: true` reports the same failure.", "file": "F23-llm-catalog-and-connections.md", "line": 30 }, { "feature_id": "F23.auth-required", "description": "all seven routes answer 401 `{\"detail\": \"Unauthorized\"}` without a valid `X-Session-API-Key`, and nothing is created.", "file": "F23-llm-catalog-and-connections.md", "line": 31 }, { "feature_id": "F23.providers", "description": "`GET /api/llm/providers` returns LiteLLM's provider names, sorted and unique, including `deepseek`, `openai` and `anthropic`.", "file": "F23-llm-catalog-and-connections.md", "line": 32 }, { "feature_id": "F23.models", "description": "`GET /api/llm/models` returns every LiteLLM model, sorted and unique, with no `bedrock`-prefixed id, including `deepseek/deepseek-flash`.", "file": "F23-llm-catalog-and-connections.md", "line": 33 }, { "feature_id": "F23.models-filter", "description": "`?provider=deepseek` returns only DeepSeek models (prefixed and bare verified ids such as `deepseek-chat`), `?provider=openrouter` only `openrouter/` ids (a namespaced verified id like `deepseek/deepseek-chat` stays with DeepSeek), and an unknown provider an empty list.", "file": "F23-llm-catalog-and-connections.md", "line": 34 }, { "feature_id": "F23.models-no-bedrock", "description": "the provider filters list no AWS Bedrock-only model ids (`deepseek.r1-v1:0`, `anthropic.claude-...-v1:0`), as the route promises without AWS credentials.", "file": "F23-llm-catalog-and-connections.md", "line": 35 }, { "feature_id": "F23.verified-models", "description": "`GET /api/llm/models/verified` groups the curated models by provider (`openhands`, `anthropic`, `openai`, `deepseek`, `openrouter`, ...), each list non-empty and free of duplicates.", "file": "F23-llm-catalog-and-connections.md", "line": 36 }, { "feature_id": "F23.catalog-ts-client", "description": "the TypeScript `LLMMetadataClient` returns the same providers, models, filtered models and verified groups as REST, and a wrong key rejects with `HttpError` 401.", "file": "F23-llm-catalog-and-connections.md", "line": 37 }, { "feature_id": "F23.conn-create", "description": "`POST /api/llm/provider-connections` answers 201 with a server-minted 32-hex `id`, the fields, equal `created_at`/`updated_at` and `api_key_set: true`, never the key; `provider` defaults to `custom`; the list shows the same entries in creation order.", "file": "F23-llm-catalog-and-connections.md", "line": 38 }, { "feature_id": "F23.conn-key-encrypted", "description": "the connection file `home/.openhands/provider-connections/provider_connections.json` is mode `600` with `schema_version` 1 and a Fernet `gAAAAA` token as `api_key`; the plaintext key appears in no response and in no file under the server's home.", "file": "F23-llm-catalog-and-connections.md", "line": 39 }, { "feature_id": "F23.no-cipher", "description": "on a server with neither `OH_SECRET_KEY` nor a session key (no cipher), a connection's key is stored in plaintext in the `600` file, the list still never returns it, and activating a linked profile applies it.", "file": "F23-llm-catalog-and-connections.md", "line": 40 }, { "feature_id": "F23.conn-create-validation", "description": "a missing or empty `api_key` or `display_name`, an empty `provider`, a 129-character `display_name`, and any extra field (`extra_headers`, a client-chosen `id`) are 422 and create nothing.", "file": "F23-llm-catalog-and-connections.md", "line": 41 }, { "feature_id": "F23.conn-create-blank-key", "description": "a whitespace-only or redacted-placeholder (``) `api_key` is refused like an empty one instead of creating a keyless connection.", "file": "F23-llm-catalog-and-connections.md", "line": 42 }, { "feature_id": "F23.linked-run", "description": "a profile saved with `provider_connection_id` stores no inline key; activating it writes the connection's key into `agent_settings.llm`, and a conversation started from those settings runs on the connection's DeepSeek key and finishes.", "file": "F23-llm-catalog-and-connections.md", "line": 43 }, { "feature_id": "F23.conn-update", "description": "`PATCH` renames a connection and sets or clears (`null`) its `base_url`, keeping `id`, `created_at` and the key; the list shows the change, and the next activation of a linked profile applies the connection's `base_url` (and clears it).", "file": "F23-llm-catalog-and-connections.md", "line": 44 }, { "feature_id": "F23.conn-update-validation", "description": "an empty body, `api_key: null` (\"cannot be cleared\"), a `null` `display_name` or `provider`, an empty `display_name` and extra fields are 422, an unknown id is 404, and none of them changes the stored connections.", "file": "F23-llm-catalog-and-connections.md", "line": 45 }, { "feature_id": "F23.conn-patch-blank-key", "description": "a `PATCH` whose `api_key` is empty, whitespace-only or the redacted placeholder never replaces the stored key: the next activation of a linked profile still applies the previous key.", "file": "F23-llm-catalog-and-connections.md", "line": 46 }, { "feature_id": "F23.conn-rotate-key", "description": "rotating `api_key` leaves already-activated settings on the old key; the next activation (`PATCH /api/settings` `active_profile` or `POST /api/profiles/{name}/activate`) applies the new key, which is what the provider then receives, and rotating back makes conversations finish again.", "file": "F23-llm-catalog-and-connections.md", "line": 47 }, { "feature_id": "F23.validate-linked-draft", "description": "`POST /api/profiles/{name}/validate` of a draft that references a connection checks it with the connection's key, so a connection holding a working DeepSeek key validates `valid: true` like the same key inline.", "file": "F23-llm-catalog-and-connections.md", "line": 48 }, { "feature_id": "F23.persist-restart", "description": "connections (ids, names, timestamps, `api_key_set`) survive a restart unchanged, and a linked profile still resolves the key afterwards.", "file": "F23-llm-catalog-and-connections.md", "line": 49 }, { "feature_id": "F23.strict-decrypt", "description": "after the server's secret key changes, list, create, update and delete answer 400 `api_key is encrypted but cannot be decrypted with the current cipher` and leave the file byte-identical; with the old key back everything works and the key is intact.", "file": "F23-llm-catalog-and-connections.md", "line": 50 }, { "feature_id": "F23.corrupt-file", "description": "a connection file that is not valid JSON, or has a newer `schema_version`, makes the routes answer 400 without rewriting the file; restoring it restores the connections.", "file": "F23-llm-catalog-and-connections.md", "line": 51 }, { "feature_id": "F23.store-busy", "description": "while another process holds the store's lock, a create waits out the 30-second lock timeout and answers 503 `Profile store is busy. Please retry.` without creating anything.", "file": "F23-llm-catalog-and-connections.md", "line": 52 }, { "feature_id": "F23.store-busy-responsive", "description": "while a connection request waits for the store lock, the server keeps answering other requests (`GET /alive` within 3 seconds).", "file": "F23-llm-catalog-and-connections.md", "line": 53 }, { "feature_id": "F23.delete-guard", "description": "deleting a connection that an LLM profile or the active `agent_settings.llm` references is 409 naming the references, and the connection stays.", "file": "F23-llm-catalog-and-connections.md", "line": 54 }, { "feature_id": "F23.conn-delete", "description": "deleting an unreferenced connection answers 200 with its fields and `api_key_set: false`, removes it from the list and the file, and a second delete or a `PATCH` of that id is 404.", "file": "F23-llm-catalog-and-connections.md", "line": 55 }, { "feature_id": "F23.conn-limit", "description": "with 64 connections stored a create is 409 `Provider connection limit reached (64). Delete one before adding another.`; deleting one frees the slot.", "file": "F23-llm-catalog-and-connections.md", "line": 56 }, { "feature_id": "F24.models", "description": "`GET /api/llm/subscription/openai/models` answers `vendor: openai` and the SDK's subscription allowlist (`OPENAI_CODEX_MODELS`), sorted.", "file": "F24-openai-subscription.md", "line": 26 }, { "feature_id": "F24.status-disconnected", "description": "with no stored login, status answers 200 `connected: false` with `account_email` and `expires_at` null, without network, and tightens the `auth/` directory to mode 700 without creating a credentials file.", "file": "F24-openai-subscription.md", "line": 27 }, { "feature_id": "F24.status-connected", "description": "a stored, unexpired login shows `connected: true` with its `expires_at`; neither token appears in the response and `account_email` stays null.", "file": "F24-openai-subscription.md", "line": 28 }, { "feature_id": "F24.auth-required", "description": "all five routes answer 401 without a key or with a wrong key, and a refused logout leaves the stored login in place.", "file": "F24-openai-subscription.md", "line": 29 }, { "feature_id": "F24.status-restart", "description": "a stored login survives a server restart.", "file": "F24-openai-subscription.md", "line": 30 }, { "feature_id": "F24.logout", "description": "logout deletes the stored credentials file and answers `connected: false`; status agrees, and a second logout with nothing stored is still 200.", "file": "F24-openai-subscription.md", "line": 31 }, { "feature_id": "F24.status-refresh-rejected", "description": "a login that expires within 60 seconds makes status try a token refresh at `auth.openai.com`; when OpenAI rejects the refresh token, status answers 200 `connected: false` and keeps the file unchanged.", "file": "F24-openai-subscription.md", "line": 32 }, { "feature_id": "F24.status-invalid-file", "description": "a credentials file that is not valid JSON or misses fields reads as `connected: false`, and status deletes it.", "file": "F24-openai-subscription.md", "line": 33 }, { "feature_id": "F24.device-poll-invalid", "description": "a poll without a body, without `device_code` or with a non-string one answers 422.", "file": "F24-openai-subscription.md", "line": 34 }, { "feature_id": "F24.device-poll-unknown", "description": "polling a token the server never issued answers 404 `Subscription device login not found or expired`.", "file": "F24-openai-subscription.md", "line": 35 }, { "feature_id": "F24.device-start", "description": "`device/start` answers 200 with an opaque 43-character `device_code`, a `user_code`, `verification_uri` `https://auth.openai.com/codex/device`, `verification_uri_complete` null, `expires_at` 15 minutes ahead and `interval_seconds` of at least 1, and stores nothing on disk.", "file": "F24-openai-subscription.md", "line": 36 }, { "feature_id": "F24.device-poll-pending", "description": "polling a started login before anyone approves it answers 200 `connected: false`, the token stays usable for the next poll, and no credentials are written.", "file": "F24-openai-subscription.md", "line": 37 }, { "feature_id": "F24.logout-cancels-pending", "description": "after logout, polling a token issued before it answers 404.", "file": "F24-openai-subscription.md", "line": 38 }, { "feature_id": "F24.pending-lost-on-restart", "description": "pending logins live in server memory, so after a restart a token that was pending answers 404.", "file": "F24-openai-subscription.md", "line": 39 }, { "feature_id": "F24.ts-client", "description": "the TypeScript `LLMMetadataClient` subscription methods list models, read status, start, poll and log out, and surface 404 and 401 as `HttpError` statuses.", "file": "F24-openai-subscription.md", "line": 40 }, { "feature_id": "F24.offline-safe", "description": "with no route to `auth.openai.com`, models, status without an expiring login, unknown-token polls and logout still answer normally.", "file": "F24-openai-subscription.md", "line": 41 }, { "feature_id": "F24.device-poll-outage", "description": "a poll whose call to OpenAI fails (the network drops after `device/start`) answers a 5xx but keeps the login pending: once OpenAI is reachable again, the same token polls 200 `connected: false`.", "file": "F24-openai-subscription.md", "line": 42 }, { "feature_id": "F24.status-offline", "description": "with an expired login and no route to `auth.openai.com`, status answers 200 `connected: false`, as it does for a rejected refresh.", "file": "F24-openai-subscription.md", "line": 43 }, { "feature_id": "F24.device-start-offline", "description": "with no route to `auth.openai.com`, `device/start` and a poll of a login started before the outage answer a gateway error (502, 503 or 504), not an unhandled 500.", "file": "F24-openai-subscription.md", "line": 44 }, { "feature_id": "F24.login-complete", "description": "after a human approves the user code with a ChatGPT account, a poll answers `connected: true` with `expires_at`, writes `auth/openai_oauth.json` with mode 600, consumes the token (the next poll is 404), and status shows the login.", "file": "F24-openai-subscription.md", "line": 45 }, { "feature_id": "F25.auth", "description": "every skills route answers 401 without a valid session key, and nothing is installed.", "file": "F25-skills.md", "line": 28 }, { "feature_id": "F25.public-repo", "description": "against the real public extensions repository (network), sync clones it into the public cache and `POST /api/skills` with `load_public` lists skills loaded from that clone.", "file": "F25-skills.md", "line": 29 }, { "feature_id": "F25.marketplace-public", "description": "against the real public extensions repository, `GET /api/skills/marketplace` lists its installable entries; today it returns an empty catalog because the repository has `.plugin/marketplace.json` but no `marketplaces/default.json`.", "file": "F25-skills.md", "line": 30 }, { "feature_id": "F25.catalog-project", "description": "`POST /api/skills` offline with a `project_dir` lists the project's AgentSkills (`agentskills`), legacy trigger skill (`knowledge`) and `AGENTS.md` (`repo`), counts them in `sources.project`, and silently skips a SKILL.md with an invalid name.", "file": "F25-skills.md", "line": 31 }, { "feature_id": "F25.catalog-user-precedence", "description": "user skills from `~/.agents/skills` appear only with `load_user`, and a project skill overrides a user skill of the same name.", "file": "F25-skills.md", "line": 32 }, { "feature_id": "F25.catalog-server-sources", "description": "`sandbox_config` URLs named `WORKER_*` become one always-on `work_hosts` skill, and `org_configs` repositories are cloned from a git URL (an unreachable one contributes nothing).", "file": "F25-skills.md", "line": 33 }, { "feature_id": "F25.catalog-validation", "description": "a wrong type, an incomplete `org_configs` entry or a traversing marketplace `repo_path` is 422.", "file": "F25-skills.md", "line": 34 }, { "feature_id": "F25.install-local", "description": "installing a local skill directory copies it into the installed store, records metadata (`resolved_ref` null), and the skill appears in the installed list, the single view, `.installed.json` and the user tier of `POST /api/skills`.", "file": "F25-skills.md", "line": 35 }, { "feature_id": "F25.enable-disable", "description": "`PATCH {\"enabled\": false}` keeps the files but hides the skill from the catalog, `{\"enabled\": true}` restores it, and a body without `enabled` is 422.", "file": "F25-skills.md", "line": 36 }, { "feature_id": "F25.install-conflict", "description": "re-installing an installed skill is 409 and changes nothing; `force: true` reinstalls it and keeps the disabled flag.", "file": "F25-skills.md", "line": 37 }, { "feature_id": "F25.install-errors", "description": "an unfetchable source (missing path, unparsable source, escaping `repo_path`) is 400, an invalid skill (no SKILL.md, name mismatch, invalid directory name) is 422, an empty source is 422, and nothing is installed.", "file": "F25-skills.md", "line": 38 }, { "feature_id": "F25.install-bad-frontmatter", "description": "a SKILL.md with malformed YAML frontmatter is rejected as an invalid skill (422); today it is a 500.", "file": "F25-skills.md", "line": 39 }, { "feature_id": "F25.install-git", "description": "a `file://` git source with `repo_path` installs offline and records the source, `repo_path` and the resolved commit SHA.", "file": "F25-skills.md", "line": 40 }, { "feature_id": "F25.sdk-load-skills", "description": "`RemoteWorkspace.load_skills_from_agent_server()` turns the merged catalog of its `working_dir` into SDK `Skill` objects (project, user and enabled installed skills, keyword triggers kept) and an `AgentContext` that does not reload public skills.", "file": "F25-skills.md", "line": 41 }, { "feature_id": "F25.install-github", "description": "the working GitHub form (`github:` plus `repo_path`) installs a public skill from github.com at a full commit SHA, and uninstalling it removes it.", "file": "F25-skills.md", "line": 42 }, { "feature_id": "F25.install-documented-sources", "description": "every source form the OpenAPI description of `InstallSkillRequest.source` advertises installs; today both GitHub examples are 400 while `github:owner/repo` plus `repo_path` works.", "file": "F25-skills.md", "line": 43 }, { "feature_id": "F25.refresh", "description": "refresh re-fetches from the recorded source: a new commit in the git source updates `resolved_ref` and the installed files, an edited local source updates the description, and the enabled flag is preserved.", "file": "F25-skills.md", "line": 44 }, { "feature_id": "F25.install-ref", "description": "installing a git source with `ref` pins that commit (`resolved_ref`, the installed files, `requested_ref` in `.installed.json`); refresh moves it to the source's latest commit and drops `requested_ref`.", "file": "F25-skills.md", "line": 45 }, { "feature_id": "F25.install-unknown-ref", "description": "a `ref` the git source does not have is 400 and leaves the installation unchanged; today that holds only for a first clone, while a source whose clone is cached installs the cached HEAD with 200.", "file": "F25-skills.md", "line": 46 }, { "feature_id": "F25.refresh-unfetchable", "description": "refreshing a skill whose recorded source no longer exists is a client error and keeps the installation; today it is a 500.", "file": "F25-skills.md", "line": 47 }, { "feature_id": "F25.name-errors", "description": "an unknown skill name is 404 and a name outside `^[a-z0-9]+(-[a-z0-9]+)*$` is 422 on get, enable/disable, uninstall and refresh.", "file": "F25-skills.md", "line": 48 }, { "feature_id": "F25.self-heal", "description": "a skill directory copied into the store by hand is invisible to the single view until the list call discovers it with source `local`; a tracked directory removed by hand is pruned from the metadata by the list call.", "file": "F25-skills.md", "line": 49 }, { "feature_id": "F25.sync", "description": "`POST /api/skills/sync` updates the cached public repository from its origin and clears the 60 s public-skills cache, so a skill committed upstream is listed immediately (translated: a local git origin stands in for GitHub).", "file": "F25-skills.md", "line": 50 }, { "feature_id": "F25.sync-offline", "description": "when the public repository cannot be fetched, sync reports `status: error`; today it reports success and keeps serving the stale clone.", "file": "F25-skills.md", "line": 51 }, { "feature_id": "F25.catalog-registered-marketplace", "description": "a registered marketplace with `auto_load` adds its standalone skills and plugin skills (and plugin commands) to the catalog, a name list selects some, and `auto_load: false` or `load_public: false` adds none.", "file": "F25-skills.md", "line": 52 }, { "feature_id": "F25.conversation-activation", "description": "in a real DeepSeek conversation that loads user skills, a trigger word activates an enabled installed skill: `activated_knowledge_skills`, the user `MessageEvent` (REST and WebSocket replay) and the system prompt name it.", "file": "F25-skills.md", "line": 53 }, { "feature_id": "F25.invoke-skill", "description": "an enabled installed AgentSkill without triggers is reachable only through the auto-attached `invoke_skill` tool; when the agent calls it, the observation carries the skill body and the conversation lists the skill in `invoked_skills`, not in `activated_knowledge_skills`.", "file": "F25-skills.md", "line": 54 }, { "feature_id": "F25.restart-persist", "description": "installed skills, their enabled flags and resolved commits survive a server restart and keep their catalog visibility.", "file": "F25-skills.md", "line": 55 }, { "feature_id": "F25.catalog-server-marketplaces", "description": "a `registered_marketplaces` entry in the server config contributes without a request entry, a request entry of the same name replaces it, and one of another name adds to it.", "file": "F25-skills.md", "line": 56 }, { "feature_id": "F25.marketplace-catalog", "description": "the catalog lists the public repository's entries (name, description, absolute source) with `installed: false`, and installing an entry's `source` flips it to `installed: true` immediately (translated: a local stand-in with `marketplaces/default.json`).", "file": "F25-skills.md", "line": 57 }, { "feature_id": "F25.ts-client", "description": "the TypeScript `SkillsClient` installs a skill from a git source, lists, reads, disables and re-enables it (`toggleSkill`, with `getSkills` following the flag), refreshes, syncs, reads the marketplace and uninstalls it; an unknown skill is a 404 `HttpError` and a client without a key gets 401.", "file": "F25-skills.md", "line": 58 }, { "feature_id": "F25.marketplace-stale-cache", "description": "after a sync brings a new entry into the public repository, the catalog lists it; today the catalog keeps a 5-minute cache (including an empty one) that sync does not clear.", "file": "F25-skills.md", "line": 59 }, { "feature_id": "F25.uninstall", "description": "uninstall deletes the directory and the metadata entry, the single view and a second uninstall are 404, the catalog drops the skill and the marketplace entry reads `installed: false` again.", "file": "F25-skills.md", "line": 60 }, { "feature_id": "F26.auth", "description": "every plugins route answers 401 without a valid session key and installs nothing.", "file": "F26-plugins.md", "line": 26 }, { "feature_id": "F26.catalog-project", "description": "`POST /api/plugins` with a `project_dir` lists the plugins under its `.agents/plugins` and `.openhands/plugins` (also from the enclosing git root) with their skills, command skills and files; `load_project: false` or no `project_dir` lists none, and a request without a JSON body is 422.", "file": "F26-plugins.md", "line": 27 }, { "feature_id": "F26.catalog-user-precedence", "description": "`load_user` lists the plugins under `~/.agents/plugins` and `$OH_PERSISTENCE_DIR/plugins`; a project plugin wins over a user plugin of the same name, and a user-directory plugin wins over an enabled installed plugin of the same name.", "file": "F26-plugins.md", "line": 28 }, { "feature_id": "F26.install-local", "description": "installing a local Claude Code layout plugin answers 200 with its metadata, skills and files, copies it into the store, records it in `.installed.json`, and the ambient catalog lists it (never the store itself).", "file": "F26-plugins.md", "line": 29 }, { "feature_id": "F26.install-agent-plugins", "description": "a root `plugin.json` with the Agent Plugins `$schema` installs with its `skills/` and `dev.openhands/commands`; the same manifest without `$schema` is 422.", "file": "F26-plugins.md", "line": 30 }, { "feature_id": "F26.install-git", "description": "installing from a `file://` git source with `repo_path` records the source, `repo_path` and the resolved commit, and copies only that subdirectory.", "file": "F26-plugins.md", "line": 31 }, { "feature_id": "F26.install-repo-root-name", "description": "a git source installed without `repo_path` and without a manifest is named after the repository, not after the server's cache directory.", "file": "F26-plugins.md", "line": 32 }, { "feature_id": "F26.install-conflict", "description": "a second install of the same plugin is 409 and changes nothing; `force: true` reinstalls and keeps the enabled flag.", "file": "F26-plugins.md", "line": 33 }, { "feature_id": "F26.install-errors", "description": "unfetchable sources are 400, invalid plugins 422, malformed bodies 422, and none of them leaves anything in the store.", "file": "F26-plugins.md", "line": 34 }, { "feature_id": "F26.install-github", "description": "the GitHub shorthand `github:owner/repo` with a `repo_path` installs that subdirectory of the public repository with its resolved commit.", "file": "F26-plugins.md", "line": 35 }, { "feature_id": "F26.install-documented-source", "description": "the GitHub source form that the OpenAPI description of `InstallPluginRequest.source` advertises installs the plugin.", "file": "F26-plugins.md", "line": 36 }, { "feature_id": "F26.enable-disable", "description": "`PATCH` with `enabled` toggles the flag, which the single view, the list and `.installed.json` show; a disabled plugin leaves the ambient catalog and comes back when re-enabled.", "file": "F26-plugins.md", "line": 37 }, { "feature_id": "F26.name-errors", "description": "an unknown plugin is 404 and a name outside `^[a-z0-9]+(-[a-z0-9]+)*$` is 422 on get, enable/disable, uninstall and refresh.", "file": "F26-plugins.md", "line": 38 }, { "feature_id": "F26.refresh", "description": "refresh re-fetches from the recorded source (a new commit, an edited local manifest), drops a pinned ref, and keeps the enabled flag.", "file": "F26-plugins.md", "line": 39 }, { "feature_id": "F26.self-heal", "description": "a directory copied into the store is unknown to the single view until a listing adopts it with source `local`; listing also ignores a directory whose manifest name differs and drops a record whose directory vanished.", "file": "F26-plugins.md", "line": 40 }, { "feature_id": "F26.refresh-unfetchable", "description": "refreshing a plugin whose source vanished, whose source now holds an invalid manifest, or one adopted with source `local`, is a client error and keeps the installation.", "file": "F26-plugins.md", "line": 41 }, { "feature_id": "F26.uninstall", "description": "uninstall removes the directory and the record; the single view, the list and the ambient catalog no longer show it, and a second uninstall is 404.", "file": "F26-plugins.md", "line": 42 }, { "feature_id": "F26.conversation-ambient", "description": "a new conversation loads an enabled installed plugin on its first message: its `UserPromptSubmit` hook runs (a `HookExecutionEvent`, also on the events socket), its command skill activates, and its skills show in the conversation.", "file": "F26-plugins.md", "line": 43 }, { "feature_id": "F26.conversation-disabled", "description": "a conversation started after the plugin was disabled loads neither its hook nor its skills, while the user-directory plugins still load.", "file": "F26-plugins.md", "line": 44 }, { "feature_id": "F26.conversation-resume-disabled", "description": "after a restart, an existing conversation drops a since-disabled plugin entirely, its command as well as its hook.", "file": "F26-plugins.md", "line": 45 }, { "feature_id": "F26.restart-persist", "description": "installed plugins, their records and their enabled flags survive a restart.", "file": "F26-plugins.md", "line": 46 }, { "feature_id": "F26.marketplace-local", "description": "against a local stand-in for the extensions repository, `GET /api/plugins/marketplace` lists only `./plugins/` entries (local contents for local entries, attach coordinates for git entries) with a fresh `installed` flag; a repository without a manifest gives an empty catalog that is not cached.", "file": "F26-plugins.md", "line": 47 }, { "feature_id": "F26.marketplace-cache", "description": "a non-empty catalog is served from a per-process cache (a new upstream entry appears only after a restart; the 5-minute TTL itself is not waited out), while `installed` stays fresh.", "file": "F26-plugins.md", "line": 48 }, { "feature_id": "F26.ts-client", "description": "every `PluginsClient` method of the TypeScript client works against the server, and errors surface as `HttpError` with the status and detail.", "file": "F26-plugins.md", "line": 49 }, { "feature_id": "F26.marketplace-public", "description": "against the real `` repository, the catalog lists exactly its `./plugins/` entries, with their contents.", "file": "F26-plugins.md", "line": 50 }, { "feature_id": "F27.auth", "description": "every route answers 401 without a session key or with a wrong one; the slash redirect of `/api/tools` answers before authentication but its target still needs the key.", "file": "F27-hooks-subagents-tools.md", "line": 28 }, { "feature_id": "F27.validation", "description": "a missing body or a wrongly typed field is 422 on `POST /api/hooks` and `POST /api/sub-agents`.", "file": "F27-hooks-subagents-tools.md", "line": 29 }, { "feature_id": "F27.tools-list", "description": "`GET /api/tools/` lists the registered tool names, including the default tools, `browser_tool_set`, the task and planning tools and registrations under a class name (`ClassifyAndSwitchLLMTool`).", "file": "F27-hooks-subagents-tools.md", "line": 30 }, { "feature_id": "F27.tools-redirect", "description": "`GET /api/tools` (no trailing slash) answers 307 to `/api/tools/`, and `/api/tools/catalog/` answers 307 to `/api/tools/catalog`; the CLI does not follow redirects.", "file": "F27-hooks-subagents-tools.md", "line": 31 }, { "feature_id": "F27.catalog", "description": "`GET /api/tools/catalog` describes each offered tool with `user_selectable`, `usable`, `description` and `in_default_set`: the default tools are selectable default-set entries, internal tools are not selectable, and built-ins appear under their tool names.", "file": "F27-hooks-subagents-tools.md", "line": 32 }, { "feature_id": "F27.hooks-project", "description": "the project fixture's `hooks.json` (legacy `{\"hooks\": {...}}` wrapper with PascalCase keys) comes back as snake_case `HookConfig` with defaults filled in.", "file": "F27-hooks-subagents-tools.md", "line": 33 }, { "feature_id": "F27.hooks-formats", "description": "a snake_case `hooks.json` keeps its matcher, timeout and `async` flag, a PascalCase key next to it is normalized, a `prompt` hook keeps its prompt, and a matcher left out becomes `*`.", "file": "F27-hooks-subagents-tools.md", "line": 34 }, { "feature_id": "F27.hooks-absent", "description": "no `project_dir`, a misspelled field (`projectDir`), a missing directory, a directory without `.openhands/hooks.json`, a `hooks.json` only under `.agents/`, and an empty `{}` file all return 200 with `hook_config: null`.", "file": "F27-hooks-subagents-tools.md", "line": 35 }, { "feature_id": "F27.hooks-invalid", "description": "unparsable JSON, an unknown event (PascalCase or snake_case), both spellings of one event, and a command hook without a command return 200 with `hook_config: null` and log `Failed to load hooks`.", "file": "F27-hooks-subagents-tools.md", "line": 36 }, { "feature_id": "F27.hooks-conversation", "description": "the config `POST /api/hooks` returns, passed as `hook_config` to a DeepSeek conversation in that project, runs: a `HookExecutionEvent` with the hook's stdout appears over REST and the WebSocket replay, and the conversation keeps its `hook_config` across a restart.", "file": "F27-hooks-subagents-tools.md", "line": 37 }, { "feature_id": "F27.hooks-not-automatic", "description": "a conversation in the same project started without `hook_config` runs no hook, although the project has `hooks.json` and the agent used the terminal.", "file": "F27-hooks-subagents-tools.md", "line": 38 }, { "feature_id": "F27.hooks-block", "description": "a `PreToolUse` hook that exits 2 stops the agent's terminal call (a `HookExecutionEvent` with `blocked: true`, a `UserRejectObservation` from the hook with its stderr as the reason, no terminal observation, no file) and records the action in `blocked_actions` until the agent consumes it; a `UserPromptSubmit` hook that exits 2 records the queued message in `blocked_messages`, and running finishes at once without a model call.", "file": "F27-hooks-subagents-tools.md", "line": 39 }, { "feature_id": "F27.subagents-builtin", "description": "without project and user agents (switched off, or a `project_dir` that does not exist) the catalog is exactly the four built-ins (`bash-runner`, `code-explorer`, `general-purpose`, `web-researcher`) with `level: builtin` and `is_builtin: true`, `mcp_servers` mirrors `mcp_config`, and `load_builtin: false` empties it.", "file": "F27-hooks-subagents-tools.md", "line": 40 }, { "feature_id": "F27.subagents-project", "description": "project agents from `.agents/agents` and `.openhands/agents` are listed with `level: project`, their `source` file, and every frontmatter field (tools, skills, color, permission mode, iteration limit, `` triggers, unknown keys in `metadata`, the body as `system_prompt`).", "file": "F27-hooks-subagents-tools.md", "line": 41 }, { "feature_id": "F27.subagents-user", "description": "user agents from `~/.agents/agents` and `$OH_PERSISTENCE_DIR/agents` are listed with `level: user` only while `load_user` is on, and `load_project: false` hides the project's agents.", "file": "F27-hooks-subagents-tools.md", "line": 42 }, { "feature_id": "F27.subagents-precedence", "description": "names are unique and the first source wins: a project agent shadows the built-in and the user agent of the same name, and `.agents/agents` shadows `.openhands/agents`.", "file": "F27-hooks-subagents-tools.md", "line": 43 }, { "feature_id": "F27.subagents-invalid", "description": "definitions that do not validate (bad `permission_mode`, `max_iteration_per_run: 0`) are skipped with a logged warning, and `README.md`, subdirectories and non-`.md` files are ignored, while the request still answers 200.", "file": "F27-hooks-subagents-tools.md", "line": 44 }, { "feature_id": "F27.subagents-bad-skill", "description": "a project agent whose `skills` names a skill that does not exist should not break conversations in that workspace; today the catalog lists it as valid and a message to such a conversation is a 500.", "file": "F27-hooks-subagents-tools.md", "line": 45 }, { "feature_id": "F27.catalog-sealed", "description": "a client tool registered by a conversation (`client_tools`) is part of that conversation's agent but never enters the catalog, which is sealed at startup.", "file": "F27-hooks-subagents-tools.md", "line": 46 }, { "feature_id": "F27.client-tool-call", "description": "when the agent calls a client tool, the events socket delivers a `ClientAction_` `ActionEvent` with the model's arguments, the server answers it with the `Tool call dispatched to client.` acknowledgment and the run finishes; the tool is registered again when the conversation resumes after a restart; a duplicate name in one request, a name that collides with `terminal` and a reused name with a different schema are each 422 and create nothing, while the same name with the same schema is accepted.", "file": "F27-hooks-subagents-tools.md", "line": 47 }, { "feature_id": "F27.agent-definitions", "description": "`agent_definitions` in the create request make that sub-agent available to the conversation's task tool, which delegates to it by name (the delegate runs the command, the `TaskObservation` names it); the definition is stored with the conversation and is not listed by `POST /api/sub-agents`.", "file": "F27-hooks-subagents-tools.md", "line": 48 }, { "feature_id": "F27.custom-tool-modules", "description": "with `OH_EXTRA_PYTHON_PATH` naming a directory, `tool_module_qualnames` imports a module from it at create, the agent runs its tool, the conversation reports the mapping and `GET /api/tools/` lists the tool (the sealed catalog does not); after a restart the tool is gone until the conversation is loaded again, and a new message runs it again; an unimportable module is a logged warning and the create still succeeds.", "file": "F27-hooks-subagents-tools.md", "line": 49 }, { "feature_id": "F27.import-modules", "description": "a server started with `--import-modules` (and `OH_EXTRA_PYTHON_PATH`) lists the module's tool in `GET /api/tools/` and offers it in the catalog before any conversation exists; a module that cannot be imported stops the server before it binds its port.", "file": "F27-hooks-subagents-tools.md", "line": 50 }, { "feature_id": "F27.custom-tool-missing-on-resume", "description": "when a conversation's tool module can no longer be imported after a restart, the server should drop the tool with its warning and keep the conversation readable; today its event search and a new message are 500 (`Unknown kind`).", "file": "F27-hooks-subagents-tools.md", "line": 51 }, { "feature_id": "F27.browser-playwright-path", "description": "the browser probe should find a Playwright Chromium in `$PLAYWRIGHT_BROWSERS_PATH`; today it only looks in standard paths, `~/.cache/ms-playwright` and `PATH`, and reports `browser_tool_set` unusable.", "file": "F27-hooks-subagents-tools.md", "line": 52 }, { "feature_id": "F27.browser-probe", "description": "without a Chromium the server can find, the catalog reports `browser_tool_set` unusable and `/server_info` `usable_tools` leaves it out; with a Playwright Chromium in the server's `~/.cache/ms-playwright` and a restart, the catalog reports it usable and `usable_tools` lists it.", "file": "F27-hooks-subagents-tools.md", "line": 53 }, { "feature_id": "F27.browser-disabled-launch", "description": "a conversation started from the saved settings (default tools) gets `browser_tool_set` while the server finds Chromium, and leaves it out after a restart with `enable_browser: false`; a conversation created before keeps its stored tools.", "file": "F27-hooks-subagents-tools.md", "line": 54 }, { "feature_id": "F27.browser-disabled", "description": "with `enable_browser: false` the catalog reports `browser_tool_set` unusable although Chromium is present, while `GET /api/tools/` still lists it.", "file": "F27-hooks-subagents-tools.md", "line": 55 }, { "feature_id": "F27.server-info-browser-disabled", "description": "with `enable_browser: false`, `/server_info` `usable_tools` should agree with the catalog and leave out `browser_tool_set`; today it still lists it.", "file": "F27-hooks-subagents-tools.md", "line": 56 }, { "feature_id": "F27.subagents-browser-disabled", "description": "with `enable_browser: false`, the sub-agents catalog should leave out the browser-only `web-researcher`; today it still lists it.", "file": "F27-hooks-subagents-tools.md", "line": 57 }, { "feature_id": "F27.catalog-docker-runtime", "description": "with `conversation_runtime: docker` the catalog probes nothing in the server process: `browser_tool_set` is usable for the stock image although the server finds no Chromium, and unusable for a `-minimal` image tag or with `conversation_image_has_browser: false`.", "file": "F27-hooks-subagents-tools.md", "line": 58 }, { "feature_id": "F28.install-local", "description": "installing from a local directory answers 200 with the app disabled (even when the body says `\"enabled\": true`), `resolved_ref` null and the validated manifest, copies it under `canvas-extensions/installed//` and records it in `.installed.json`, and the list and GET views agree.", "file": "F28-canvas-extensions.md", "line": 39 }, { "feature_id": "F28.install-git", "description": "a `file://` git URL installs as a git source: `resolved_ref` is the repository's HEAD commit and the clone is cached under `cache/extensions/`.", "file": "F28-canvas-extensions.md", "line": 40 }, { "feature_id": "F28.install-ref", "description": "`ref` pins a git install to that commit (`version` and `resolved_ref` are the pinned commit's, `.installed.json` records `requested_ref`), a forced install without `ref` moves to the branch head, and an unknown ref on a source that was never fetched is 400 and installs nothing.", "file": "F28-canvas-extensions.md", "line": 41 }, { "feature_id": "F28.install-repo-path", "description": "`repo_path` installs a subdirectory of the source and is echoed back; a missing subdirectory or one that escapes the source is 400, and a source root without a manifest is 422.", "file": "F28-canvas-extensions.md", "line": 42 }, { "feature_id": "F28.install-errors", "description": "an unreadable source is 400 (missing path, unparseable string, bad `github:` shorthand, `file://` URL of a non-repository), a missing or empty `source` or an invalid manifest (bad JSON, bad name, escaping or missing entrypoint, wrong `schema_version`) is 422, no key is 401, and nothing is installed.", "file": "F28-canvas-extensions.md", "line": 43 }, { "feature_id": "F28.bundle", "description": "`GET .../bundle` serves the installed entrypoint byte for byte with `Cache-Control: no-cache`, an `ETag` and a JavaScript content type, whether or not the app is enabled.", "file": "F28-canvas-extensions.md", "line": 44 }, { "feature_id": "F28.icon", "description": "`GET .../icon` serves the declared SVG byte for byte as `image/svg+xml` with a sandboxing CSP and `nosniff`; an app without an icon is 404.", "file": "F28-canvas-extensions.md", "line": 45 }, { "feature_id": "F28.enable-disable", "description": "`PATCH .../installed/{name}` toggles `enabled`, and both GET and `.installed.json` show the new value; a non-boolean body is 422.", "file": "F28-canvas-extensions.md", "line": 46 }, { "feature_id": "F28.unknown-name", "description": "an unknown name is 404 on GET, PATCH, DELETE, bundle and icon, a non-kebab-case name is 422 on every route, and the routes need the session key.", "file": "F28-canvas-extensions.md", "line": 47 }, { "feature_id": "F28.list-discovery", "description": "a valid app directory copied into `installed/` by hand is invisible to GET by name until a list call registers it (disabled, `source` `local`), and a list call prunes an entry whose directory was deleted.", "file": "F28-canvas-extensions.md", "line": 48 }, { "feature_id": "F28.broken-install", "description": "an installed app whose manifest stops validating, or whose entrypoint becomes a symlink out of the package, stays listed with `manifest` null; its bundle and icon are 404 and its backend reports `missing`, while PATCH and DELETE still work. An icon that becomes a symlink out of the package is 404 while the manifest and bundle stay valid.", "file": "F28-canvas-extensions.md", "line": 49 }, { "feature_id": "F28.backend-status", "description": "`GET .../backend` reports `missing` for an unknown name, `unsupported` for a browser-only app and `stopped` with a revision for a backend app (`sha256:` of the manifest for a local install, the commit SHA for a git install), never 404.", "file": "F28-canvas-extensions.md", "line": 50 }, { "feature_id": "F28.backend-approval-gate", "description": "starting an unprepared revision, preparing a wrong revision, preparing a tampered archive, preparing an archive with an escaping member and preparing a remote (`url`) artifact that cannot be downloaded are all 409, and none of them writes an approval or extracts or keeps a file.", "file": "F28-canvas-extensions.md", "line": 51 }, { "feature_id": "F28.backend-prepare", "description": "preparing the current revision verifies the checksum, extracts the artifact into a read-only `artifacts///` directory, records the approval (shown as `prepared_revision`), and is idempotent.", "file": "F28-canvas-extensions.md", "line": 52 }, { "feature_id": "F28.backend-start", "description": "starting the prepared revision returns `ready` with a pid and port; the backend answers on `127.0.0.1:`, runs in its own process group from the installed package directory with only `HOME=/home` in its environment; a repeated start keeps the same pid, a different revision is 409, and a forced install or a data deletion is 409 while it runs.", "file": "F28-canvas-extensions.md", "line": 53 }, { "feature_id": "F28.backend-stop", "description": "`POST .../backend/stop` kills the process group (a child the backend forked dies too) and reports `stopped`; the port closes, and a second stop is a harmless 200.", "file": "F28-canvas-extensions.md", "line": 54 }, { "feature_id": "F28.force-reinstall", "description": "installing an already installed app without `force` is 409 and leaves the record untouched; `force: true` replaces the files (new bundle bytes and ETag, new `installed_at`), keeps the previous `enabled` value, and deletes the backend approval so the revision must be prepared again.", "file": "F28-canvas-extensions.md", "line": 55 }, { "feature_id": "F28.backend-logs", "description": "while a backend is `starting` its stdout and stderr lines are readable with `[stdout] `/`[stderr] ` prefixes; `stop` abandons the pending start, the logs stay readable after it, `limit_bytes` bounds the tail and sets `truncated`, values outside 1..262144 are 422, and an unknown app has empty logs.", "file": "F28-canvas-extensions.md", "line": 56 }, { "feature_id": "F28.ts-client", "description": "the TypeScript `CanvasExtensionsClient` takes a backend app through `getBackendStatus`, `prepareBackend`, `startBackend`, `getBackendLogs`, `stopBackend` and `deleteBackendData` with the REST results, rejects refusals as `HttpError` with the server's status and detail, and a client timeout shorter than the backend's health wait rejects `startBackend` while the server keeps the backend `starting`.", "file": "F28-canvas-extensions.md", "line": 57 }, { "feature_id": "F28.backend-start-fails", "description": "a backend whose health check never passes ends `unhealthy` with `backend health check timed out` after `health.timeout_seconds`, one that exits ends `unhealthy` with its exit code, and neither leaves a process behind.", "file": "F28-canvas-extensions.md", "line": 58 }, { "feature_id": "F28.backend-inherit-environment", "description": "a manifest whose backend asks to inherit a variable outside the allow-list (`OH_SECRET_KEY`) or names one twice is refused at install with 422, and a backend that asks for `PATH`, `TZ` and `LANG` runs with exactly `HOME` plus the server's values of those it has, none of the server's secrets.", "file": "F28-canvas-extensions.md", "line": 59 }, { "feature_id": "F28.disable-stops-backend", "description": "disabling an app with a running backend stops its process group, and the backend reports `stopped`.", "file": "F28-canvas-extensions.md", "line": 60 }, { "feature_id": "F28.restart-persistence", "description": "after a graceful restart the installs, their `enabled` values and the backend approval are still there, the backend that was running is gone, and `start` works without a new `prepare`.", "file": "F28-canvas-extensions.md", "line": 61 }, { "feature_id": "F28.uninstall", "description": "`DELETE .../installed/{name}` stops the backend and removes the package and its record, keeps the backend data, approval and artifact, reports the backend `missing`, and a second DELETE is 404; `DELETE .../backend/data` then removes the data directory.", "file": "F28-canvas-extensions.md", "line": 62 }, { "feature_id": "F28.prepare-corrupt-archive", "description": "an artifact that is not a gzip archive, with a matching sha256, is refused with 409 like any other bad artifact.", "file": "F28-canvas-extensions.md", "line": 63 }, { "feature_id": "F28.install-ref-cached", "description": "an unknown `ref` is 400 and leaves the installed app alone even when the source's clone is already cached.", "file": "F28-canvas-extensions.md", "line": 64 }, { "feature_id": "F28.status-probe-latch", "description": "a backend that missed one status health probe (it answered slower than the 1 s probe timeout) is reported `ready` again once it answers its health check.", "file": "F28-canvas-extensions.md", "line": 65 }, { "feature_id": "F28.status-exit-hang", "description": "when a backend's leader process exits while a child it forked still holds its stdout, `GET .../backend` answers promptly with `unhealthy` and `stop` reaps the child.", "file": "F28-canvas-extensions.md", "line": 66 }, { "feature_id": "F28.hard-restart-orphan", "description": "a backend that was running when the server was SIGKILLed does not survive the server's restart.", "file": "F28-canvas-extensions.md", "line": 67 }, { "feature_id": "F29.ingress-advertised", "description": "with `app_backend_public_url` set, `/server_info` reports it as `app_backend_ingress_url` and lists the `canvas_app_backend_bridge_v1` capability.", "file": "F29-app-backend-bridge.md", "line": 41 }, { "feature_id": "F29.ingress-config-errors", "description": "on a server without `app_backend_public_url`, the session routes and the HTTP proxy answer 503 `Canvas App backend ingress is not configured`, the WebSocket handshake is refused with 403, and `/server_info` has neither the URL nor the capability; a value that is not an http(s) URL answers 503 `... is misconfigured`, and a plain-HTTP ingress on a non-loopback host refuses to mint a session (503 `... require HTTPS or a loopback secure context`).", "file": "F29-app-backend-bridge.md", "line": 42 }, { "feature_id": "F29.session-bootstrap", "description": "`POST /app-backends/{name}/session` with the key and a control `Origin` returns `ingress_url`, `expires_at` about 300 s ahead and `iframe_sandbox`, and sets `oh_app_backend_session` with `HttpOnly; Max-Age=300; Path=/app-backends/{name}; SameSite=none; Secure; Partitioned`; a cookie jar replays it to the app's proxy, and the browser's CORS preflight for the bootstrap is allowed for the control origin with credentials.", "file": "F29-app-backend-bridge.md", "line": 43 }, { "feature_id": "F29.session-key-required", "description": "the bootstrap answers 401 with no cookie to a missing or wrong `X-Session-API-Key`, to `Authorization: Bearer` and to an app session cookie.", "file": "F29-app-backend-bridge.md", "line": 44 }, { "feature_id": "F29.session-ingress-host", "description": "a bootstrap or proxy request whose `Host` is not the ingress host answers 421, and `X-Forwarded-Host` does not change that while `trust_forwarded_headers` is off.", "file": "F29-app-backend-bridge.md", "line": 45 }, { "feature_id": "F29.session-forwarded-proto", "description": "with `trust_forwarded_headers` off, a client-supplied `X-Forwarded-Proto: https` does not change the request origin, so a bootstrap on the plain-HTTP ingress still succeeds.", "file": "F29-app-backend-bridge.md", "line": 46 }, { "feature_id": "F29.session-origin-rules", "description": "a missing, foreign or `null` `Origin` gets 403 and the ingress origin itself gets 409, both without a cookie; any `localhost` or `127.0.0.1` origin is accepted.", "file": "F29-app-backend-bridge.md", "line": 47 }, { "feature_id": "F29.session-configured-control-origin", "description": "after a restart with `allow_cors_origins`, `allow_cors_origin_regex` and `DOCKER_HOST_ADDR`, a bootstrap from a listed origin, a regex match and the Docker host address succeeds (its CORS preflight is allowed too), while an unlisted origin, the listed host on another scheme or port, a partial regex match and another address get 403; a configured control origin does not count as the ingress `Origin` for unsafe proxied methods, and clearing the settings refuses those origins again.", "file": "F29-app-backend-bridge.md", "line": 48 }, { "feature_id": "F29.session-backend-not-ready", "description": "bootstrapping or proxying an app whose backend is not ready (unknown app) answers 503 `Canvas App backend is not ready`; an invalid app name answers 422.", "file": "F29-app-backend-bridge.md", "line": 49 }, { "feature_id": "F29.proxy-get", "description": "safe methods (GET, HEAD, OPTIONS) with only the cookie reach the backend with the method, sub-path and query, including `GET /session`; a CORS preflight is answered by the server and never reaches the backend.", "file": "F29-app-backend-bridge.md", "line": 50 }, { "feature_id": "F29.proxy-cookie-required", "description": "proxied requests without a cookie, with a bogus cookie, with another app's cookie or with only the session key answer 401.", "file": "F29-app-backend-bridge.md", "line": 51 }, { "feature_id": "F29.proxy-unsafe-methods", "description": "POST, PUT, PATCH and DELETE with the cookie and the ingress `Origin` reach the backend with their body; without that `Origin` (or with the control origin) they answer 403 before the cookie is checked.", "file": "F29-app-backend-bridge.md", "line": 52 }, { "feature_id": "F29.proxy-root", "description": "`/app-backends/{name}` and `/app-backends/{name}/` reach the backend's `/` for all five methods, and on `/app-backends/{name}/` a `path` query parameter is an ordinary parameter.", "file": "F29-app-backend-bridge.md", "line": 53 }, { "feature_id": "F29.proxy-root-path-query", "description": "on the bare root route (no trailing slash), a `path` query parameter is forwarded as a query parameter to the backend's `/`, like any other parameter.", "file": "F29-app-backend-bridge.md", "line": 54 }, { "feature_id": "F29.proxy-credentials-stripped", "description": "the backend never sees `Cookie`, `Authorization`, `X-Session-API-Key` or the `session_api_key`, `X-Session-API-Key` and `Authorization` query parameters, while other headers and parameters pass.", "file": "F29-app-backend-bridge.md", "line": 55 }, { "feature_id": "F29.proxy-request-framing", "description": "a proxied request keeps its framing: a GET reaches the backend without `Transfer-Encoding`, and a POST with a 13-byte JSON body reaches it with `Content-Length: 13`, as it does when sent to the backend directly.", "file": "F29-app-backend-bridge.md", "line": 56 }, { "feature_id": "F29.proxy-response-filter", "description": "an upstream redirect becomes 502 `Upstream redirect is not allowed` with no `Location`, and an upstream `Set-Cookie` never reaches the client.", "file": "F29-app-backend-bridge.md", "line": 57 }, { "feature_id": "F29.proxy-response-headers", "description": "a proxied response carries a single `Date` and a single `Server` header.", "file": "F29-app-backend-bridge.md", "line": 58 }, { "feature_id": "F29.proxy-path-traversal", "description": "`.`/`..` segments and backslashes, percent-encoded at any depth, answer 400 `Invalid backend path` before any credential check, on HTTP and WebSocket.", "file": "F29-app-backend-bridge.md", "line": 59 }, { "feature_id": "F29.ws-bridge", "description": "a WebSocket with the cookie and the ingress `Origin` is bridged to the backend, which echoes frames and receives no cookie, `Origin`, session key or `session_api_key` parameter; a path the backend does not upgrade (the root route) is accepted and then closed with 1011.", "file": "F29-app-backend-bridge.md", "line": 60 }, { "feature_id": "F29.ws-handshake-rejections", "description": "a WebSocket without the ingress `Origin`, without a valid cookie for this app, or with only the session key is refused at the handshake with 403.", "file": "F29-app-backend-bridge.md", "line": 61 }, { "feature_id": "F29.ws-subprotocol", "description": "a WebSocket subprotocol requested by the client reaches the backend and is negotiated through the bridge.", "file": "F29-app-backend-bridge.md", "line": 62 }, { "feature_id": "F29.session-revoke", "description": "`DELETE /app-backends/{name}/session` answers 204 with a `Max-Age=0` cookie, closes that session's bridged WebSocket and makes its cookie 401, while other sessions of the app keep working; it needs the key and a control origin but no cookie.", "file": "F29-app-backend-bridge.md", "line": 63 }, { "feature_id": "F29.backend-lifecycle-revokes", "description": "stopping or disabling the backend closes bridged WebSockets and makes the bridge answer 503; after the backend starts again on a new port, old cookies get 401 and a new session works. A backend that dies while ready is reported `unhealthy` (exited) by a status poll, after which the bridge answers 503, and once it starts again a session minted before the crash gets 401 because sessions are bound to the backend's port.", "file": "F29-app-backend-bridge.md", "line": 64 }, { "feature_id": "F29.backend-crash-detected", "description": "once a ready backend's process has exited, the bridge answers 503 `Canvas App backend is not ready` without waiting for anyone to poll the backend's status.", "file": "F29-app-backend-bridge.md", "line": 65 }, { "feature_id": "F29.trusted-forwarded-headers", "description": "with `trust_forwarded_headers` on, `X-Forwarded-Host` naming the ingress is accepted from a request whose `Host` is not, and a forwarded `https` that does not match the ingress answers 421; turned off again, the same header is ignored.", "file": "F29-app-backend-bridge.md", "line": 66 }, { "feature_id": "F29.sessions-not-persisted", "description": "after a server restart the backend is stopped (the bridge answers 503), its prepared revision survives so it starts without a new prepare, and cookies minted before the restart get 401.", "file": "F29-app-backend-bridge.md", "line": 67 }, { "feature_id": "F29.session-ttl", "description": "an idle bridged WebSocket is closed when its session expires 300 s after the bootstrap, and its cookie then gets 401.", "file": "F29-app-backend-bridge.md", "line": 68 }, { "feature_id": "F29.uninstall-revokes", "description": "uninstalling the app closes its bridged WebSockets, stops its backend and makes the bridge answer 503 to old cookies and new bootstraps.", "file": "F29-app-backend-bridge.md", "line": 69 }, { "feature_id": "F30.models-list", "description": "`GET /v1/models` lists one `{\"id\": \"openhands_\", \"object\": \"model\", \"created\": 0, \"owned_by\": \"openhands\"}` per saved LLM profile, sorted by id, and follows renames and deletes of profiles at once.", "file": "F30-openai-gateway.md", "line": 23 }, { "feature_id": "F30.auth-bearer-or-header", "description": "with session keys configured, all three routes accept the key as `Authorization: Bearer` or as `X-Session-API-Key`, and answer 401 `Unauthorized` to a missing or wrong key before the body is validated, while `/api/*` still refuses Bearer.", "file": "F30-openai-gateway.md", "line": 24 }, { "feature_id": "F30.auth-open-without-keys", "description": "a server with no session keys serves `/v1` without any credential and with any Bearer value.", "file": "F30-openai-gateway.md", "line": 25 }, { "feature_id": "F30.chat-basic", "description": "`POST /v1/chat/completions` on a real model returns a `chat.completion` whose single choice carries the agent's final answer with `finish_reason` `stop`, echoes the model, reports usage, and names a new finished conversation in `X-OpenHands-ServerConversation-ID`.", "file": "F30-openai-gateway.md", "line": 26 }, { "feature_id": "F30.chat-system-suffix", "description": "`system` and `developer` messages become the new conversation's `agent_context.system_message_suffix` (in the system prompt) and never appear as user messages.", "file": "F30-openai-gateway.md", "line": 27 }, { "feature_id": "F30.chat-latest-user-only", "description": "a new conversation receives only the last user message of `messages[]`; earlier user and assistant turns are dropped and stored nowhere.", "file": "F30-openai-gateway.md", "line": 28 }, { "feature_id": "F30.chat-stream", "description": "`stream: true` answers `text/event-stream` with the conversation header and the frames role, whole content, `finish_reason` `stop`, usage (only with `stream_options.include_usage`) and `[DONE]`; a failing streaming request is a plain JSON error, not an SSE frame.", "file": "F30-openai-gateway.md", "line": 29 }, { "feature_id": "F30.chat-reuse-header", "description": "sending `X-OpenHands-ServerConversation-ID` back appends the new user message to that conversation, reruns it (live on its events socket) and returns the same id.", "file": "F30-openai-gateway.md", "line": 30 }, { "feature_id": "F30.chat-reuse-after-restart", "description": "after a server restart the header still continues the persisted conversation.", "file": "F30-openai-gateway.md", "line": 31 }, { "feature_id": "F30.chat-header-new-id", "description": "a well-formed but unknown id in the header creates the conversation with exactly that id; a malformed id is 422 and creates nothing.", "file": "F30-openai-gateway.md", "line": 32 }, { "feature_id": "F30.chat-reuse-keeps-llm", "description": "on a continued conversation the requested model must still exist (404 otherwise) but is not applied: the conversation keeps its own LLM and system suffix, and the other profile's provider is never called.", "file": "F30-openai-gateway.md", "line": 33 }, { "feature_id": "F30.chat-usage-per-request", "description": "`usage` on a continued conversation reports that request's tokens, not the conversation's running total (reproduces a real bug).", "file": "F30-openai-gateway.md", "line": 34 }, { "feature_id": "F30.chat-reuse-concurrent", "description": "two overlapping requests on one conversation each get the answer to their own turn (reproduces a real bug: the first request returns the second request's answer).", "file": "F30-openai-gateway.md", "line": 35 }, { "feature_id": "F30.chat-agent-tools", "description": "the agent runs its own tools in the server workspace (a file appears there), the answer carries no `tool_calls`, and the client's `tools`/`tool_choice` are ignored.", "file": "F30-openai-gateway.md", "line": 36 }, { "feature_id": "F30.chat-input-errors", "description": "bad chat input fails before any conversation exists: 400 without a user message or with a bad content part, 404 for a model without the `openhands_` prefix or an unknown profile, 400 for an invalid profile name, 422 for schema errors.", "file": "F30-openai-gateway.md", "line": 37 }, { "feature_id": "F30.observability-headers", "description": "`X-OpenHands-Observability-*` headers are stored on the new conversation (chat and responses); invalid values are 422 before anything runs.", "file": "F30-openai-gateway.md", "line": 38 }, { "feature_id": "F30.chat-run-error-500", "description": "a run that ends in `error` answers 500 with the rewritten body (`exception` `500: Agent run ended with status: error`), without the conversation header, and leaves the conversation in `error`.", "file": "F30-openai-gateway.md", "line": 39 }, { "feature_id": "F30.chat-confirmation-409", "description": "with confirmation mode saved in the settings, a new conversation that stops `waiting_for_confirmation` answers 409 at once.", "file": "F30-openai-gateway.md", "line": 40 }, { "feature_id": "F30.chat-reuse-confirmation", "description": "a continued conversation that stops `waiting_for_confirmation` must also answer 409 promptly (reproduces a real bug: it waits out 120 s and answers 504).", "file": "F30-openai-gateway.md", "line": 41 }, { "feature_id": "F30.chat-run-limit-429", "description": "when `max_concurrent_runs` is full a new gateway conversation is refused with 429 and nothing is created; the same request succeeds once a slot is free.", "file": "F30-openai-gateway.md", "line": 42 }, { "feature_id": "F30.chat-reuse-run-limit-429", "description": "on a continued conversation a 429 must also leave nothing behind, so a client retry is safe (reproduces a real bug: each refused try appends the user message again).", "file": "F30-openai-gateway.md", "line": 43 }, { "feature_id": "F30.chat-timeout-504", "description": "a run longer than the fixed 120 s answers 504 `Agent run timed out`, and the run keeps going server-side.", "file": "F30-openai-gateway.md", "line": 44 }, { "feature_id": "F30.responses-basic", "description": "`POST /v1/responses` on a real model returns a completed `response` with one `output_text` message equal to the agent's final answer, echoes `instructions` and `metadata`, and puts the instructions in the system suffix.", "file": "F30-openai-gateway.md", "line": 45 }, { "feature_id": "F30.responses-stateless-replay", "description": "every responses call is a new conversation (the conversation header is ignored); replayed items become one user message wrapped in `` markers, and developer items go to the system suffix.", "file": "F30-openai-gateway.md", "line": 46 }, { "feature_id": "F30.responses-errors", "description": "`stream`, `store: true` and `previous_response_id` are 400 before anything else is checked, bad input is 400 or 422, an unknown model 404, and nothing is created.", "file": "F30-openai-gateway.md", "line": 47 }, { "feature_id": "F30.openai-sdk", "description": "the official OpenAI Python SDK lists models, reads the conversation header through `with_raw_response`, continues the conversation with `extra_headers`, parses a stream and creates a response.", "file": "F30-openai-gateway.md", "line": 48 }, { "feature_id": "F31.lease-claimed", "description": "loading a conversation writes `owner_lease.json` in its directory with generation 1, the server's pid and host, and an expiry about 45 s ahead.", "file": "F31-persistence-and-limits.md", "line": 23 }, { "feature_id": "F31.lease-renewed", "description": "while the server lives, the lease's `expires_at` moves forward within 15 s, with the same owner and generation.", "file": "F31-persistence-and-limits.md", "line": 24 }, { "feature_id": "F31.lease-released-graceful", "description": "a graceful restart deletes the lease of every loaded conversation, catalog reads do not claim it again, and the next load claims generation 1 under the new pid.", "file": "F31-persistence-and-limits.md", "line": 25 }, { "feature_id": "F31.lease-dead-owner-takeover", "description": "after a SIGKILL the lease still names the dead pid; the restarted server takes it over on first load with generation 2 and logs the takeover.", "file": "F31-persistence-and-limits.md", "line": 26 }, { "feature_id": "F31.crash-tool-call", "description": "a conversation SIGKILLed during a tool call is loaded again at startup, comes back `error` with one `AgentErrorEvent` answering the interrupted call and no observation, and the resumed run hands that error to the model.", "file": "F31-persistence-and-limits.md", "line": 27 }, { "feature_id": "F31.state-survives-restarts", "description": "a conversation's title, tags and events, the settings, a stored secret and the LLM profiles read back unchanged after a graceful restart and after a SIGKILL.", "file": "F31-persistence-and-limits.md", "line": 28 }, { "feature_id": "F31.no-plaintext-at-rest", "description": "after a real model run, the provider key, a stored secret and a conversation secret appear nowhere in the run's persisted files or the server log, while the server still hands out the provider key and the stored secret and keeps the conversation secret encrypted in its registry.", "file": "F31-persistence-and-limits.md", "line": 29 }, { "feature_id": "F31.second-instance-catalog", "description": "a second server sharing the conversations directory lists, counts and reads conversations that existed when it started, without claiming their leases.", "file": "F31-persistence-and-limits.md", "line": 30 }, { "feature_id": "F31.second-instance-lease-held", "description": "while the owner lives, the second server cannot load the conversation: event reads, sending a message, `/run` and `agent_final_response` are 404, PATCH is `success: false`, pause, interrupt and DELETE are 400, its events socket closes 4004, and nothing changes.", "file": "F31-persistence-and-limits.md", "line": 31 }, { "feature_id": "F31.second-instance-takeover", "description": "once the owner is SIGKILLed, the second server takes the conversation over at once (generation 2, its pid) with the same events, and the restarted owner is now the one refused.", "file": "F31-persistence-and-limits.md", "line": 32 }, { "feature_id": "F31.takeover-keeps-metadata", "description": "a title the owner stored after the second server started survives the second server's takeover, in its GET and in `meta.json`.", "file": "F31-persistence-and-limits.md", "line": 33 }, { "feature_id": "F31.lease-expiry-fencing", "description": "an owner that stops renewing (frozen, pid alive) loses the lease once it expires, and after it resumes its writes are refused and never reach disk.", "file": "F31-persistence-and-limits.md", "line": 34 }, { "feature_id": "F31.fenced-write-no-trace", "description": "a write refused for lost ownership answers a 4xx and leaves no trace: the stale owner's own GET does not report the refused title.", "file": "F31-persistence-and-limits.md", "line": 35 }, { "feature_id": "F31.lease-disabled", "description": "with `lease_ttl_seconds` 0 no lease file is written and a second server loads and writes the same conversation concurrently.", "file": "F31-persistence-and-limits.md", "line": 36 }, { "feature_id": "F31.idle-eviction", "description": "an idle conversation is unloaded at the first eviction pass after its TTL (lease deleted, `Evicted idle` logged), stays listed with its title and status, and loads again with the same events on the next access.", "file": "F31-persistence-and-limits.md", "line": 37 }, { "feature_id": "F31.eviction-skips-busy", "description": "the same pass keeps a conversation with an open events socket and a running one loaded; once the socket closes and the run is interrupted, the next pass unloads both.", "file": "F31-persistence-and-limits.md", "line": 38 }, { "feature_id": "F31.create-capacity", "description": "with every run slot taken, `POST ` answers 429 and creates nothing, on disk or in the catalog; the same request succeeds once the slot is free.", "file": "F31-persistence-and-limits.md", "line": 39 }, { "feature_id": "F31.config-precedence", "description": "the configuration file is applied and an `OH_*` variable overrides the same field (seen through the lease TTL).", "file": "F31-persistence-and-limits.md", "line": 40 }, { "feature_id": "F31.config-null-override", "description": "`OH__IS_NONE=1` clears a value the configuration file sets, as a non-null `OH_*` value replaces it (seen through `app_backend_public_url` in `/server_info`).", "file": "F31-persistence-and-limits.md", "line": 41 }, { "feature_id": "F31.config-invalid-fails-fast", "description": "a non-numeric `OH_MAX_CONCURRENT_RUNS` or an out-of-range limit in the configuration file stops the server during startup instead of serving with a default.", "file": "F31-persistence-and-limits.md", "line": 42 }, { "feature_id": "F31.config-silent-env", "description": "an `OH_*` boolean other than `1`/`true` is read as false (it overrides a `true` in the configuration file) and a `Literal` variable outside its choices falls back to the default; the server starts either way, while the same invalid choice in the file stops startup.", "file": "F31-persistence-and-limits.md", "line": 43 }, { "feature_id": "F31.tmux-dir-cleanup", "description": "a server that defaulted its own per-process `TMUX_TMPDIR` removes that directory when it stops.", "file": "F31-persistence-and-limits.md", "line": 44 }, { "feature_id": "F31.storage-retention", "description": "with the Docker runtime, `conversation_storage_retention_days` archives a stopped conversation inactive past the limit (events readable, resume and proxied routes 410, runtime directory gone), and `conversation_storage_disk_budget` sheds a stopped workspace's gitignored directories while keeping its files (blocked here: no Docker daemon).", "file": "F31-persistence-and-limits.md", "line": 45 }, { "feature_id": "F32.events-post", "description": "a run's events reach `POST {base_url}/events/` as JSON arrays, through the agent's reply and the `finished` status update.", "file": "F32-webhooks-and-telemetry.md", "line": 30 }, { "feature_id": "F32.events-session-key", "description": "every webhook request carries `Content-Type: application/json` and the server's first session key as `X-Session-API-Key`.", "file": "F32-webhooks-and-telemetry.md", "line": 31 }, { "feature_id": "F32.events-full-state", "description": "each attach of a conversation (its creation, and its first use after a restart) posts a transient `ConversationStateUpdateEvent` with key `full_state`.", "file": "F32-webhooks-and-telemetry.md", "line": 32 }, { "feature_id": "F32.events-no-deltas", "description": "a streaming run's `StreamingDeltaEvent`s reach the events WebSocket but never a webhook.", "file": "F32-webhooks-and-telemetry.md", "line": 33 }, { "feature_id": "F32.conversation-created", "description": "creating a conversation POSTs its `ConversationInfo` to `{base_url}/conversations`.", "file": "F32-webhooks-and-telemetry.md", "line": 34 }, { "feature_id": "F32.conversation-lifecycle", "description": "pausing, interrupting, updating and deleting a conversation each POST its info again (`paused`, `paused`, the new title, `deleting`).", "file": "F32-webhooks-and-telemetry.md", "line": 35 }, { "feature_id": "F32.custom-headers", "description": "a webhook's `headers` and a `base_url` path prefix apply to both event and conversation posts.", "file": "F32-webhooks-and-telemetry.md", "line": 36 }, { "feature_id": "F32.buffer-size", "description": "once `event_buffer_size` events are queued they are posted at once, without waiting for `flush_delay`.", "file": "F32-webhooks-and-telemetry.md", "line": 37 }, { "feature_id": "F32.buffer-stranded", "description": "events that arrive while a batch is in flight are posted as soon as the buffer is full again, not held until `flush_delay` (reproduces a real delay; see Gotchas).", "file": "F32-webhooks-and-telemetry.md", "line": 38 }, { "feature_id": "F32.flush-delay", "description": "a partial buffer is posted `flush_delay` seconds after its first event; later events do not restart the timer.", "file": "F32-webhooks-and-telemetry.md", "line": 39 }, { "feature_id": "F32.webhook-spec-docs", "description": "the `WebhookSpec` field descriptions published in the OpenAPI schema (the item type of `POST /api/init`'s `webhooks`) state the real event path and flush timer (reproduces a documentation bug; see Gotchas).", "file": "F32-webhooks-and-telemetry.md", "line": 40 }, { "feature_id": "F32.flush-on-close", "description": "deleting a conversation posts its queued events before the `DELETE` returns, and a graceful server stop posts every conversation's queue.", "file": "F32-webhooks-and-telemetry.md", "line": 41 }, { "feature_id": "F32.backpressure", "description": "`max_batch_bytes` caps each request (an event larger than it is posted alone), and `max_queue_size` and `max_queue_bytes` drop the oldest queued events with a warning.", "file": "F32-webhooks-and-telemetry.md", "line": 42 }, { "feature_id": "F32.retry", "description": "a receiver that answers 500 gets `num_retries + 1` attempts per event batch and per conversation post.", "file": "F32-webhooks-and-telemetry.md", "line": 43 }, { "feature_id": "F32.requeue", "description": "a batch that exhausted its retries is requeued and sent again on the next flush, not dropped.", "file": "F32-webhooks-and-telemetry.md", "line": 44 }, { "feature_id": "F32.slow-receiver", "description": "a slow webhook receiver must not keep a conversation that reports `finished` from accepting a new run (reproduces a real 409; see Gotchas).", "file": "F32-webhooks-and-telemetry.md", "line": 45 }, { "feature_id": "F32.env-config", "description": "`OH_WEBHOOKS_0_*` environment variables configure a webhook and override the config file's first webhook field by field.", "file": "F32-webhooks-and-telemetry.md", "line": 46 }, { "feature_id": "F32.telemetry-off-by-default", "description": "no exporter, or the `http` exporter without consent, delivers nothing.", "file": "F32-webhooks-and-telemetry.md", "line": 47 }, { "feature_id": "F32.telemetry-lifecycle", "description": "with the `http` exporter and consent, `server_started` arrives at boot and `server_stopped` at a graceful stop, as `{\"schema_version\": 1, \"events\": [...]}` with the configured Bearer token.", "file": "F32-webhooks-and-telemetry.md", "line": 48 }, { "feature_id": "F32.telemetry-conversation", "description": "a run emits `conversation_created` and `conversation_finished` under the request's `user_id`, with a pseudonymous `conversation_ref` and without the prompt or the conversation id.", "file": "F32-webhooks-and-telemetry.md", "line": 49 }, { "feature_id": "F32.telemetry-request-failed", "description": "an unhandled 500 emits `request_failed` with the route template (never the concrete path), the exception class and the response's `error_id`, attributed to `X-OpenHands-Telemetry-Distinct-Id`, without the exception message.", "file": "F32-webhooks-and-telemetry.md", "line": 50 }, { "feature_id": "F32.telemetry-conversation-failed", "description": "a run that ends in `error` emits `conversation_failed` (`terminal_status` `error`) and `conversation_error` with the error code as `error_class`, without the error's detail text.", "file": "F32-webhooks-and-telemetry.md", "line": 51 }, { "feature_id": "F32.telemetry-usage-buckets", "description": "`conversation_finished` reports token and cost buckets from the run's real usage (reproduces a real gap; see Gotchas).", "file": "F32-webhooks-and-telemetry.md", "line": 52 }, { "feature_id": "F32.telemetry-consent-settings", "description": "granting or denying consent with `PATCH /api/settings` `misc_settings_diff` starts or stops delivery immediately, and persisted consent enables telemetry after a restart.", "file": "F32-webhooks-and-telemetry.md", "line": 53 }, { "feature_id": "F32.telemetry-env-override", "description": "`OH_TELEMETRY_CONSENT=denied` with `OH_TELEMETRY_CONSENT_MODE=override` beats a granted setting.", "file": "F32-webhooks-and-telemetry.md", "line": 54 }, { "feature_id": "F32.telemetry-kill-switch", "description": "`DO_NOT_TRACK=1` beats every consent source.", "file": "F32-webhooks-and-telemetry.md", "line": 55 }, { "feature_id": "F32.telemetry-salt-and-kind", "description": "without a salt `conversation_ref` is keyed by `OH_SECRET_KEY` (the same ref across restarts); `OH_TELEMETRY_SALT` re-keys it (again the same across restarts), and `OH_TELEMETRY_DEPLOYMENT_KIND` tags every event.", "file": "F32-webhooks-and-telemetry.md", "line": 56 }, { "feature_id": "F32.telemetry-salt-docs", "description": "the `TelemetrySpec.salt` description published in the OpenAPI schema states the real fallback key (reproduces a documentation bug; see Gotchas).", "file": "F32-webhooks-and-telemetry.md", "line": 57 }, { "feature_id": "F32.deferred-init", "description": "a dormant pod sends nothing until `POST /api/init` delivers `telemetry` and `webhooks`; then `server_started`, conversation telemetry and webhooks flow.", "file": "F32-webhooks-and-telemetry.md", "line": 58 }, { "feature_id": "F32.deferred-init-flag", "description": "events from an initialized deferred pod report `deferred_init: true` (reproduces a real mislabel; see Gotchas).", "file": "F32-webhooks-and-telemetry.md", "line": 59 }, { "feature_id": "F33.workspace-exec", "description": "`Workspace(host=...)` returns a `RemoteWorkspace` whose `execute_command` returns the command's exit code, stdout and stderr, runs in the given `cwd`, gives up after `timeout` with exit `-1` and `timeout_occurred`; `alive` and `get_server_info()` match the server.", "file": "F33-sdk-remote-client.md", "line": 26 }, { "feature_id": "F33.workspace-bad-key", "description": "with a wrong `api_key`, `execute_command` and `file_upload` report a 401 failure instead of raising, creating a conversation raises a 401 `HTTPStatusError`, and the server records no command, file or conversation.", "file": "F33-sdk-remote-client.md", "line": 27 }, { "feature_id": "F33.workspace-background", "description": "`start_command` returns a command id at once, `get_command_output` shows no exit code while it runs, `stop_command` ends it (no later output), and stopping an unknown id is a no-op.", "file": "F33-sdk-remote-client.md", "line": 28 }, { "feature_id": "F33.workspace-files", "description": "`file_upload` (from a path or from bytes) and `file_download` round-trip a binary file byte for byte; a missing remote file or a relative destination reports `success: false` with the server's 404 or 400 and leaves no local file.", "file": "F33-sdk-remote-client.md", "line": 29 }, { "feature_id": "F33.workspace-git", "description": "`git_changes` (a path relative to `working_dir`) and `git_diff` return what the server's git routes return for a repository with a modified and an untracked file.", "file": "F33-sdk-remote-client.md", "line": 30 }, { "feature_id": "F33.workspace-settings", "description": "`get_llm()` resolves the server's active LLM profile with a usable key, `get_llm(profile_name=...)` a named one (unknown names raise `FileNotFoundError`), `get_secrets()` returns `LookupSecret` references without values, and `get_mcp_config()` is empty on a fresh server.", "file": "F33-sdk-remote-client.md", "line": 31 }, { "feature_id": "F33.async-workspace", "description": "`AsyncRemoteWorkspace` runs a command, starts, reads and stops a background command, uploads and downloads a file, reads git changes, a git diff and server info, and closes its client on context exit.", "file": "F33-sdk-remote-client.md", "line": 32 }, { "feature_id": "F33.workspace-runtime-scope", "description": "a `RemoteWorkspace(runtime_conversation_id=...)` routes through `/{id}/...`: commands default to the conversation's workspace and land in its own bash history, paths outside that workspace are refused with 422, and an unknown conversation gives 404.", "file": "F33-sdk-remote-client.md", "line": 33 }, { "feature_id": "F33.workspace-runtime-lifecycle", "description": "`get_runtime_session_key()` and `release_runtime()` hand out a scoped worker key and stop a conversation's runtime on a server with a Docker conversation runtime.", "file": "F33-sdk-remote-client.md", "line": 34 }, { "feature_id": "F33.conversation-create", "description": "`Conversation(agent, workspace=RemoteWorkspace)` returns a `RemoteConversation` that creates the server conversation with the agent, working directory, tags and `max_iteration_per_run`; `send_message` stores the message without running and the agent's key is not echoed back.", "file": "F33-sdk-remote-client.md", "line": 35 }, { "feature_id": "F33.conversation-create-request", "description": "`RemoteConversation.create(workspace, StartConversationRequest(...))` creates the conversation with the requested id from server-built `agent_settings`, tags and iteration limit, and runs its `initial_message`; repeating the same request returns the same conversation and adds no conversation or message.", "file": "F33-sdk-remote-client.md", "line": 36 }, { "feature_id": "F33.conversation-create-profile", "description": "`RemoteConversation.create(workspace, StartConversationRequest(agent_profile_id=...))` builds the agent from a saved agent profile (its LLM profile, tool selection and system message suffix), records the launched profile and runs the `initial_message` to `finished`; `get_secrets(agent_profile_id=...)` returns only the profile's `secret_refs`.", "file": "F33-sdk-remote-client.md", "line": 37 }, { "feature_id": "F33.conversation-run", "description": "`send_message` plus a blocking `run()` returns once the agent finished; the server reports `finished` and the file the agent wrote exists.", "file": "F33-sdk-remote-client.md", "line": 38 }, { "feature_id": "F33.conversation-callbacks", "description": "callbacks receive the run's events live over the events WebSocket, and after `run()` the callback counts and `state.events` match the server's persisted events kind by kind.", "file": "F33-sdk-remote-client.md", "line": 39 }, { "feature_id": "F33.conversation-stats", "description": "`conversation_stats` and `state.execution_status` read back the server's numbers (accumulated cost and prompt tokens of the agent's LLM).", "file": "F33-sdk-remote-client.md", "line": 40 }, { "feature_id": "F33.conversation-stop-hook", "description": "with a `Stop` hook that refuses the first stop (exit 2), a blocking `run()` returns only after the extra agent turn the hook forced: the server is `finished` when `run()` returns, and `state.events` already holds both stop `HookExecutionEvent`s (blocked, then allowed) and the hook's feedback, matching the server kind by kind.", "file": "F33-sdk-remote-client.md", "line": 41 }, { "feature_id": "F33.client-tools", "description": "`Conversation(client_tools=[ClientToolSpec(...)])` delivers the model's call to the callbacks as a typed `ClientAction_` `ActionEvent`, the server acknowledges it with a `ClientToolObservation` and finishes, and a second program that attaches without `client_tools` replays those events.", "file": "F33-sdk-remote-client.md", "line": 42 }, { "feature_id": "F33.conversation-streaming", "description": "with a `stream=True` agent LLM, callbacks receive the run's `StreamingDeltaEvent` frames live and the server stores none of them.", "file": "F33-sdk-remote-client.md", "line": 43 }, { "feature_id": "F33.conversation-token-callbacks", "description": "`Conversation(token_callbacks=[...])` on a remote workspace calls the token callbacks with the run's streamed deltas.", "file": "F33-sdk-remote-client.md", "line": 44 }, { "feature_id": "F33.conversation-streaming-state", "description": "streamed deltas stay out of `state.events`, which keeps matching the server's stored events.", "file": "F33-sdk-remote-client.md", "line": 45 }, { "feature_id": "F33.completion-logs", "description": "an agent LLM with `log_completions=True` gets one log file per model call in its `log_completions_folder`, written by the client from the `LLMCompletionLogEvent`s the server streams and stores (the server writes no file).", "file": "F33-sdk-remote-client.md", "line": 46 }, { "feature_id": "F33.conversation-attach", "description": "a second program reattaches by id with `RemoteConversation.attach()` or `Conversation(conversation_id=...)` and replays the same history without creating anything; `attach()` on an unknown id raises 404 while `Conversation(conversation_id=)` creates that id.", "file": "F33-sdk-remote-client.md", "line": 47 }, { "feature_id": "F33.conversation-attach-large", "description": "attaching to a conversation with more than 100 stored events pages through the whole history: `state.events` holds the server's events in the server's order (plus the one unpersisted socket snapshot), and the first reads of `state.confirmation_policy`, `state.security_analyzer`, `state.hook_config` and `state.agent` equal the server's.", "file": "F33-sdk-remote-client.md", "line": 48 }, { "feature_id": "F33.conversation-state-reread", "description": "reading `state.confirmation_policy`, `state.security_analyzer`, `state.agent` and `is_confirmation_mode_active` a second time gives the same answer as the first.", "file": "F33-sdk-remote-client.md", "line": 49 }, { "feature_id": "F33.conversation-ask-title", "description": "`ask_agent()` answers without adding events, `generate_title()` returns a title within `max_length` without adding events, and `set_title()` changes the server's title.", "file": "F33-sdk-remote-client.md", "line": 50 }, { "feature_id": "F33.conversation-fork", "description": "`fork(title=..., tags=...)` returns a `RemoteConversation` for a new server conversation with the same history, the title and the tags; `fork(agent=...)` is refused client-side.", "file": "F33-sdk-remote-client.md", "line": 51 }, { "feature_id": "F33.conversation-fork-branch", "description": "`fork(from_event_id=..., reset_metrics=False)` returns a fork whose history ends at that event (its HEAD) with the source's stats, and `navigate_to(None)` and `navigate_to(event_id)` move the fork's HEAD without changing its events.", "file": "F33-sdk-remote-client.md", "line": 52 }, { "feature_id": "F33.conversation-plugins-tag", "description": "`Conversation(plugins=[PluginSource(...)])` with inline URL credentials stores the `plugins` tag with the credentials masked, and the token reaches no server state.", "file": "F33-sdk-remote-client.md", "line": 53 }, { "feature_id": "F33.conversation-confirmation", "description": "`set_confirmation_policy(AlwaysConfirm())` makes `run()` return at `waiting_for_confirmation`; `reject_pending_actions()` records a `UserRejectObservation`, never runs the action and leaves the conversation `idle`.", "file": "F33-sdk-remote-client.md", "line": 54 }, { "feature_id": "F33.conversation-pause", "description": "`pause()` from another thread during a blocking `run()` lets the tool step in flight finish, makes `run()` return with `paused` and a `PauseEvent`, and a later `run()` resumes to `finished`.", "file": "F33-sdk-remote-client.md", "line": 55 }, { "feature_id": "F33.conversation-interrupt", "description": "`interrupt()` from another thread while a blocking `run()` waits on a model call makes `run()` return at once without raising, at `paused` with one `InterruptEvent` and no `PauseEvent`, and a later `run()` reaches `finished`.", "file": "F33-sdk-remote-client.md", "line": 56 }, { "feature_id": "F33.conversation-run-timeout", "description": "`run(timeout=...)` raises `ConversationRunError` wrapping a `TimeoutError` on time while the server keeps running, and a second `run()` joins the run in flight (the server's 409 is not an error) and returns at `finished`.", "file": "F33-sdk-remote-client.md", "line": 57 }, { "feature_id": "F33.conversation-run-join-long-tool", "description": "a second `run()` also joins a run whose current tool call lasts longer than 30 s, instead of raising.", "file": "F33-sdk-remote-client.md", "line": 58 }, { "feature_id": "F33.conversation-stuck", "description": "when the server's stuck detector stops a looping agent, `run()` raises `ConversationRunError` (\"got stuck\") at once and the server reports `stuck`.", "file": "F33-sdk-remote-client.md", "line": 59 }, { "feature_id": "F33.conversation-secrets", "description": "`update_secrets()` with a plain value and with a `LookupSecret` from `get_secrets()` makes both available to the agent's commands, masked in observations and not stored in plaintext.", "file": "F33-sdk-remote-client.md", "line": 60 }, { "feature_id": "F33.conversation-run-error", "description": "when the model provider rejects the key, `run()` raises `ConversationRunError` carrying the server's `ConversationErrorEvent` code, and the server reports `error`.", "file": "F33-sdk-remote-client.md", "line": 61 }, { "feature_id": "F33.ws-unavailable", "description": "when the events socket cannot connect, `Conversation(...)` raises `WebSocketConnectionError` after `OPENHANDS_REMOTE_WS_READY_TIMEOUT` seconds (the conversation it created stays on the server); with `OPENHANDS_REMOTE_WS_READY_REQUIRED=false` it continues, and `run()` returns at `finished` through the REST fallback about 30 s after the server finished, with `state.events` matching the server.", "file": "F33-sdk-remote-client.md", "line": 62 }, { "feature_id": "F33.conversation-reconnect", "description": "a live `RemoteConversation` survives a server restart: its events socket reconnects, a new message runs to `finished`, and `state.events` again matches the server.", "file": "F33-sdk-remote-client.md", "line": 63 }, { "feature_id": "F33.conversation-reconnect-callbacks", "description": "an event the server records while a live `RemoteConversation`'s events socket is down reaches its callbacks once the socket is back, not only `state.events`.", "file": "F33-sdk-remote-client.md", "line": 64 }, { "feature_id": "F33.conversation-restart", "description": "after a restart a new program reattaches to a finished conversation, sees the same history, and continues it to `finished`.", "file": "F33-sdk-remote-client.md", "line": 65 }, { "feature_id": "F33.custom-tool-module", "description": "a tool registered in the client process is sent as `tool_module_qualnames`, imported by a server started with `OH_EXTRA_PYTHON_PATH`, and runs on the server when the agent calls it; a client-registered module the server cannot import is only a logged warning; reattaching from a process without the module raises `ImportError` naming the tool.", "file": "F33-sdk-remote-client.md", "line": 66 }, { "feature_id": "F33.custom-tool-module-unimportable", "description": "a conversation whose agent names a tool from a module the server cannot import fails with a client error (a 4xx, or a recorded conversation error), never with a 500.", "file": "F33-sdk-remote-client.md", "line": 67 }, { "feature_id": "F33.conversation-close", "description": "`close()` keeps the server conversation when `delete_on_close=False` and deletes it when the `Conversation(...)` factory's default `delete_on_close=True` applies.", "file": "F33-sdk-remote-client.md", "line": 68 }, { "feature_id": "F34.server-client", "description": "`ServerClient` (through `ConversationManager.server` and `AgentServerClient.server`) reads `/alive`, `/health`, `/ready`, `/server_info` and `/` with the server's own values, the probes without a key; the version-gated clients (`HooksClient`, `WorkspacesClient`, `MCPClient`) read `/server_info` once per transport, accept this server and return its data.", "file": "F34-typescript-client.md", "line": 28 }, { "feature_id": "F34.http-errors", "description": "a missing or wrong key rejects with an `HttpError` (`status` 401, `detail` `Unauthorized`, recognized by `isHttpError`), a 404 and a 422 carry the server's `detail` and `validationErrors`, while an unreachable host and a client timeout reject with plain errors.", "file": "F34-typescript-client.md", "line": 29 }, { "feature_id": "F34.workspace-exec", "description": "`RemoteWorkspace.executeCommand` returns stdout, stderr and the exit code of a command run in the given `cwd`, and exit `-1` once the command's own timeout kills it.", "file": "F34-typescript-client.md", "line": 30 }, { "feature_id": "F34.workspace-files", "description": "`RemoteWorkspace.uploadText`/`fileUpload` and `downloadAsText`, and `FileClient.downloadFile`/`uploadTextFile`, round-trip files (bytes exact through `FileClient`); a missing file, a relative path and a directory reject with the server's 404 and 400.", "file": "F34-typescript-client.md", "line": 31 }, { "feature_id": "F34.workspace-binary-download", "description": "`RemoteWorkspace.downloadAsBlob` returns a binary file byte for byte.", "file": "F34-typescript-client.md", "line": 32 }, { "feature_id": "F34.workspace-scoped", "description": "a `RemoteWorkspace` with `conversationId` routes through `/{id}/bash|file/...`: commands default to the conversation's workspace and land in its own bash history, paths and `cwd` outside it get 422, an unknown conversation 404.", "file": "F34-typescript-client.md", "line": 33 }, { "feature_id": "F34.workspace-git", "description": "`RemoteWorkspace.gitChanges` and `gitDiff` return the server's changes and diff for a repository, globally and through `/{id}/git/...` for a scoped workspace, and wrap a 422 (path outside the conversation workspace) or 401 in an `Error` whose `cause` is the `HttpError`.", "file": "F34-typescript-client.md", "line": 34 }, { "feature_id": "F34.workspace-git-types", "description": "the statuses `gitChanges` returns type-check as the exported `GitChange['status']`, as they do against the client's generated contract.", "file": "F34-typescript-client.md", "line": 35 }, { "feature_id": "F34.bash-client", "description": "`RemoteWorkspace.bash` starts a background command, finds its output with `searchEvents`, reads events one at a time and in batches (unknown ids are `null`), stops a running command, treats an unknown id as already stopped, and `clearEvents` empties the history.", "file": "F34-typescript-client.md", "line": 36 }, { "feature_id": "F34.bash-websocket", "description": "`BashWebSocketClient` in Node delivers the bash events of a command (replayed with `resendMode: 'all'`) to its callback without errors.", "file": "F34-typescript-client.md", "line": 37 }, { "feature_id": "F34.workspace-session", "description": "`startWorkspaceSession(id)` answers with the workspace base URL that serves the conversation's files, `deleteWorkspaceSession()` succeeds, a mismatched scoped id is refused before any request, and no key is 401.", "file": "F34-typescript-client.md", "line": 38 }, { "feature_id": "F34.conversation-create", "description": "`RemoteConversation.start()` creates the server conversation with the client's agent (LLM, tools), working directory and `max_iterations`, and rebinds its workspace to the new conversation.", "file": "F34-typescript-client.md", "line": 39 }, { "feature_id": "F34.conversation-run", "description": "a reattached `RemoteConversation` stores a `sendMessage` without running, `run()` drives a real model to `finished`, the `hookConfig` given at creation runs before the tool call and is reported by `getHookConfig()`, the client reads the agent's file, stats, final response and trajectory ZIP, and `RemoteState` reports the server's status, agent, workspace, policy and persistence directory.", "file": "F34-typescript-client.md", "line": 40 }, { "feature_id": "F34.events-list", "description": "`RemoteEventsList` counts, searches (source, fully-qualified kind, body, pages), fetches events by id and iterates the same events the server stores.", "file": "F34-typescript-client.md", "line": 41 }, { "feature_id": "F34.events-unknown-id", "description": "`RemoteEventsList.getEventById` rejects an unknown event id with a 404 `HttpError` and `getEventsById` puts `null` in its slot.", "file": "F34-typescript-client.md", "line": 42 }, { "feature_id": "F34.events-search-kind", "description": "`RemoteEventsList.search({kind: 'MessageEvent'})` and `count({kind: 'MessageEvent'})`, the short kind its own docs show (`ActionEvent, MessageEvent`), find the same events as the fully-qualified kind.", "file": "F34-typescript-client.md", "line": 43 }, { "feature_id": "F34.event-stream", "description": "`ConversationEventStream` on Node's global `WebSocket` authenticates with the first frame, replays the stored events with `resend_mode: 'all'`, sends a user message that runs the agent to `finished` on the same socket, and reports a wrong key as a 4001 close.", "file": "F34-typescript-client.md", "line": 44 }, { "feature_id": "F34.ws-callback-client", "description": "`RemoteConversation.startWebSocketClient()` in Node delivers the conversation's events to the conversation callback without errors.", "file": "F34-typescript-client.md", "line": 45 }, { "feature_id": "F34.conversation-controls", "description": "`setTitle` and `setConfirmationPolicy` change the server's title and policy.", "file": "F34-typescript-client.md", "line": 46 }, { "feature_id": "F34.conversation-confirm", "description": "under `AlwaysConfirm`, `sendConfirmationResponse(false, reason)` records one `UserRejectObservation` with that reason and leaves the command unrun, `sendConfirmationResponse(true)` runs the pending command, and `updateSecrets` (a string and a function value) puts both values into that command's shell without any event containing them.", "file": "F34-typescript-client.md", "line": 47 }, { "feature_id": "F34.conversation-branching", "description": "`fork({title})` returns a `RemoteConversation` bound to the fork (its id, the fork's workspace and agent, the copied events), `navigateTo` refreshes the cached state to the new `leaf_event_id`, and `askAgent` answers without adding events to the source.", "file": "F34-typescript-client.md", "line": 48 }, { "feature_id": "F34.manager", "description": "`ConversationManager` searches with paging, counts by status, batch-gets (unknown ids are `null`), lists all, loads a conversation with its server working directory, creates and deletes one, and an unknown id rejects with 404.", "file": "F34-typescript-client.md", "line": 49 }, { "feature_id": "F34.manager-update-result", "description": "`ConversationManager.updateConversation` stores the new title and resolves with the updated `ConversationInfo` it is typed to return.", "file": "F34-typescript-client.md", "line": 50 }, { "feature_id": "F34.manager-create-options", "description": "`ConversationManager.createConversation(agent, {maxIterations, stuckDetection})` creates the conversation with those settings.", "file": "F34-typescript-client.md", "line": 51 }, { "feature_id": "F34.manager-acp", "description": "`ConversationManager.acp` creates a conversation for an inline ACP agent (with its `maxIterations`) and gets, batch-gets, lists and counts it with its ACP agent fields.", "file": "F34-typescript-client.md", "line": 52 }, { "feature_id": "F34.hooks-client", "description": "`HooksClient.loadHooks` and `RemoteConversation.loadHooks` read a project's `.openhands/hooks.json` as a snake_case `HookConfig` the client's hook helpers understand, and `null` for a directory without hooks.", "file": "F34-typescript-client.md", "line": 53 }, { "feature_id": "F34.mcp-client", "description": "`MCPClient.testServer` lists a stdio MCP server's tools and runs one tool call, reports a server that fails to start as `{ok: false}` with an `error_kind`, and an empty command rejects with 422.", "file": "F34-typescript-client.md", "line": 54 }, { "feature_id": "F34.restart-reload", "description": "after a server restart a new `ConversationManager` loads the same conversation with its status, title, events, hook config and workspace files.", "file": "F34-typescript-client.md", "line": 55 }, { "feature_id": "F34.event-stream-reconnect", "description": "a `ConversationEventStream` with `reconnect: {enabled: true}` and `queryParams: {resend_mode: 'all'}`, on the `ws` package's constructor, reconnects by itself after a server restart, replays the history again with the same query, and delivers a message posted after the restart.", "file": "F34-typescript-client.md", "line": 56 }, { "feature_id": "F34.event-stream-reconnect-native", "description": "the same stream on Node's global `WebSocket` also reconnects after a server restart.", "file": "F34-typescript-client.md", "line": 57 }, { "feature_id": "F34.endpoint-audit", "description": "the client's endpoint audit against this checkout's public OpenAPI contract finds no client call the server lacks, and regenerating the client's types from that contract removes no exported type.", "file": "F34-typescript-client.md", "line": 58 }, { "feature_id": "F34.version-gate-old", "description": "against a real 1.22.0 agent-server release, the gated methods reject with `AgentServerVersionError` (required and actual versions) and never call the gated route.", "file": "F34-typescript-client.md", "line": 59 } ], "confirmed_bugs": [ { "id": "F02.workspace-cookie-root-path", "title": "Workspace session cookie drops the server URL prefix", "severity": "medium", "feature_ids": [ "F02.workspace-cookie-root-path" ], "expected": "A cookie minted through a prefixed server URL authenticates the prefixed workspace file URL.", "actual": "Cookie Path is regardless of root_path. The browser-like cookie jar gets 401 at /qa-f02-prefix/{id}/workspace/qa-f02.html; the same valid cookie gets 200 at the unprefixed URL.", "repro_command": "control-agent-server map run --file F02 --only F02.workspace-cookie-root-path,F02.workspace-session-ipv6-loopback --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-022326-2081-map-f02", "/workspace/verification-artifacts/runs/20261013-022650-a777-map-f02" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-022430-59fc-f02-prefix/evidence/F02.workspace-cookie-root-path/mint.json", "/workspace/verification-artifacts/runs/20261013-022430-59fc-f02-prefix/evidence/F02.workspace-cookie-root-path/file-with-cookie.json", "/workspace/verification-artifacts/runs/20261013-022749-a380-f02-prefix/evidence/F02.workspace-cookie-root-path/mint.json", "/workspace/verification-artifacts/runs/20261013-022749-a380-f02-prefix/evidence/F02.workspace-cookie-root-path/file-with-cookie.json" ], "notes": "Known map bug reproduced at the marked assertion in both fresh attempts. Header-authenticated prefixed URL and cookie-authenticated unprefixed URL are successful controls. No model needed. Source: auth_router.py cookie Path ignores root_path." }, { "id": "F02.workspace-session-ipv6-loopback", "title": "IPv6 loopback workspace cookies omit Secure and Partitioned", "severity": "medium", "feature_ids": [ "F02.workspace-session-ipv6-loopback" ], "expected": "Host [::1]:8000 receives SameSite=none; Secure; Partitioned, matching localhost and 127.0.0.1.", "actual": "The IPv6 Host request returns 204 with SameSite=none but omits Secure and Partitioned, so browsers reject the cookie. The two IPv4/localhost controls include both flags.", "repro_command": "control-agent-server map run --file F02 --only F02.workspace-cookie-root-path,F02.workspace-session-ipv6-loopback --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-022326-2081-map-f02", "/workspace/verification-artifacts/runs/20261013-022650-a777-map-f02" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-022326-2081-map-f02/evidence/F02.workspace-session-ipv6-loopback/ipv6-host.json", "/workspace/verification-artifacts/runs/20261013-022650-a777-map-f02/evidence/F02.workspace-session-ipv6-loopback/ipv6-host.json" ], "notes": "Known map bug reproduced twice. The test changes the HTTP Host header to IPv6 loopback while reaching the same loopback server; it does not claim an IPv6 network bind test. Source: auth_router.py:42-65 splits request.url.hostname on ':' before loopback matching." }, { "id": "F03.init-delivered-cors-origins", "title": "Deferred init accepts CORS origins but leaves the boot allowlist unchanged", "severity": "medium", "feature_ids": [ "F03.init-delivered-cors-origins" ], "expected": "The allow_cors_origins delivered by successful POST /api/init authorizes that origin on a CORS preflight.", "actual": "The delivered https://qa-f03.example gets 400 Disallowed CORS origin and no Access-Control-Allow-Origin. Localhost control gets200; an undelivered origin also gets400.", "repro_command": "control-agent-server map run --file F03 --only F03.init-success,F03.init-delivered-cors-origins --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-023017-4c4d-map-f03", "/workspace/verification-artifacts/runs/20261013-023328-d486-map-f03" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-023017-4c4d-map-f03/evidence/F03.init-delivered-cors-origins/preflight.json", "/workspace/verification-artifacts/runs/20261013-023328-d486-map-f03/evidence/F03.init-delivered-cors-origins/preflight.json" ], "notes": "Known map bug at marked assertion in both fresh attempts. CORSDispatcher captures config.allow_cors_origins when create_app runs; successful init updates the config but does not rebuild middleware." }, { "id": "F04.create-initial-message-run-false", "title": "Conversation create executes an initial message with run=false", "severity": "medium", "feature_ids": [ "F04.create-initial-message-run-false" ], "expected": "Persist the initial message and remain idle with no model run when initial_message.run is false.", "actual": "The false request runs to finished on the controlled provider just like the true control.", "repro_command": "control-agent-server map run --file F04 --only F04.create-initial-message-run-false,F04.create-secrets-get,F04.create-secrets-plain,F04.batch-get-limit,F04.patch-whitespace-title,F04.patch-unknown-id,F04.delete-unknown-id --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04/evidence/F04.create-initial-message-run-false/start.json", "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04/evidence/map-run/F04-conversation-lifecycle-02-initial-message-run-false-f04-create-ini.json", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04/evidence/F04.create-initial-message-run-false/start.json", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04/evidence/map-run/F04-conversation-lifecycle-02-initial-message-run-false-f04-create-ini.json", "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04/evidence/F04.create-initial-message-run-false/observed-actions.json", "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04/evidence/F04.create-initial-message-run-false/observed-status.json", "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04/evidence/F04.create-initial-message-run-false/supplement-actions.json", "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04/evidence/F04.create-initial-message-run-false/supplement-get.json", "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04/evidence/F04.create-initial-message-run-false/supplement-message.json", "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04/evidence/F04.create-initial-message-run-false/supplement-start.json", "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04/evidence/map-run/F04-conversation-lifecycle-04-run-false-final-status-f04-create-initia.json", "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04/evidence/map-run/F04-conversation-lifecycle-05-run-false-action-count-f04-create-initia.json", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04/evidence/F04.create-initial-message-run-false/observed-actions.json", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04/evidence/F04.create-initial-message-run-false/observed-status.json", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04/evidence/F04.create-initial-message-run-false/supplement-actions.json", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04/evidence/F04.create-initial-message-run-false/supplement-get.json", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04/evidence/F04.create-initial-message-run-false/supplement-message.json", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04/evidence/F04.create-initial-message-run-false/supplement-start.json", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04/evidence/map-run/F04-conversation-lifecycle-04-run-false-final-status-f04-create-initia.json", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04/evidence/map-run/F04-conversation-lifecycle-05-run-false-action-count-f04-create-initia.json" ], "notes": "The live recipe checks the persisted user message, then observes execution starting/finishing. The create schema advertises run=false, but _start_conversation unconditionally calls run(). Supplemental GET reports finished and exactly one ActionEvent on both fresh attempts.", "supplemental_runs": [ "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04" ], "supplemental_commands": [ "python3 /workspace/verification-artifacts/adapted-f04-supplement/run-adapted-f04-supplement.py map run --file F04 --fresh --repeat 2 --record --keep-going --timeout 420" ] }, { "id": "F04.create-secrets-plain", "title": "Plain-string create secrets cause an unhandled server error", "severity": "low", "feature_ids": [ "F04.create-secrets-plain" ], "expected": "Accept the plain-string secret like the secrets update route, or reject it with 422.", "actual": "POST returns 500: 'str' object has no attribute 'pop'.", "repro_command": "control-agent-server map run --file F04 --only F04.create-initial-message-run-false,F04.create-secrets-get,F04.create-secrets-plain,F04.batch-get-limit,F04.patch-whitespace-title,F04.patch-unknown-id,F04.delete-unknown-id --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04/evidence/F04.create-secrets-plain/start.json", "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04/evidence/map-run/F04-conversation-lifecycle-07-plain-string-secret-value-f04-create-sec.json", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04/evidence/F04.create-secrets-plain/start.json", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04/evidence/map-run/F04-conversation-lifecycle-07-plain-string-secret-value-f04-create-sec.json" ], "notes": "This is a malformed-input/status-only defect. Synthetic QA values are used; no real secret is posted in the negative test." }, { "id": "F04.batch-get-limit", "title": "A conversation batch of 100 IDs returns 500 instead of a client error", "severity": "low", "feature_ids": [ "F04.batch-get-limit" ], "expected": "Refuse 100 IDs with 400/422 naming the limit.", "actual": "A batch of 100 well-formed UUIDs returns 500 with an empty exception string.", "repro_command": "control-agent-server map run --file F04 --only F04.create-initial-message-run-false,F04.create-secrets-get,F04.create-secrets-plain,F04.batch-get-limit,F04.patch-whitespace-title,F04.patch-unknown-id,F04.delete-unknown-id --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04/evidence/F04.batch-get-limit/hundred.json", "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04/evidence/map-run/F04-conversation-lifecycle-08-100-ids-in-one-batch-f04-batch-get-limit.json", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04/evidence/F04.batch-get-limit/hundred.json", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04/evidence/map-run/F04-conversation-lifecycle-08-100-ids-in-one-batch-f04-batch-get-limit.json" ], "notes": "batch_get_conversations uses assert len(ids)<100 instead of request validation; the ordinary 99-ID control is covered by the primary full family." }, { "id": "F04.patch-whitespace-title", "title": "Whitespace title passes validation and erases the title", "severity": "low", "feature_ids": [ "F04.patch-whitespace-title" ], "expected": "Reject a whitespace-only title with 422 and retain the existing title.", "actual": "PATCH with a whitespace-only title returns 200; a subsequent GET reports title=\"\" on two fresh supplemental runs.", "repro_command": "control-agent-server map run --file F04 --only F04.create-initial-message-run-false,F04.create-secrets-get,F04.create-secrets-plain,F04.batch-get-limit,F04.patch-whitespace-title,F04.patch-unknown-id,F04.delete-unknown-id --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04/evidence/F04.patch-whitespace-title/patch.json", "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04/evidence/map-run/F04-conversation-lifecycle-09-whitespace-only-title-f04-patch-whitespa.json", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04/evidence/F04.patch-whitespace-title/patch.json", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04/evidence/map-run/F04-conversation-lifecycle-09-whitespace-only-title-f04-patch-whitespa.json", "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04/evidence/F04.patch-whitespace-title/accepted-whitespace.json", "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04/evidence/F04.patch-whitespace-title/before.json", "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04/evidence/F04.patch-whitespace-title/get-after-whitespace.json", "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04/evidence/map-run/F04-conversation-lifecycle-02-whitespace-title-persisted-get-f04-patch.json", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04/evidence/F04.patch-whitespace-title/accepted-whitespace.json", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04/evidence/F04.patch-whitespace-title/before.json", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04/evidence/F04.patch-whitespace-title/get-after-whitespace.json", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04/evidence/map-run/F04-conversation-lifecycle-02-whitespace-title-persisted-get-f04-patch.json" ], "notes": "Metadata defect: validation checks min_length before stripping. Acceptance and persisted title erasure are both independently proven twice.", "supplemental_runs": [ "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04" ], "supplemental_commands": [ "python3 /workspace/verification-artifacts/adapted-f04-supplement/run-adapted-f04-supplement.py map run --file F04 --fresh --repeat 2 --record --keep-going --timeout 420" ] }, { "id": "F04.patch-unknown-id", "title": "PATCH on a missing conversation contradicts its documented 404", "severity": "low", "feature_ids": [ "F04.patch-unknown-id" ], "expected": "PATCH returns 404 for the same missing ID that GET returns 404 for.", "actual": "PATCH returns 200 with success=false despite the OpenAPI 404 response declaration.", "repro_command": "control-agent-server map run --file F04 --only F04.create-initial-message-run-false,F04.create-secrets-get,F04.create-secrets-plain,F04.batch-get-limit,F04.patch-whitespace-title,F04.patch-unknown-id,F04.delete-unknown-id --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04/evidence/F04.patch-unknown-id/unknown.json", "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04/evidence/map-run/F04-conversation-lifecycle-10-patch-on-an-unknown-id-f04-patch-unknown.json", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04/evidence/F04.patch-unknown-id/unknown.json", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04/evidence/map-run/F04-conversation-lifecycle-10-patch-on-an-unknown-id-f04-patch-unknown.json" ], "notes": "Status-contract defect; the source test currently locks the 200 behavior." }, { "id": "F04.delete-unknown-id", "title": "DELETE on a missing conversation contradicts its documented 404", "severity": "low", "feature_ids": [ "F04.delete-unknown-id" ], "expected": "DELETE returns 404 for a missing or already-deleted conversation.", "actual": "DELETE on an absent ID returns 400, and a second DELETE after successful deletion also returns 400, despite the documented 404.", "repro_command": "control-agent-server map run --file F04 --only F04.create-initial-message-run-false,F04.create-secrets-get,F04.create-secrets-plain,F04.batch-get-limit,F04.patch-whitespace-title,F04.patch-unknown-id,F04.delete-unknown-id --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04/evidence/F04.delete-unknown-id/unknown.json", "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04/evidence/map-run/F04-conversation-lifecycle-12-delete-on-an-unknown-id-f04-delete-unkno.json", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04/evidence/F04.delete-unknown-id/unknown.json", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04/evidence/map-run/F04-conversation-lifecycle-12-delete-on-an-unknown-id-f04-delete-unkno.json", "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04/evidence/F04.delete-unknown-id/first-delete.json", "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04/evidence/F04.delete-unknown-id/get-deleted.json", "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04/evidence/F04.delete-unknown-id/repeat.json", "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04/evidence/map-run/F04-conversation-lifecycle-03-repeated-delete-f04-delete-unknown-id-re.json", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04/evidence/F04.delete-unknown-id/first-delete.json", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04/evidence/F04.delete-unknown-id/get-deleted.json", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04/evidence/F04.delete-unknown-id/repeat.json", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04/evidence/map-run/F04-conversation-lifecycle-03-repeated-delete-f04-delete-unknown-id-re.json" ], "notes": "Both the never-created and already-deleted branches reproduced twice.", "supplemental_runs": [ "/workspace/verification-artifacts/runs/20261013-032709-376d-map-f04", "/workspace/verification-artifacts/runs/20261013-032756-3c22-map-f04" ], "supplemental_commands": [ "python3 /workspace/verification-artifacts/adapted-f04-supplement/run-adapted-f04-supplement.py map run --file F04 --fresh --repeat 2 --record --keep-going --timeout 420" ] }, { "id": "F05.run-error-responsive", "title": "Rich exception rendering stalls the entire agent server", "severity": "high", "feature_ids": [ "F05.run-error-responsive" ], "expected": "Health and event-count probes remain below 1.5 seconds while a model-provider failure is handled.", "actual": "A positive-control /alive request took 67 ms; a failure-time probe took 4,512 ms. The second fresh attempt recorded 10,909 ms, while the request before the failure met the 1.5-second threshold.", "repro_command": "control-agent-server map run --file F05 --only F05.run-error,F05.run-error-responsive,F05.unknown-id-pause,F05.run-conflict,F05.pause-running,F05.pause-running-get,F05.pause-running-release,F05.restart-running-graceful,F05.restart-hard-running,F05.pause-running-crash --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-024519-40d0-map-f05", "/workspace/verification-artifacts/runs/20261013-024951-9459-map-f05" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024519-40d0-map-f05/evidence/F05.run-error-responsive/after.json", "/workspace/verification-artifacts/runs/20261013-024519-40d0-map-f05/evidence/F05.run-error-responsive/alive-before.json", "/workspace/verification-artifacts/runs/20261013-024519-40d0-map-f05/evidence/map-run/F05-run-pause-interrupt-02-responsive-while-a-run-fails-f05-run-err.json", "/workspace/verification-artifacts/runs/20261013-024951-9459-map-f05/evidence/F05.run-error-responsive/after.json", "/workspace/verification-artifacts/runs/20261013-024951-9459-map-f05/evidence/F05.run-error-responsive/alive-before.json", "/workspace/verification-artifacts/runs/20261013-024951-9459-map-f05/evidence/map-run/F05-run-pause-interrupt-02-responsive-while-a-run-fails-f05-run-err.json" ], "notes": "Default console logger.exception renders a large Rich traceback synchronously on the event loop. LOG_JSON=true or LOG_RICH_TRACEBACKS=false is the documented workaround; production Docker uses JSON logging." }, { "id": "F05.unknown-id-pause", "title": "Pause/interrupt on an unknown conversation use undocumented 400", "severity": "low", "feature_ids": [ "F05.unknown-id-pause" ], "expected": "Both pause and interrupt return their documented 404 for a missing conversation.", "actual": "Both pause and interrupt return 400 for an absent valid UUID, despite GET returning 404 and both OpenAPI routes documenting 404.", "repro_command": "control-agent-server map run --file F05 --only F05.run-error,F05.run-error-responsive,F05.unknown-id-pause,F05.run-conflict,F05.pause-running,F05.pause-running-get,F05.pause-running-release,F05.restart-running-graceful,F05.restart-hard-running,F05.pause-running-crash --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-024519-40d0-map-f05", "/workspace/verification-artifacts/runs/20261013-024951-9459-map-f05" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024519-40d0-map-f05/evidence/F05.unknown-id-pause/pause.json", "/workspace/verification-artifacts/runs/20261013-024519-40d0-map-f05/evidence/map-run/F05-run-pause-interrupt-03-pause-or-interrupt-an-unknown-id-f05-unk.json", "/workspace/verification-artifacts/runs/20261013-024951-9459-map-f05/evidence/F05.unknown-id-pause/pause.json", "/workspace/verification-artifacts/runs/20261013-024951-9459-map-f05/evidence/map-run/F05-run-pause-interrupt-03-pause-or-interrupt-an-unknown-id-f05-unk.json", "/workspace/verification-artifacts/runs/20261013-033533-16b0-map-f05/evidence/F05.unknown-id-interrupt/get-control.json", "/workspace/verification-artifacts/runs/20261013-033533-16b0-map-f05/evidence/F05.unknown-id-interrupt/interrupt.json", "/workspace/verification-artifacts/runs/20261013-033533-16b0-map-f05/evidence/F05.unknown-id-interrupt/openapi.json", "/workspace/verification-artifacts/runs/20261013-033533-16b0-map-f05/evidence/map-run/F05-run-pause-interrupt-01-unknown-interrupt-f05-unknown-id-interru.json", "/workspace/verification-artifacts/runs/20261013-033615-9b59-map-f05/evidence/F05.unknown-id-interrupt/get-control.json", "/workspace/verification-artifacts/runs/20261013-033615-9b59-map-f05/evidence/F05.unknown-id-interrupt/interrupt.json", "/workspace/verification-artifacts/runs/20261013-033615-9b59-map-f05/evidence/F05.unknown-id-interrupt/openapi.json", "/workspace/verification-artifacts/runs/20261013-033615-9b59-map-f05/evidence/map-run/F05-run-pause-interrupt-01-unknown-interrupt-f05-unknown-id-interru.json" ], "notes": "Each method reproduced twice. Status-only defect.", "supplemental_runs": [ "/workspace/verification-artifacts/runs/20261013-033533-16b0-map-f05", "/workspace/verification-artifacts/runs/20261013-033615-9b59-map-f05" ], "supplemental_commands": [ ".venv/bin/python /workspace/verification-artifacts/adapted-f05-secondary/run-adapted-f05-secondary.py map run --file F05 --fresh --repeat 2 --record --keep-going --timeout 120" ] }, { "id": "pause-state-snapshot-stale", "title": "A mid-call pause is invisible to GET and lost on crash recovery", "severity": "medium", "feature_ids": [ "F05.pause-running-get", "F05.pause-running-crash" ], "expected": "GET agrees with the accepted paused state; after SIGKILL/restart the accepted pause remains paused.", "actual": "During an accepted mid-call pause, status search and PauseEvent say paused, but GET and base_state.json say running. After SIGKILL, lease expiry, and restart, both PauseEvents remain but GET reports error.", "repro_command": "control-agent-server map run --file F05 --only F05.run-error,F05.run-error-responsive,F05.unknown-id-pause,F05.run-conflict,F05.pause-running,F05.pause-running-get,F05.pause-running-release,F05.restart-running-graceful,F05.restart-hard-running,F05.pause-running-crash --fresh --repeat 2 --record --keep-going --timeout 420\n.venv/bin/python /workspace/verification-artifacts/adapted-f05/run-adapted-f05.py map run --file F05 --only F05.restart-running-graceful,F05.restart-hard-running,F05.pause-running-crash --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-024519-40d0-map-f05", "/workspace/verification-artifacts/runs/20261013-024951-9459-map-f05", "/workspace/verification-artifacts/runs/20261013-032311-dcf8-map-f05", "/workspace/verification-artifacts/runs/20261013-032700-5284-map-f05" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024519-40d0-map-f05/evidence/F05.pause-running-get/get.json", "/workspace/verification-artifacts/runs/20261013-024519-40d0-map-f05/evidence/map-run/F05-run-pause-interrupt-06-get-after-a-mid-call-pause-f05-pause-run.json", "/workspace/verification-artifacts/runs/20261013-024951-9459-map-f05/evidence/F05.pause-running-get/get.json", "/workspace/verification-artifacts/runs/20261013-024951-9459-map-f05/evidence/map-run/F05-run-pause-interrupt-06-get-after-a-mid-call-pause-f05-pause-run.json", "/workspace/verification-artifacts/runs/20261013-032311-dcf8-map-f05/evidence/F05.pause-running-crash/get.json", "/workspace/verification-artifacts/runs/20261013-032311-dcf8-map-f05/evidence/F05.pause-running-crash/lease-expiry.json", "/workspace/verification-artifacts/runs/20261013-032311-dcf8-map-f05/evidence/map-run/F05-run-pause-interrupt-03-crash-after-a-mid-call-pause-f05-pause-r.json", "/workspace/verification-artifacts/runs/20261013-032700-5284-map-f05/evidence/F05.pause-running-crash/get.json", "/workspace/verification-artifacts/runs/20261013-032700-5284-map-f05/evidence/F05.pause-running-crash/lease-expiry.json", "/workspace/verification-artifacts/runs/20261013-032700-5284-map-f05/evidence/map-run/F05-run-pause-interrupt-03-crash-after-a-mid-call-pause-f05-pause-r.json", "/workspace/verification-artifacts/runs/20261013-033533-16b0-map-f05/evidence/F05.pause-running-base-state/accepted.json", "/workspace/verification-artifacts/runs/20261013-033533-16b0-map-f05/evidence/F05.pause-running-base-state/pause-count.json", "/workspace/verification-artifacts/runs/20261013-033533-16b0-map-f05/evidence/F05.pause-running-base-state/search-control.json", "/workspace/verification-artifacts/runs/20261013-033533-16b0-map-f05/evidence/map-run/F05-run-pause-interrupt-03-paused-base-snapshot-f05-pause-running-b.json", "/workspace/verification-artifacts/runs/20261013-033615-9b59-map-f05/evidence/F05.pause-running-base-state/accepted.json", "/workspace/verification-artifacts/runs/20261013-033615-9b59-map-f05/evidence/F05.pause-running-base-state/pause-count.json", "/workspace/verification-artifacts/runs/20261013-033615-9b59-map-f05/evidence/F05.pause-running-base-state/search-control.json", "/workspace/verification-artifacts/runs/20261013-033615-9b59-map-f05/evidence/map-run/F05-run-pause-interrupt-03-paused-base-snapshot-f05-pause-running-b.json" ], "notes": "One root cause across GET, disk snapshot, and crash recovery: autosave is deferred while model I/O holds save_depth open. In this PID-1 container the killed owner remains a zombie; the adaptation verifies that state, records the remaining owner-lease TTL, waits for expiry, and restarts before the target assertion. Instantaneous dead-owner recovery is not certified.", "supplemental_runs": [ "/workspace/verification-artifacts/runs/20261013-033533-16b0-map-f05", "/workspace/verification-artifacts/runs/20261013-033615-9b59-map-f05" ], "supplemental_commands": [ ".venv/bin/python /workspace/verification-artifacts/adapted-f05-secondary/run-adapted-f05-secondary.py map run --file F05 --fresh --repeat 2 --record --keep-going --timeout 120" ] }, { "id": "F06.search-kind-short", "title": "Documented short event kinds silently return empty results", "severity": "medium", "feature_ids": [ "F06.search-kind-short", "F34.events-search-kind" ], "expected": "kind=MessageEvent returns the two user messages, matching their wire kind and the TypeScript client contract.", "actual": "The fully qualified MessageEvent filter finds two messages; the documented short kind returns an empty search and count 0. Both routes independently reproduce twice.", "repro_command": "control-agent-server map run --file F06 --only F06.send-no-run,F06.search-kind-short,F06.get-one,F06.unknown-event,F06.send-non-user-role,F06.ws-delete-close,F06.search-timestamp-order --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-025406-9fbb-map-f06", "/workspace/verification-artifacts/runs/20261013-025528-20b8-map-f06", "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025406-9fbb-map-f06/evidence/F06.search-kind-short/short-name.json", "/workspace/verification-artifacts/runs/20261013-025406-9fbb-map-f06/evidence/map-run/F06-conversation-events-02-short-kind-names-f06-search-kind-short-k.json", "/workspace/verification-artifacts/runs/20261013-025528-20b8-map-f06/evidence/F06.search-kind-short/short-name.json", "/workspace/verification-artifacts/runs/20261013-025528-20b8-map-f06/evidence/map-run/F06-conversation-events-02-short-kind-names-f06-search-kind-short-k.json", "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06/evidence/F06.search-kind-short-count/fqn-control.json", "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06/evidence/F06.search-kind-short-count/short-count.json", "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06/evidence/map-run/F06-conversation-events-04-short-kind-count-f06-search-kind-short-c.json", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/F06.search-kind-short-count/fqn-control.json", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/F06.search-kind-short-count/short-count.json", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/map-run/F06-conversation-events-04-short-kind-count-f06-search-kind-short-c.json", "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-04-short-event-kinds-f34-events-search-kind.json", "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-04-short-event-kinds-f34-events-search-kind.json" ], "notes": "The service compares module.ClassName rather than the documented wire kind; TypeScript clients following the documentation silently lose history. Aggregate merge: F34.events-search-kind. The TypeScript search drives the same server short-kind versus module.ClassName comparison.", "supplemental_runs": [ "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06" ], "supplemental_commands": [ ".venv/bin/python /workspace/verification-artifacts/adapted-f06-supplement/run-adapted-f06.py map run --file F06 --only F06.send-no-run,F06.get-one,F06.send-run,F06.search-kind-short-count,F06.unknown-event-batch,F06.send-non-user-role-empty,F06.send-non-user-role-empty-mutation,F06.search-timestamp-replay,F06.search-timestamp-order-sorted --fresh --repeat 2 --record --keep-going --timeout 420" ], "merged_cases": [ { "id": "F34.events-search-kind", "title": "TypeScript documented short event kinds find no events", "severity": "medium", "feature_ids": [ "F34.events-search-kind" ], "expected": "MessageEvent should find the same queued user message as the fully qualified kind.", "actual": "The qualified kind finds one; the short name returns no items and count zero. Same server root cause as F06.search-kind-short.", "repro_command": "control-agent-server map run --file F34 --only F34.events-search-kind --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-04-short-event-kinds-f34-events-search-kind.json", "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-04-short-event-kinds-f34-events-search-kind.json" ], "notes": [ "Same server defect as F06.search-kind-short; cross-family aggregate must deduplicate." ] } ] }, { "id": "F06.unknown-event", "title": "One absent event crashes get-one and batch history reads", "severity": "medium", "feature_ids": [ "F06.unknown-event", "F34.events-unknown-id" ], "expected": "Get-one returns 404 for a missing ID; a mixed batch preserves known events and returns null for the missing slot.", "actual": "Get-one returns 500 for an absent event ID; a batch containing one valid event and one absent event also returns 500 instead of [event, null]. Both branches reproduce twice.", "repro_command": "control-agent-server map run --file F06 --only F06.send-no-run,F06.search-kind-short,F06.get-one,F06.unknown-event,F06.send-non-user-role,F06.ws-delete-close,F06.search-timestamp-order --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-025406-9fbb-map-f06", "/workspace/verification-artifacts/runs/20261013-025528-20b8-map-f06", "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025406-9fbb-map-f06/evidence/F06.unknown-event/get-one.json", "/workspace/verification-artifacts/runs/20261013-025406-9fbb-map-f06/evidence/map-run/F06-conversation-events-04-unknown-event-id-f06-unknown-event-known.json", "/workspace/verification-artifacts/runs/20261013-025528-20b8-map-f06/evidence/F06.unknown-event/get-one.json", "/workspace/verification-artifacts/runs/20261013-025528-20b8-map-f06/evidence/map-run/F06-conversation-events-04-unknown-event-id-f06-unknown-event-known.json", "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06/evidence/F06.unknown-event-batch/known-control.json", "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06/evidence/F06.unknown-event-batch/mixed.json", "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06/evidence/map-run/F06-conversation-events-05-mixed-batch-f06-unknown-event-batch-know.json", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/F06.unknown-event-batch/known-control.json", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/F06.unknown-event-batch/mixed.json", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/map-run/F06-conversation-events-05-mixed-batch-f06-unknown-event-batch-know.json", "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-03-unknown-event-id-f34-events-unknown-id-k.json", "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-03-unknown-event-id-f34-events-unknown-id-k.json" ], "notes": "EventLog.get_index raises KeyError before the router can return 404/null; an absent item aborts the whole batch. Aggregate merge: F34.events-unknown-id. The TypeScript lookup drives the same EventLog.get_index missing-ID failure as the REST consumer.", "supplemental_runs": [ "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06" ], "supplemental_commands": [ ".venv/bin/python /workspace/verification-artifacts/adapted-f06-supplement/run-adapted-f06.py map run --file F06 --only F06.send-no-run,F06.get-one,F06.send-run,F06.search-kind-short-count,F06.unknown-event-batch,F06.send-non-user-role-empty,F06.send-non-user-role-empty-mutation,F06.search-timestamp-replay,F06.search-timestamp-order-sorted --fresh --repeat 2 --record --keep-going --timeout 420" ], "merged_cases": [ { "id": "F34.events-unknown-id", "title": "TypeScript event lookup receives 500 for a missing event", "severity": "low", "feature_ids": [ "F34.events-unknown-id" ], "expected": "A missing event should reject with 404 and batch lookup should return null in that slot.", "actual": "Single and batch calls both reject with 500 Unknown event_id. This is the server defect also exercised by F06.unknown-event.", "repro_command": "control-agent-server map run --file F34 --only F34.events-unknown-id --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-03-unknown-event-id-f34-events-unknown-id-k.json", "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-03-unknown-event-id-f34-events-unknown-id-k.json" ], "notes": [ "Same server defect as F06.unknown-event; cross-family aggregate must deduplicate." ] } ] }, { "id": "F06.send-non-user-role", "title": "A schema-valid non-user message causes 500", "severity": "low", "feature_ids": [ "F06.send-non-user-role" ], "expected": "Reject assistant-role input with 4xx and append no events, including on an empty log.", "actual": "An assistant-role POST returns 500 on both initialized and empty logs. The rejected POST to an empty log also persists a SystemPromptEvent, leaving count 1 instead of 0.", "repro_command": "control-agent-server map run --file F06 --only F06.send-no-run,F06.search-kind-short,F06.get-one,F06.unknown-event,F06.send-non-user-role,F06.ws-delete-close,F06.search-timestamp-order --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-025406-9fbb-map-f06", "/workspace/verification-artifacts/runs/20261013-025528-20b8-map-f06" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025406-9fbb-map-f06/evidence/F06.send-non-user-role/assistant.json", "/workspace/verification-artifacts/runs/20261013-025406-9fbb-map-f06/evidence/map-run/F06-conversation-events-05-non-user-role-f06-send-non-user-role-kno.json", "/workspace/verification-artifacts/runs/20261013-025528-20b8-map-f06/evidence/F06.send-non-user-role/assistant.json", "/workspace/verification-artifacts/runs/20261013-025528-20b8-map-f06/evidence/map-run/F06-conversation-events-05-non-user-role-f06-send-non-user-role-kno.json", "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06/evidence/F06.send-non-user-role-empty-mutation/after.json", "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06/evidence/F06.send-non-user-role-empty-mutation/before.json", "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06/evidence/F06.send-non-user-role-empty-mutation/events.json", "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06/evidence/F06.send-non-user-role-empty-mutation/rejected.json", "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06/evidence/F06.send-non-user-role-empty/before.json", "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06/evidence/F06.send-non-user-role-empty/post.json", "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06/evidence/map-run/F06-conversation-events-06-non-user-empty-log-f06-send-non-user-rol.json", "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06/evidence/map-run/F06-conversation-events-07-non-user-empty-log-mutation-f06-send-non.json", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/F06.send-non-user-role-empty-mutation/after.json", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/F06.send-non-user-role-empty-mutation/before.json", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/F06.send-non-user-role-empty-mutation/events.json", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/F06.send-non-user-role-empty-mutation/rejected.json", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/F06.send-non-user-role-empty/before.json", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/F06.send-non-user-role-empty/post.json", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/map-run/F06-conversation-events-06-non-user-empty-log-f06-send-non-user-rol.json", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/map-run/F06-conversation-events-07-non-user-empty-log-mutation-f06-send-non.json" ], "notes": "Role is schema-valid but LocalConversation asserts user. Both response and empty-log mutation branches reproduced twice. Count remains one low-severity malformed-input bug.", "supplemental_runs": [ "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06" ], "supplemental_commands": [ ".venv/bin/python /workspace/verification-artifacts/adapted-f06-supplement/run-adapted-f06.py map run --file F06 --only F06.send-no-run,F06.get-one,F06.send-run,F06.search-kind-short-count,F06.unknown-event-batch,F06.send-non-user-role-empty,F06.send-non-user-role-empty-mutation,F06.search-timestamp-replay,F06.search-timestamp-order-sorted --fresh --repeat 2 --record --keep-going --timeout 420" ] }, { "id": "F06.search-timestamp-order", "title": "Timestamp-based event resume skips delayed stats updates", "severity": "medium", "feature_ids": [ "F06.search-timestamp-order" ], "expected": "A timestamp cut at an observation keeps that observation and every later-listed event in REST and WebSocket replay.", "actual": "The marked REST count is lower than the number of events listed from the observation onward.", "repro_command": "control-agent-server map run --file F06 --only F06.send-no-run,F06.search-kind-short,F06.get-one,F06.unknown-event,F06.send-non-user-role,F06.ws-delete-close,F06.search-timestamp-order --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-025406-9fbb-map-f06", "/workspace/verification-artifacts/runs/20261013-025528-20b8-map-f06" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025406-9fbb-map-f06/evidence/F06.search-timestamp-order/count-since-observation.json", "/workspace/verification-artifacts/runs/20261013-025406-9fbb-map-f06/evidence/map-run/F06-conversation-events-08-log-order-is-timestamp-order-f06-search-.json", "/workspace/verification-artifacts/runs/20261013-025528-20b8-map-f06/evidence/F06.search-timestamp-order/count-since-observation.json", "/workspace/verification-artifacts/runs/20261013-025528-20b8-map-f06/evidence/map-run/F06-conversation-events-08-log-order-is-timestamp-order-f06-search-.json", "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06/evidence/F06.search-timestamp-order-sorted/all.json", "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06/evidence/F06.search-timestamp-replay/socket.json", "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06/evidence/map-run/F06-conversation-events-08-timestamp-socket-replay-f06-search-times.json", "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06/evidence/map-run/F06-conversation-events-09-ascending-timestamps-f06-search-timestam.json", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/F06.search-timestamp-order-sorted/all.json", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/F06.search-timestamp-replay/socket.json", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/map-run/F06-conversation-events-08-timestamp-socket-replay-f06-search-times.json", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06/evidence/map-run/F06-conversation-events-09-ascending-timestamps-f06-search-timestam.json" ], "notes": "Stats updates are timestamped at model response time, then appended after tool observations; both REST count and legacy socket replay skip a subsequently appended stats event. Replay and unsorted-timestamp assertions separately reproduced twice using real DeepSeek file-editor turns.", "supplemental_runs": [ "/workspace/verification-artifacts/runs/20261013-032934-9891-map-f06", "/workspace/verification-artifacts/runs/20261013-033036-477e-map-f06" ], "supplemental_commands": [ ".venv/bin/python /workspace/verification-artifacts/adapted-f06-supplement/run-adapted-f06.py map run --file F06 --only F06.send-no-run,F06.get-one,F06.send-run,F06.search-kind-short-count,F06.unknown-event-batch,F06.send-non-user-role-empty,F06.send-non-user-role-empty-mutation,F06.search-timestamp-replay,F06.search-timestamp-order-sorted --fresh --repeat 2 --record --keep-going --timeout 420" ] }, { "id": "conversation-deletion-leaves-sockets-open", "title": "Deleting a conversation leaves its event/session sockets open", "severity": "medium", "feature_ids": [ "F06.ws-delete-close", "F07.deleted-close" ], "expected": "DELETE closes already subscribed sockets before answering; subsequent reads are 404 and new sockets 4004.", "actual": "DELETE succeeds, REST returns 404 and a new socket 4004, but the existing capture remains open with close=null.", "repro_command": "control-agent-server map run --file F06 --only F06.send-no-run,F06.search-kind-short,F06.get-one,F06.unknown-event,F06.send-non-user-role,F06.ws-delete-close,F06.search-timestamp-order --fresh --repeat 2 --record --keep-going --timeout 420\ncontrol-agent-server map run --file F07 --only F07.live-order,F07.deleted-close --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-025406-9fbb-map-f06", "/workspace/verification-artifacts/runs/20261013-025528-20b8-map-f06", "/workspace/verification-artifacts/runs/20261013-025721-700e-map-f07", "/workspace/verification-artifacts/runs/20261013-025834-053c-map-f07" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025406-9fbb-map-f06/evidence/F06.ws-delete-close/delete.json", "/workspace/verification-artifacts/runs/20261013-025406-9fbb-map-f06/evidence/F06.ws-delete-close/socket.json", "/workspace/verification-artifacts/runs/20261013-025406-9fbb-map-f06/evidence/map-run/F06-conversation-events-06-deleting-a-conversation-closes-its-socke.json", "/workspace/verification-artifacts/runs/20261013-025528-20b8-map-f06/evidence/F06.ws-delete-close/delete.json", "/workspace/verification-artifacts/runs/20261013-025528-20b8-map-f06/evidence/F06.ws-delete-close/socket.json", "/workspace/verification-artifacts/runs/20261013-025528-20b8-map-f06/evidence/map-run/F06-conversation-events-06-deleting-a-conversation-closes-its-socke.json", "/workspace/verification-artifacts/runs/20261013-025721-700e-map-f07/evidence/F07.deleted-close/delete-5b5a3419-e47a-4af2-979e-a7365d32f6f5.json", "/workspace/verification-artifacts/runs/20261013-025721-700e-map-f07/evidence/F07.deleted-close/socket-5b5a3419-e47a-4af2-979e-a7365d32f6f5.json", "/workspace/verification-artifacts/runs/20261013-025721-700e-map-f07/evidence/map-run/F07-session-socket-02-deleting-closes-the-socket-f07-deleted-c.json", "/workspace/verification-artifacts/runs/20261013-025834-053c-map-f07/evidence/F07.deleted-close/delete-96a192c6-60d8-4692-b733-7a3c14b0d646.json", "/workspace/verification-artifacts/runs/20261013-025834-053c-map-f07/evidence/F07.deleted-close/socket-96a192c6-60d8-4692-b733-7a3c14b0d646.json", "/workspace/verification-artifacts/runs/20261013-025834-053c-map-f07/evidence/map-run/F07-session-socket-02-deleting-closes-the-socket-f07-deleted-c.json" ], "notes": "The legacy events and session protocols reproduce independently twice. Subscriber.close does not close the WebSocket, leaving connected clients waiting after successful DELETE. This is a socket/task leak; no OS process orphan is claimed." }, { "id": "F07.live-order", "title": "Out-of-order durable frames can make a session cursor lose a message", "severity": "high", "feature_ids": [ "F07.live-order" ], "expected": "The first user message yields durable seq 0,1,2 in order, matching persisted log order.", "actual": "The live socket emits 0,2,1: last_user_message_id is published before the user message at seq 1.", "repro_command": "control-agent-server map run --file F07 --only F07.live-order,F07.deleted-close --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-025721-700e-map-f07", "/workspace/verification-artifacts/runs/20261013-025834-053c-map-f07" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025721-700e-map-f07/evidence/F07.live-order/order-4b477523-fc4b-41e1-bd20-141a666eb5ce.json", "/workspace/verification-artifacts/runs/20261013-025721-700e-map-f07/evidence/map-run/F07-session-socket-01-live-frames-in-seq-order-f07-live-order-.json", "/workspace/verification-artifacts/runs/20261013-025834-053c-map-f07/evidence/F07.live-order/order-d06266ef-6639-4e3d-82e9-708fd2dd8bc7.json", "/workspace/verification-artifacts/runs/20261013-025834-053c-map-f07/evidence/map-run/F07-session-socket-01-live-frames-in-seq-order-f07-live-order-.json" ], "notes": "A client saving the highest sequence and disconnecting after 2 resumes past the missing message. Replay itself is ordered; the message remains on disk." }, { "id": "F08.secret-runtime-persistence", "title": "Runtime secret updates are not saved", "severity": "high", "feature_ids": [ "F08.secrets-visible", "F08.secrets-unsaved-restart", "F04.create-secrets-get" ], "expected": "GET immediately reflects an updated runtime secret and the secret survives restart.", "actual": "GET omits the new secret until another state save; restart before that save loses the credential.", "notes": "Merged: both observations originate from updating the secret registry without triggering a conversation state save. Not filed by this worker: external reporting and publication remain parent-owned. Aggregate merge: F04.create-secrets-get. Constructor and runtime secret updates mutate LocalConversation.update_secrets without triggering a state save.", "runs": [ "/workspace/verification-artifacts/runs/20261013-024737-a894-map-f08", "/workspace/verification-artifacts/runs/20261013-025113-2057-map-f08", "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024737-a894-map-f08/evidence/map-run/F08-confirmation-security-secrets-01-runtime-secret-visible-to-get-f08-secret.json", "/workspace/verification-artifacts/runs/20261013-024737-a894-map-f08/evidence/F08.secrets-visible/get.json", "/workspace/verification-artifacts/runs/20261013-024737-a894-map-f08/evidence/map-run/F08-confirmation-security-secrets-05-unsaved-secret-restart-f08-secrets-unsav.json", "/workspace/verification-artifacts/runs/20261013-024737-a894-map-f08/evidence/F08.secrets-unsaved-restart/get.json", "/workspace/verification-artifacts/runs/20261013-025113-2057-map-f08/evidence/map-run/F08-confirmation-security-secrets-01-runtime-secret-visible-to-get-f08-secret.json", "/workspace/verification-artifacts/runs/20261013-025113-2057-map-f08/evidence/F08.secrets-visible/get.json", "/workspace/verification-artifacts/runs/20261013-025113-2057-map-f08/evidence/map-run/F08-confirmation-security-secrets-05-unsaved-secret-restart-f08-secrets-unsav.json", "/workspace/verification-artifacts/runs/20261013-025113-2057-map-f08/evidence/F08.secrets-unsaved-restart/get.json", "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04/evidence/F04.create-secrets-get/get.json", "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04/evidence/map-run/F04-conversation-lifecycle-06-start-time-secret-visible-to-get-f04-cre.json", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04/evidence/F04.create-secrets-get/get.json", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04/evidence/map-run/F04-conversation-lifecycle-06-start-time-secret-visible-to-get-f04-cre.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F08 --only F08.secrets-visible,F08.secrets-unsaved-restart --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "merged_cases": [ { "id": "F04.create-secrets-get", "title": "Start-time secrets disappear from the immediate conversation GET", "severity": "medium", "feature_ids": [ "F04.create-secrets-get" ], "expected": "GET immediately lists the start-time StaticSecret that the create response lists, masked.", "actual": "Create lists QA_F 04_TOKEN; the immediate GET returns an empty secret_sources map.", "repro_command": "control-agent-server map run --file F04 --only F04.create-initial-message-run-false,F04.create-secrets-get,F04.create-secrets-plain,F04.batch-get-limit,F04.patch-whitespace-title,F04.patch-unknown-id,F04.delete-unknown-id --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04/evidence/F04.create-secrets-get/get.json", "/workspace/verification-artifacts/runs/20261013-024143-6004-map-f04/evidence/map-run/F04-conversation-lifecycle-06-start-time-secret-visible-to-get-f04-cre.json", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04/evidence/F04.create-secrets-get/get.json", "/workspace/verification-artifacts/runs/20261013-024259-9a86-map-f04/evidence/map-run/F04-conversation-lifecycle-06-start-time-secret-visible-to-get-f04-cre.json" ], "notes": "The secret exists in live memory but is absent from the autosaved base_state snapshot until a later state save." } ] }, { "id": "F08.secrets-bad-value", "title": "Non-string secret values return 500", "severity": "low", "feature_ids": [ "F08.secrets-bad-value" ], "expected": "Numbers, null and booleans are rejected with 422.", "actual": "Number, null, and boolean secret values all return 500 instead of 422 in both additional fresh sibling runs.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned. Number-only original confirmations are supplemented with separate saved exchanges for all three input shapes; no extra defect group is counted.", "runs": [ "/workspace/verification-artifacts/runs/20261013-024737-a894-map-f08", "/workspace/verification-artifacts/runs/20261013-025113-2057-map-f08" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024737-a894-map-f08/evidence/map-run/F08-confirmation-security-secrets-02-non-string-secret-value-f08-secrets-bad-.json", "/workspace/verification-artifacts/runs/20261013-024737-a894-map-f08/evidence/F08.secrets-bad-value/number.json", "/workspace/verification-artifacts/runs/20261013-025113-2057-map-f08/evidence/map-run/F08-confirmation-security-secrets-02-non-string-secret-value-f08-secrets-bad-.json", "/workspace/verification-artifacts/runs/20261013-025113-2057-map-f08/evidence/F08.secrets-bad-value/number.json", "/workspace/verification-artifacts/runs/20261013-034151-6d5f-map-f08/evidence/map-run/F08-confirmation-security-secrets-01-non-string-secret-value-f08-secrets-bad-.json", "/workspace/verification-artifacts/runs/20261013-034151-6d5f-map-f08/evidence/F08.secrets-bad-value/number.json", "/workspace/verification-artifacts/runs/20261013-034151-6d5f-map-f08/evidence/F08.secrets-bad-value/null.json", "/workspace/verification-artifacts/runs/20261013-034151-6d5f-map-f08/evidence/F08.secrets-bad-value/bool.json", "/workspace/verification-artifacts/runs/20261013-034314-5b2d-map-f08/evidence/map-run/F08-confirmation-security-secrets-01-non-string-secret-value-f08-secrets-bad-.json", "/workspace/verification-artifacts/runs/20261013-034314-5b2d-map-f08/evidence/F08.secrets-bad-value/number.json", "/workspace/verification-artifacts/runs/20261013-034314-5b2d-map-f08/evidence/F08.secrets-bad-value/null.json", "/workspace/verification-artifacts/runs/20261013-034314-5b2d-map-f08/evidence/F08.secrets-bad-value/bool.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F08 --only F08.secrets-bad-value --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F09.cached-switch", "title": "Switch requests reuse stale registered LLM configuration", "severity": "medium", "feature_ids": [ "F09.switch-profile-edited", "F09.switch-llm-registered-usage-id" ], "expected": "A switch installs posted or current profile configuration, or rejects the request.", "actual": "A 200 success silently retains an older LLM when its usage_id was registered earlier; editing a saved profile is ignored on later switch.", "notes": "Merged: both variants use the first-write-wins LLM registry and silently reuse the previous registration. Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/map-run/F09-model-switching-plugins-01-switch-to-an-edited-profile-f09-switch-p.json", "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/F09.switch-profile-edited/get.json", "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/F09.switch-profile-edited/switch-again.json", "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/map-run/F09-model-switching-plugins-02-switch-onto-a-registered-usage-id-f09-sw.json", "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/F09.switch-llm-registered-usage-id/get.json", "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/F09.switch-llm-registered-usage-id/switch.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-01-switch-to-an-edited-profile-f09-switch-p.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/F09.switch-profile-edited/get.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/F09.switch-profile-edited/switch-again.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-02-switch-onto-a-registered-usage-id-f09-sw.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/F09.switch-llm-registered-usage-id/get.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/F09.switch-llm-registered-usage-id/switch.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F09 --only F09.switch-profile-edited,F09.switch-llm-registered-usage-id --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F09.switch-llm-before-first-run", "title": "Switching before first run loses usage accounting", "severity": "low", "feature_ids": [ "F09.switch-llm-before-first-run" ], "expected": "The successful switched model call appears under its usage_id in stats.", "actual": "The run finishes but switched usage has no token or cost entry.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/map-run/F09-model-switching-plugins-03-switch-before-the-first-run-f09-switch-l.json", "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/F09.switch-llm-before-first-run/stats.json", "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/F09.switch-llm-before-first-run/switch.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-03-switch-before-the-first-run-f09-switch-l.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/F09.switch-llm-before-first-run/stats.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/F09.switch-llm-before-first-run/switch.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F09 --only F09.switch-llm-before-first-run --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F09.switch-llm-mid-run-get", "title": "GET omits an acknowledged mid-run model switch", "severity": "medium", "feature_ids": [ "F09.switch-llm-mid-run-get" ], "expected": "GET and persisted state reflect the acknowledged LLM switch.", "actual": "The acknowledged switch appears immediately on the WebSocket, but GET still reports the previous default usage_id while the tool step is running, in both fresh repeats.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned. The following contemporaneous base_state assertion was not reached; persisted-state loss and loss on crash are not claimed.", "runs": [ "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/map-run/F09-model-switching-plugins-05-get-right-after-a-mid-run-switch-f09-swi.json", "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/F09.switch-llm-mid-run-get/frames.json", "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/F09.switch-llm-mid-run-get/get.json", "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/F09.switch-llm-mid-run-get/switch.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-05-get-right-after-a-mid-run-switch-f09-swi.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/F09.switch-llm-mid-run-get/frames.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/F09.switch-llm-mid-run-get/get.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/F09.switch-llm-mid-run-get/switch.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F09 --only F09.switch-llm-mid-run-get --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F09.acp-model-blank", "title": "Pre-run ACP model switch accepts whitespace", "severity": "low", "feature_ids": [ "F09.acp-model-blank" ], "expected": "Blank model ids are rejected and the prior model remains.", "actual": "Whitespace is accepted with 200 before the ACP session exists.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/map-run/F09-model-switching-plugins-07-blank-acp-model-f09-acp-model-blank-know.json", "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/F09.acp-model-blank/blank.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-07-blank-acp-model-f09-acp-model-blank-know.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/F09.acp-model-blank/blank.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F09 --only F09.acp-model-blank --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F09.switch-llm-acp-refused", "title": "Native model switch endpoints alter ACP placeholder LLM", "severity": "medium", "feature_ids": [ "F09.switch-llm-acp-refused" ], "expected": "switch_llm and switch_profile reject ACP conversations and retain acp-managed config.", "actual": "The endpoints accept and replace the ACP agent placeholder LLM.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/map-run/F09-model-switching-plugins-08-llm-switches-on-an-acp-conversation-f09-.json", "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/F09.switch-llm-acp-refused/switch-llm.json", "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/F09.switch-llm-acp-refused/switch-profile.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-08-llm-switches-on-an-acp-conversation-f09-.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/F09.switch-llm-acp-refused/switch-llm.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/F09.switch-llm-acp-refused/switch-profile.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F09 --only F09.switch-llm-acp-refused --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F09.acp-model-live-unknown", "title": "Unsupported ACP model id returns 500", "severity": "low", "feature_ids": [ "F09.acp-model-live-unknown" ], "expected": "A model id absent from available_models is rejected with a client error.", "actual": "The Claude ACP adapter model switch surfaces an internal 500.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/map-run/F09-model-switching-plugins-10-unknown-model-on-a-live-session-f09-acp-.json", "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/F09.acp-model-live-unknown/switch.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-10-unknown-model-on-a-live-session-f09-acp-.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/F09.acp-model-live-unknown/switch.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F09 --only F09.acp-model-live-unknown --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F09.ts-client-acp-docs", "title": "TypeScript ACP switch docs promise an absent 409", "severity": "low", "feature_ids": [ "F09.ts-client-acp-docs" ], "expected": "Docs explain that a pre-run switch is stored and returns 200.", "actual": "Both switchAcpModel docstrings promise 409 before first message.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/map-run/F09-model-switching-plugins-11-typescript-docstrings-for-a-pre-run-acp-.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-11-typescript-docstrings-for-a-pre-run-acp-.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F09 --only F09.ts-client-acp-docs --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F09.load-plugin-twice", "title": "Loading a plugin twice duplicates hook side effects", "severity": "medium", "feature_ids": [ "F09.load-plugin-twice" ], "expected": "Repeated plugin loading keeps one copy of its hooks.", "actual": "Skills stay deduplicated but identical user_prompt_submit hooks are appended twice.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/map-run/F09-model-switching-plugins-14-load-the-same-plugin-twice-f09-load-plug.json", "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/F09.load-plugin-twice/get.json", "/workspace/verification-artifacts/runs/20261013-025803-aa01-map-f09/evidence/F09.load-plugin-twice/second-load.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/map-run/F09-model-switching-plugins-14-load-the-same-plugin-twice-f09-load-plug.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/F09.load-plugin-twice/get.json", "/workspace/verification-artifacts/runs/20261013-030236-b017-map-f09/evidence/F09.load-plugin-twice/second-load.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F09 --only F09.load-plugin-twice --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F10.fork-tags-consistent", "title": "Fork tags differ between state and stored metadata", "severity": "low", "feature_ids": [ "F10.fork-tags-consistent" ], "expected": "GET and meta.json agree on inherited tags.", "actual": "GET drops source qa=src while meta.json retains it.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-024131-0603-map-f10", "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024131-0603-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-05-fork-tags-agree-with-the-stored-metadata.json", "/workspace/verification-artifacts/runs/20261013-024131-0603-map-f10/evidence/F10.fork-tags-consistent/fork.json", "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-05-fork-tags-agree-with-the-stored-metadata.json", "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/F10.fork-tags-consistent/fork.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F10 --only F10.fork-tags-consistent --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F10.fork-credentials", "title": "Fork drops model credentials and cannot continue", "severity": "high", "feature_ids": [ "F10.fork-keeps-credentials", "F10.fork-run-memory" ], "expected": "The fork retains encrypted agent and condenser keys and can run with source history.", "actual": "Fork base_state has no API key; continuing a DeepSeek fork fails authentication.", "notes": "Merged: failed continuation follows from the same missing cipher when fork state is saved. Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-024131-0603-map-f10", "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024131-0603-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-06-the-fork-keeps-the-model-keys-f10-fork-k.json", "/workspace/verification-artifacts/runs/20261013-024131-0603-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-07-a-fork-runs-with-the-source-s-memory-f10.json", "/workspace/verification-artifacts/runs/20261013-024131-0603-map-f10/evidence/F10.fork-run-memory/auth-errors.json", "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-06-the-fork-keeps-the-model-keys-f10-fork-k.json", "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-07-a-fork-runs-with-the-source-s-memory-f10.json", "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/F10.fork-run-memory/auth-errors.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F10 --only F10.fork-keeps-credentials,F10.fork-run-memory --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F10.navigate-final-response", "title": "Final-response endpoint returns abandoned branch answer", "severity": "medium", "feature_ids": [ "F10.navigate-final-response" ], "expected": "Navigating HEAD before the answer makes the active-branch final response empty.", "actual": "agent_final_response still returns the abandoned answer.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-024131-0603-map-f10", "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024131-0603-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-09-final-response-after-navigating-back-f10.json", "/workspace/verification-artifacts/runs/20261013-024131-0603-map-f10/evidence/F10.navigate-final-response/after-navigate.json", "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-09-final-response-after-navigating-back-f10.json", "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/F10.navigate-final-response/after-navigate.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F10 --only F10.navigate-final-response --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F10.condense-status", "title": "Disabled condenser is reported as an internal server error", "severity": "low", "feature_ids": [ "F10.condense-status" ], "expected": "Missing or no-op condenser is a client configuration error.", "actual": "The disabled condenser path returns an unhandled 500 with traceback in both fresh repeats.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned. The separate no-op condenser assertion was not reached after the disabled-case failure.", "runs": [ "/workspace/verification-artifacts/runs/20261013-024131-0603-map-f10", "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024131-0603-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-10-refusal-status-f10-condense-status-known.json", "/workspace/verification-artifacts/runs/20261013-024131-0603-map-f10/evidence/F10.condense-status/disabled.json", "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-10-refusal-status-f10-condense-status-known.json", "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/F10.condense-status/disabled.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F10 --only F10.condense-status --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F10.ask-agent-unknown", "title": "Side question on unknown conversation returns 500", "severity": "low", "feature_ids": [ "F10.ask-agent-unknown" ], "expected": "Unknown conversation is documented 404.", "actual": "ask_agent returns internal 500.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-024131-0603-map-f10", "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024131-0603-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-11-unknown-conversation-f10-ask-agent-unkno.json", "/workspace/verification-artifacts/runs/20261013-024131-0603-map-f10/evidence/F10.ask-agent-unknown/unknown.json", "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/map-run/F10-fork-navigate-condense-ask-11-unknown-conversation-f10-ask-agent-unkno.json", "/workspace/verification-artifacts/runs/20261013-024315-4f15-map-f10/evidence/F10.ask-agent-unknown/unknown.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F10 --only F10.ask-agent-unknown --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F11.goal-cap-overrun", "title": "Goal resume runs beyond max_iterations", "severity": "medium", "feature_ids": [ "F11.goal-cap-overrun" ], "expected": "No goal status reports iteration above max_iterations.", "actual": "Resuming after failed last judge reports capped 2 of 1 and performs another round.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-031634-2620-map-f11", "/workspace/verification-artifacts/runs/20261013-031830-28b2-map-f11" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-031634-2620-map-f11/evidence/map-run/F11-goals-02-a-resume-after-a-failed-last-audit-overr.json", "/workspace/verification-artifacts/runs/20261013-031634-2620-map-f11/evidence/F11.goal-cap-overrun/goal-events.json", "/workspace/verification-artifacts/runs/20261013-031634-2620-map-f11/evidence/F11.goal-cap-overrun/resume.json", "/workspace/verification-artifacts/runs/20261013-031830-28b2-map-f11/evidence/map-run/F11-goals-02-a-resume-after-a-failed-last-audit-overr.json", "/workspace/verification-artifacts/runs/20261013-031830-28b2-map-f11/evidence/F11.goal-cap-overrun/goal-events.json", "/workspace/verification-artifacts/runs/20261013-031830-28b2-map-f11/evidence/F11.goal-cap-overrun/resume.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F11 --only F11.goal-cap-overrun --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F11.goal-restart-active", "title": "Restart leaves an inactive goal marked running", "severity": "medium", "feature_ids": [ "F11.goal-restart-active" ], "expected": "A loop cancelled by restart is reported interrupted and inactive.", "actual": "Conversation restores paused but its latest goal status remains active/running.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-031634-2620-map-f11", "/workspace/verification-artifacts/runs/20261013-031830-28b2-map-f11" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-031634-2620-map-f11/evidence/map-run/F11-goals-03-a-loop-active-at-a-restart-stays-running.json", "/workspace/verification-artifacts/runs/20261013-031634-2620-map-f11/evidence/F11.goal-restart-active/after-restart.json", "/workspace/verification-artifacts/runs/20261013-031634-2620-map-f11/evidence/F11.goal-restart-active/goal-events.json", "/workspace/verification-artifacts/runs/20261013-031830-28b2-map-f11/evidence/map-run/F11-goals-03-a-loop-active-at-a-restart-stays-running.json", "/workspace/verification-artifacts/runs/20261013-031830-28b2-map-f11/evidence/F11.goal-restart-active/after-restart.json", "/workspace/verification-artifacts/runs/20261013-031830-28b2-map-f11/evidence/F11.goal-restart-active/goal-events.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F11 --only F11.goal-restart-active --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F12.sandbox-pause-websocket", "title": "Sandbox pause leaves open conversation sockets silent", "severity": "medium", "feature_ids": [ "F12.sandbox-pause-websocket" ], "expected": "Existing sockets close or continue delivering rehydrated conversation events.", "actual": "An open socket remains bound to the unloaded service and receives no later new events.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned. Both old events/session captures and their fresh controls were saved before the first failed assertion; both old streams lack the post-pause message and remote-close frame.", "runs": [ "/workspace/verification-artifacts/runs/20261013-025756-a01d-map-f12", "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025756-a01d-map-f12/evidence/map-run/F12-runtime-and-credentials-01-open-sockets-across-a-pause-f12-sandbox-.json", "/workspace/verification-artifacts/runs/20261013-025756-a01d-map-f12/evidence/F12.sandbox-pause-websocket/fresh-session-socket.json", "/workspace/verification-artifacts/runs/20261013-025756-a01d-map-f12/evidence/F12.sandbox-pause-websocket/fresh-socket.json", "/workspace/verification-artifacts/runs/20261013-025756-a01d-map-f12/evidence/F12.sandbox-pause-websocket/old-session-socket.json", "/workspace/verification-artifacts/runs/20261013-025756-a01d-map-f12/evidence/F12.sandbox-pause-websocket/old-socket.json", "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12/evidence/map-run/F12-runtime-and-credentials-01-open-sockets-across-a-pause-f12-sandbox-.json", "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12/evidence/F12.sandbox-pause-websocket/fresh-session-socket.json", "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12/evidence/F12.sandbox-pause-websocket/fresh-socket.json", "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12/evidence/F12.sandbox-pause-websocket/old-session-socket.json", "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12/evidence/F12.sandbox-pause-websocket/old-socket.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F12 --only F12.sandbox-pause-websocket --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F12.binding-native-cold", "title": "Pending file-credential binding poisons a native conversation", "severity": "medium", "feature_ids": [ "F12.binding-native-cold" ], "expected": "A cold native conversation never acquires an ACP credential-binding requirement.", "actual": "Binding while native conversation is unloaded creates a persistent requirement that blocks later loading after pause.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned. The post-pause API loading failure is proved; the later meta.json length assertion was not reached.", "runs": [ "/workspace/verification-artifacts/runs/20261013-025756-a01d-map-f12", "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025756-a01d-map-f12/evidence/map-run/F12-runtime-and-credentials-02-regular-conversation-not-loaded-f12-bind.json", "/workspace/verification-artifacts/runs/20261013-025756-a01d-map-f12/evidence/F12.binding-native-cold/events-after-pause.json", "/workspace/verification-artifacts/runs/20261013-025756-a01d-map-f12/evidence/F12.binding-native-cold/put.json", "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12/evidence/map-run/F12-runtime-and-credentials-02-regular-conversation-not-loaded-f12-bind.json", "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12/evidence/F12.binding-native-cold/events-after-pause.json", "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12/evidence/F12.binding-native-cold/put.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F12 --only F12.binding-native-cold --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F12.managed-key-switch", "title": "Model switch loses managed API key refresh hook", "severity": "medium", "feature_ids": [ "F12.managed-key-switch" ], "expected": "An in-scope switched LLM retries a 401 after fetching managed key.", "actual": "First run refreshes; after switch the 401 surfaces, no second fetch or retry, conversation errors.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned. The pre-assertion wait prints error after the switch in both repeats, while the source sink count stays 1; the final finished-state GET assertion was not reached.", "runs": [ "/workspace/verification-artifacts/runs/20261013-025756-a01d-map-f12", "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025756-a01d-map-f12/evidence/map-run/F12-runtime-and-credentials-05-refresh-after-a-model-switch-f12-managed.json", "/workspace/verification-artifacts/runs/20261013-025756-a01d-map-f12/evidence/F12.managed-key-switch/key-fetch.json", "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12/evidence/map-run/F12-runtime-and-credentials-05-refresh-after-a-model-switch-f12-managed.json", "/workspace/verification-artifacts/runs/20261013-030529-8df8-map-f12/evidence/F12.managed-key-switch/key-fetch.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F12 --only F12.managed-key-switch --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F13.timestamp-offset", "title": "Bash timestamp filters compare local spelling rather than instant", "severity": "medium", "feature_ids": [ "F13.timestamp-offset" ], "expected": "Equivalent UTC and non-UTC offset timestamps select the same events.", "actual": "UTC selects the command while equivalent +02:00 timestamp selects none.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-031029-c530-map-f13", "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-031029-c530-map-f13/evidence/map-run/F13-bash-01-offset-timestamps-f13-timestamp-offset-k.json", "/workspace/verification-artifacts/runs/20261013-031029-c530-map-f13/evidence/F13.timestamp-offset/plus-two.json", "/workspace/verification-artifacts/runs/20261013-031029-c530-map-f13/evidence/F13.timestamp-offset/utc.json", "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13/evidence/map-run/F13-bash-01-offset-timestamps-f13-timestamp-offset-k.json", "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13/evidence/F13.timestamp-offset/plus-two.json", "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13/evidence/F13.timestamp-offset/utc.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F13 --only F13.timestamp-offset --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F13.order-gt-paging", "title": "Bash output paging filters order after page limit", "severity": "medium", "feature_ids": [ "F13.order-gt-paging" ], "expected": "limit=1 with order__gt=0 returns the next output chunk.", "actual": "The result is empty with a next_page_id; SDK pollers can miss later chunks and time out.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-031029-c530-map-f13", "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-031029-c530-map-f13/evidence/map-run/F13-bash-03-order-filter-and-paging-f13-order-gt-pag.json", "/workspace/verification-artifacts/runs/20261013-031029-c530-map-f13/evidence/F13.order-gt-paging/limit-1.json", "/workspace/verification-artifacts/runs/20261013-031029-c530-map-f13/evidence/F13.order-gt-paging/limit-2.json", "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13/evidence/map-run/F13-bash-03-order-filter-and-paging-f13-order-gt-pag.json", "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13/evidence/F13.order-gt-paging/limit-1.json", "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13/evidence/F13.order-gt-paging/limit-2.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F13 --only F13.order-gt-paging --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F13.execute-large-output", "title": "Synchronous bash execution truncates output above 1 MiB", "severity": "medium", "feature_ids": [ "F13.execute-large-output" ], "expected": "execute_bash_command returns all 1100012 stdout characters.", "actual": "It returns only the final chunk, losing the first 1048576 characters.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-031029-c530-map-f13", "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-031029-c530-map-f13/evidence/map-run/F13-bash-04-large-output-through-execute-f13-execute.json", "/workspace/verification-artifacts/runs/20261013-031029-c530-map-f13/evidence/F13.execute-large-output/execute.json", "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13/evidence/map-run/F13-bash-04-large-output-through-execute-f13-execute.json", "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13/evidence/F13.execute-large-output/execute.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F13 --only F13.execute-large-output --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F13.scoped-bad-id", "title": "Malformed scoped bash conversation id produces 500", "severity": "low", "feature_ids": [ "F13.scoped-bad-id" ], "expected": "A non-UUID conversation id is rejected with 422.", "actual": "The dependency reads unset runtime_event_service and produces 500.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-031029-c530-map-f13", "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-031029-c530-map-f13/evidence/map-run/F13-bash-05-malformed-conversation-id-f13-scoped-bad.json", "/workspace/verification-artifacts/runs/20261013-031029-c530-map-f13/evidence/F13.scoped-bad-id/execute.json", "/workspace/verification-artifacts/runs/20261013-031029-c530-map-f13/evidence/F13.scoped-bad-id/search.json", "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13/evidence/map-run/F13-bash-05-malformed-conversation-id-f13-scoped-bad.json", "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13/evidence/F13.scoped-bad-id/execute.json", "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13/evidence/F13.scoped-bad-id/search.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F13 --only F13.scoped-bad-id --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F13.scoped-idle-eviction", "title": "Idle eviction kills a running scoped bash command", "severity": "high", "feature_ids": [ "F13.scoped-idle-eviction" ], "expected": "Running scoped commands keep their conversation loaded and can be stopped normally.", "actual": "Idle eviction kills the active scoped command and leaves the retained stream without any terminal BashOutput through shutdown 19\u201320 seconds after eviction. Consumers expecting an exit code may wait until their own timeout.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned. The independent terminal-event evidence checks both retained owned BashCommand streams. No actual client poller wait, rehydration, or later stop API was exercised. See f13-idle-eviction-terminal-evidence.json.", "runs": [ "/workspace/verification-artifacts/runs/20261013-031029-c530-map-f13", "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-031029-c530-map-f13/evidence/map-run/F13-bash-06-idle-eviction-kills-a-running-command-f1.json", "/workspace/verification-artifacts/runs/20261013-031229-ccc9-map-f13/evidence/map-run/F13-bash-06-idle-eviction-kills-a-running-command-f1.json", "/workspace/verification-artifacts/f13-idle-eviction-terminal-evidence.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F13 --only F13.scoped-idle-eviction --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F14.upload-onto-directory", "title": "Invalid upload target is reported as server failure", "severity": "low", "feature_ids": [ "F14.upload-onto-directory" ], "expected": "Directory target or parent-file collision yields 4xx.", "actual": "Upload maps both collisions to a masked 500.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned. Two additional fresh sibling runs save 500 responses for both existing-directory and parent-is-file collisions.", "runs": [ "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14/evidence/map-run/F14-files-02-upload-onto-a-directory-f14-upload-onto-.json", "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14/evidence/F14.upload-onto-directory/upload.json", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/map-run/F14-files-02-upload-onto-a-directory-f14-upload-onto-.json", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/F14.upload-onto-directory/upload.json", "/workspace/verification-artifacts/runs/20261013-034147-e66e-map-f14/evidence/map-run/F14-files-01-upload-onto-a-directory-f14-upload-onto-.json", "/workspace/verification-artifacts/runs/20261013-034147-e66e-map-f14/evidence/F14.upload-onto-directory/upload.json", "/workspace/verification-artifacts/runs/20261013-034147-e66e-map-f14/evidence/F14.upload-onto-directory/parent-is-file.json", "/workspace/verification-artifacts/runs/20261013-034324-233c-map-f14/evidence/map-run/F14-files-01-upload-onto-a-directory-f14-upload-onto-.json", "/workspace/verification-artifacts/runs/20261013-034324-233c-map-f14/evidence/F14.upload-onto-directory/upload.json", "/workspace/verification-artifacts/runs/20261013-034324-233c-map-f14/evidence/F14.upload-onto-directory/parent-is-file.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F14 --only F14.upload-onto-directory --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F14.upload-size", "title": "Remote upload results omit file_size in Python and TypeScript", "severity": "low", "feature_ids": [ "F14.upload-size" ], "expected": "Both remote clients report the 70000-byte upload size, as LocalWorkspace does.", "actual": "Bytes upload intact; Python returns None and TypeScript returns undefined for file_size.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14/evidence/map-run/F14-files-03-upload-size-in-the-consumers-result-f14-.json", "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14/evidence/F14.upload-size/rest.json", "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14/evidence/F14.upload-size/sdk.json", "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14/evidence/F14.upload-size/ts.json", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/map-run/F14-files-03-upload-size-in-the-consumers-result-f14-.json", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/F14.upload-size/rest.json", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/F14.upload-size/sdk.json", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/F14.upload-size/ts.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F14 --only F14.upload-size --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F14.upload-atomic", "title": "Failed overwrite destroys the previous file", "severity": "high", "feature_ids": [ "F14.upload-atomic" ], "expected": "Mid-write EFBIG failure leaves the previous uploaded file intact.", "actual": "The existing target is truncated and replaced by 500000 partial bytes despite 500.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14/evidence/map-run/F14-files-04-failed-overwrite-f14-upload-atomic-known.json", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/map-run/F14-files-04-failed-overwrite-f14-upload-atomic-known.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F14 --only F14.upload-atomic --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F14.subdirs-case-pagination", "title": "Case-only sibling directory names loop pagination", "severity": "medium", "feature_ids": [ "F14.subdirs-case-pagination" ], "expected": "Pagination over Foo and foo reaches both directories.", "actual": "The lowercased cursor repeats the first page indefinitely.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14/evidence/map-run/F14-files-05-paging-past-a-case-only-twin-f14-subdirs.json", "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14/evidence/F14.subdirs-case-pagination/page-2.json", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/map-run/F14-files-05-paging-past-a-case-only-twin-f14-subdirs.json", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/F14.subdirs-case-pagination/page-2.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F14 --only F14.subdirs-case-pagination --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F14.archive-format-doc", "title": "Archive format description omits nested-repository selection", "severity": "low", "feature_ids": [ "F14.archive-format-doc" ], "expected": "The OpenAPI format description matches archive contents.", "actual": "It promises the entire directory but a single nested repository excludes adjacent top.txt.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14/evidence/map-run/F14-files-06-what-format-promises-f14-archive-format-.json", "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14/evidence/F14.archive-format-doc/tar.json", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/map-run/F14-files-06-what-format-promises-f14-archive-format-.json", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/F14.archive-format-doc/tar.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F14 --only F14.archive-format-doc --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F14.trajectory-concurrent", "title": "Concurrent trajectory exports share and corrupt one temporary ZIP", "severity": "medium", "feature_ids": [ "F14.trajectory-concurrent" ], "expected": "All concurrent downloads return 200 and valid ZIP bytes.", "actual": "Shared temporary path is overwritten/unlinked across handlers, causing broken downloads and 500s.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned.", "runs": [ "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14/evidence/map-run/F14-files-08-concurrent-trajectory-downloads-f14-traj.json", "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14/evidence/F14.trajectory-concurrent/burst.json", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/map-run/F14-files-08-concurrent-trajectory-downloads-f14-traj.json", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/F14.trajectory-concurrent/burst.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F14 --only F14.trajectory-concurrent --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "F14.trajectory-init-path", "title": "Trajectory export ignores init-time conversations_path", "severity": "medium", "feature_ids": [ "F14.trajectory-init-path" ], "expected": "Both trajectory routes export the conversation stored at the initialized path.", "actual": "GET finds the conversation but trajectory export reads the launch-time default and returns 404.", "notes": "Not filed by this worker: external reporting and publication remain parent-owned. Two additional fresh sibling runs save 404 responses from both the global and scoped export routes after custom init-time conversation storage was configured.", "runs": [ "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024217-af4c-map-f14/evidence/map-run/F14-files-11-init-time-conversations-path-f14-traject.json", "/workspace/verification-artifacts/runs/20261013-024649-c36e-map-f14/evidence/map-run/F14-files-11-init-time-conversations-path-f14-traject.json", "/workspace/verification-artifacts/runs/20261013-034147-e66e-map-f14/evidence/map-run/F14-files-02-init-time-conversations-path-f14-traject.json", "/workspace/verification-artifacts/runs/20261013-034241-a692-f14-init/evidence/F14.trajectory-init-path/global.json", "/workspace/verification-artifacts/runs/20261013-034241-a692-f14-init/evidence/F14.trajectory-init-path/scoped.json", "/workspace/verification-artifacts/runs/20261013-034324-233c-map-f14/evidence/map-run/F14-files-02-init-time-conversations-path-f14-traject.json", "/workspace/verification-artifacts/runs/20261013-034407-9964-f14-init/evidence/F14.trajectory-init-path/global.json", "/workspace/verification-artifacts/runs/20261013-034407-9964-f14-init/evidence/F14.trajectory-init-path/scoped.json" ], "repro_command": "PATH=/workspace/verification-artifacts/f08-f14-harness/.agents/skills/verify-agent-server/scripts:$PATH control-agent-server map run --file F14 --only F14.trajectory-init-path --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going" }, { "id": "git-parent-changes", "title": "Changes misses a repository below the requested workspace directory", "severity": "medium", "feature_ids": [ "F15.changes-repo-below-path" ], "expected": "A plain workspace directory containing one repository returns its two changes prefixed with proj/.", "actual": "HTTP 200 [] although GET for the nested repository returns UPDATED README.md and ADDED notes.txt.", "repro_command": "control-agent-server map run --file F15 --only F15.changes-repo-below-path --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-022138-1cb7-map-f15", "/workspace/verification-artifacts/runs/20261013-022337-0fbc-map-f15" ], "evidence": [ "20261013-022138-1cb7-map-f15/evidence/F15.changes-repo-below-path/ws.json", "20261013-022337-0fbc-map-f15/evidence/F15.changes-repo-below-path/ws.json" ], "notes": "Known map bug, independently reproduced twice. Typical repository-backed workspace layout." }, { "id": "git-filename-space", "title": "Changes rejects tracked filenames with spaces and mangles non-ASCII filenames", "severity": "medium", "feature_ids": [ "F15.changes-special-names" ], "expected": "HTTP 200 [{status: UPDATED, path: my notes.md}] with the real filename.", "actual": "A modified tracked my notes.md makes the entire Changes response HTTP400: Unexpected git diff output format: M\\tmy notes.md. In both supplemental fresh runs a modified caf\u00e9.txt is returned as the quoted/mangled path \"caf/303/251.txt\" instead of its real filename.", "repro_command": "control-agent-server map run --file F15 --only F15.changes-special-names --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-022138-1cb7-map-f15", "/workspace/verification-artifacts/runs/20261013-022337-0fbc-map-f15" ], "evidence": [ "20261013-022138-1cb7-map-f15/evidence/F15.changes-special-names/space.json", "20261013-022337-0fbc-map-f15/evidence/F15.changes-special-names/space.json", "20261013-031006-7188-map-f15/evidence/F15.changes-special-names/non-ascii.json", "20261013-031237-2ccf-map-f15/evidence/F15.changes-special-names/non-ascii.json" ], "notes": "The original space case is confirmed twice. Copied recipes preserve the same assertions while continuing through sibling cases, and two further fresh runs independently prove the non-ASCII branch. Supplemental command and adaptations: F15-recipe-adaptation.md; family-results/F15-supplemental.json." }, { "id": "git-subdirectory-paths", "title": "Changes returns tracked and untracked paths on inconsistent bases when querying a repository subdirectory", "severity": "medium", "feature_ids": [ "F15.changes-subdir-paths" ], "expected": "Every returned path resolves against one consistent base (requested src directory or repository root).", "actual": "HTTP 200 [UPDATED README.md, ADDED new.py, UPDATED src/app.py]; no single base resolves all returned paths, and README.md is outside the requested src directory.", "repro_command": "control-agent-server map run --file F15 --only F15.changes-subdir-paths --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-022138-1cb7-map-f15", "/workspace/verification-artifacts/runs/20261013-022337-0fbc-map-f15" ], "evidence": [ "20261013-022138-1cb7-map-f15/evidence/F15.changes-subdir-paths/src.json", "20261013-022337-0fbc-map-f15/evidence/F15.changes-subdir-paths/src.json" ], "notes": "Known map bug. Fixture repository-root positive control returns all four changes with consistent paths." }, { "id": "git-whitespace-diff", "title": "An unchanged indented file has a phantom diff because original text loses indentation", "severity": "medium", "feature_ids": [ "F15.diff-whitespace" ], "expected": "An unchanged file has identical original and modified text.", "actual": "An unchanged indented file returns original indented line\\nbody without its two leading spaces, while modified retains them. Both supplemental fresh runs also return original body and modified body followed by two newlines for the unchanged trailing-blank-line file. Changes lists neither unchanged file.", "repro_command": "control-agent-server map run --file F15 --only F15.diff-whitespace --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-022138-1cb7-map-f15", "/workspace/verification-artifacts/runs/20261013-022337-0fbc-map-f15" ], "evidence": [ "20261013-022138-1cb7-map-f15/evidence/F15.diff-whitespace/lead.json", "20261013-022337-0fbc-map-f15/evidence/F15.diff-whitespace/lead.json", "20261013-031006-7188-map-f15/evidence/F15.diff-whitespace/trail.json", "20261013-031237-2ccf-map-f15/evidence/F15.diff-whitespace/trail.json" ], "notes": "Leading-indentation failure confirmed twice in the original recipe. CRLF positive controls pass. Copied recipe continues after the indentation assertion and independently proves trailing-blank-line loss in two further fresh runs." }, { "id": "git-deleted-diff", "title": "Diff cannot read a working-tree file that Changes correctly lists as deleted", "severity": "medium", "feature_ids": [ "F15.diff-deleted-file" ], "expected": "HTTP 200 original from HEAD and modified empty string for a tracked deleted file.", "actual": "Changes lists DELETED src/util.py, but Diff answers HTTP 400 'File does not exist: .../src/util.py'.", "repro_command": "control-agent-server map run --file F15 --only F15.diff-deleted-file --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-022138-1cb7-map-f15", "/workspace/verification-artifacts/runs/20261013-022337-0fbc-map-f15" ], "evidence": [ "20261013-022138-1cb7-map-f15/evidence/F15.diff-deleted-file/diff.json", "20261013-022337-0fbc-map-f15/evidence/F15.diff-deleted-file/diff.json" ], "notes": "Known map bug. Deletion-list positive control reaches the file before diff failure." }, { "id": "llm-profile-store-stalls-server", "title": "Waiting for the LLM profile store lock stalls unrelated server requests", "severity": "high", "feature_ids": [ "F20.store-busy-responsive" ], "expected": "GET /alive must answer within 3000ms while a profile save waits for the store lock, as it does with no lock.", "actual": "GET /alive returned 200 only after 25,771ms in the initial run and 23,523ms in the clean run. In both runs the waiting save returned 503 after approximately 30 seconds, no profile was created, and the exact timing assertion failed.", "repro_command": "control-agent-server map run --file F20 --only F20.store-busy-responsive --fresh --repeat 2 --record --keep-going --checkout /workspace/agent-sdk-verify", "runs": [ "/workspace/verification-artifacts/runs/20261013-021919-4f6c-map-f20", "/workspace/verification-artifacts/runs/20261013-024525-b13b-map-f20" ], "evidence": [ "20261013-021919-4f6c-map-f20/evidence/F20.store-busy-responsive/alive.json", "20261013-021919-4f6c-map-f20/fixtures/qa-f20/busy-save.json", "20261013-021919-4f6c-map-f20/evidence/map-run/F20-llm-profiles-11-lock-wait-blocks-the-server-f20-store-bu.json", "20261013-024525-b13b-map-f20/evidence/F20.store-busy-responsive/alive.json", "20261013-024525-b13b-map-f20/fixtures/qa-f20/busy-save.json", "20261013-024525-b13b-map-f20/evidence/map-run/F20-llm-profiles-11-lock-wait-blocks-the-server-f20-store-bu.json" ], "notes": "Already independently confirmed twice and present in triage-f15-f21.json. Initial evidence occurred before the later ENOSPC interruption and has matching saved controls and failed assertion. Related F21 busy-lock case is listed separately pending its own repeated confirmation. No rerun or shared triage mutation was performed during extraction.", "status": "confirmed" }, { "id": "F18-dangling-profile-pointers", "title": "Settings PATCH accepts nonexistent agent and meta profile targets", "severity": "medium", "feature_ids": [ "F18.dangling-pointers" ], "expected": "PATCH /api/settings must reject nonexistent active_agent_profile_id and active_meta_profile targets with 404, as their dedicated activate routes do, without enabling a switch tool for a nonexistent meta-profile.", "actual": "In both runs each dedicated activate route returned 404, but PATCH returned 200 with the nonexistent agent UUID or meta-profile name in the active pointer. The meta-profile PATCH also returned enable_classify_and_switch_llm_tool=true.", "repro_command": "control-agent-server map run --file F18 --only F18.dangling-pointers --fresh --repeat 2 --record --checkout /workspace/agent-sdk-verify", "runs": [ "/workspace/verification-artifacts/runs/20261013-024700-3f62-map-f18", "/workspace/verification-artifacts/runs/20261013-025117-43f0-map-f18" ], "evidence": [ "20261013-024700-3f62-map-f18/evidence/F18.dangling-pointers/activate-agent-profile.json", "20261013-024700-3f62-map-f18/evidence/F18.dangling-pointers/agent-profile-id.json", "20261013-024700-3f62-map-f18/evidence/F18.dangling-pointers/activate-meta-profile.json", "20261013-024700-3f62-map-f18/evidence/F18.dangling-pointers/meta-profile.json", "20261013-024700-3f62-map-f18/evidence/map-run/F18-settings-and-secrets-02-dangling-agent-profile-pointer-f18-dangl.json", "20261013-024700-3f62-map-f18/evidence/map-run/F18-settings-and-secrets-03-dangling-meta-profile-pointer-f18-dangli.json", "20261013-025117-43f0-map-f18/evidence/F18.dangling-pointers/activate-agent-profile.json", "20261013-025117-43f0-map-f18/evidence/F18.dangling-pointers/agent-profile-id.json", "20261013-025117-43f0-map-f18/evidence/F18.dangling-pointers/activate-meta-profile.json", "20261013-025117-43f0-map-f18/evidence/F18.dangling-pointers/meta-profile.json", "20261013-025117-43f0-map-f18/evidence/map-run/F18-settings-and-secrets-02-dangling-agent-profile-pointer-f18-dangl.json", "20261013-025117-43f0-map-f18/evidence/map-run/F18-settings-and-secrets-03-dangling-meta-profile-pointer-f18-dangli.json" ], "notes": "Grouped the agent and meta variants under the same feature ID and missing-target validation symptom. The saved response confirms acceptance and returned state; the map-described lazy default-profile suppression was not separately exercised in these focused repeats. Not filed: this task is read-only saved-evidence extraction; no issue lookup or external action was performed. Both fresh runs used checkout SHA acca181af1f353975965bcf6486e1d0a58f444c0. Reproduction commands are provided for review and were not rerun during extraction." }, { "id": "F18-scoped-secrets-profile-errors", "title": "Scoped secret listing returns unhandled 500 for agent profile store errors", "severity": "medium", "feature_ids": [ "F18.secret-scope-profile-errors" ], "expected": "GET /api/settings/secrets?agent_profile_id= must map malformed agent-profile data to 400 and a busy profile-store lock to 503, matching the agent-profile routes.", "actual": "In both runs the malformed profile control returned 400, while scoped secret listing returned 500 with the validation exception. With the lock held, the profile route returned 503 after about 30 seconds, while scoped secret listing returned 500 with Agent profile store lock acquisition timed out after 30.0s (about 32.6 seconds).", "repro_command": "control-agent-server map run --file F18 --only F18.secret-scope-profile-errors --fresh --repeat 2 --record --checkout /workspace/agent-sdk-verify", "runs": [ "/workspace/verification-artifacts/runs/20261013-024700-3f62-map-f18", "/workspace/verification-artifacts/runs/20261013-025117-43f0-map-f18" ], "evidence": [ "20261013-024700-3f62-map-f18/evidence/F18.secret-scope-profile-errors/agent-profile-route.json", "20261013-024700-3f62-map-f18/evidence/F18.secret-scope-profile-errors/unloadable.json", "20261013-024700-3f62-map-f18/evidence/F18.secret-scope-profile-errors/agent-profile-busy.json", "20261013-024700-3f62-map-f18/evidence/F18.secret-scope-profile-errors/secrets-busy.json", "20261013-024700-3f62-map-f18/evidence/map-run/F18-settings-and-secrets-04-agent-profile-that-no-longer-loads-f18-s.json", "20261013-024700-3f62-map-f18/evidence/map-run/F18-settings-and-secrets-05-agent-profile-store-busy-f18-secret-scop.json", "20261013-025117-43f0-map-f18/evidence/F18.secret-scope-profile-errors/agent-profile-route.json", "20261013-025117-43f0-map-f18/evidence/F18.secret-scope-profile-errors/unloadable.json", "20261013-025117-43f0-map-f18/evidence/F18.secret-scope-profile-errors/agent-profile-busy.json", "20261013-025117-43f0-map-f18/evidence/F18.secret-scope-profile-errors/secrets-busy.json", "20261013-025117-43f0-map-f18/evidence/map-run/F18-settings-and-secrets-04-agent-profile-that-no-longer-loads-f18-s.json", "20261013-025117-43f0-map-f18/evidence/map-run/F18-settings-and-secrets-05-agent-profile-store-busy-f18-secret-scop.json" ], "notes": "Grouped malformed-data and busy-lock cases because they exercise the same scoped-list entry point and absent profile-store error mapping, with both control/target pairs independently saved in each run. Not filed: this task is read-only saved-evidence extraction; no issue lookup or external action was performed. Both fresh runs used checkout SHA acca181af1f353975965bcf6486e1d0a58f444c0. Reproduction commands are provided for review and were not rerun during extraction." }, { "id": "F18-invalid-name-openapi-status", "title": "Secret GET OpenAPI promises the wrong invalid-name status", "severity": "low", "feature_ids": [ "F18.secret-name-contract" ], "expected": "The published GET /api/settings/secrets/{name} invalid-name status must match the live route response (422), or the route must return its published 400.", "actual": "Both saved OpenAPI documents say 400 if name format is invalid for GET; GET /api/settings/secrets/1bad returned 422 in both runs, failing the assertion derived from that published description.", "repro_command": "control-agent-server map run --file F18 --only F18.secret-name-contract --fresh --repeat 2 --record --checkout /workspace/agent-sdk-verify", "runs": [ "/workspace/verification-artifacts/runs/20261013-024700-3f62-map-f18", "/workspace/verification-artifacts/runs/20261013-025117-43f0-map-f18" ], "evidence": [ "20261013-024700-3f62-map-f18/fixtures/qa-f18/openapi.json", "20261013-024700-3f62-map-f18/evidence/F18.secret-name-contract/invalid-name.json", "20261013-024700-3f62-map-f18/evidence/map-run/F18-settings-and-secrets-06-documented-status-for-an-invalid-name-f1.json", "20261013-025117-43f0-map-f18/fixtures/qa-f18/openapi.json", "20261013-025117-43f0-map-f18/evidence/F18.secret-name-contract/invalid-name.json", "20261013-025117-43f0-map-f18/evidence/map-run/F18-settings-and-secrets-06-documented-status-for-an-invalid-name-f1.json" ], "notes": "The saved DELETE description also says 400, but DELETE was not driven by these focused repeats. PUT already documents 422 in both saved OpenAPI documents; the frozen recipe prose claiming PUT also says 400 is stale and is excluded from this confirmed bug. Not filed: this task is read-only saved-evidence extraction; no issue lookup or external action was performed. Both fresh runs used checkout SHA acca181af1f353975965bcf6486e1d0a58f444c0. Reproduction commands are provided for review and were not rerun during extraction." }, { "id": "F18-secret-key-rotation-data-loss", "title": "An unrelated secret write after cipher rotation destroys existing secret ciphertext", "severity": "high", "feature_ids": [ "F18.secret-key-rotation" ], "expected": "When ciphertext cannot be decrypted with the current cipher key, a write must reject the operation or preserve the old ciphertext so restoring the old key recovers the existing values.", "actual": "In both runs rotating the cipher key and restoring it without a write recovered the original secret and LLM key. After another rotation, PUT of an unrelated secret returned 200 and an unrelated misc-settings PATCH also returned 200. Once the original key was restored, the saved state assertion showed custom_secrets.QA_F18_TOKEN.secret=null instead of its previous Fernet ciphertext.", "repro_command": "control-agent-server map run --file F18 --only F18.secret-key-rotation --fresh --repeat 2 --record --checkout /workspace/agent-sdk-verify", "runs": [ "/workspace/verification-artifacts/runs/20261013-024700-3f62-map-f18", "/workspace/verification-artifacts/runs/20261013-025117-43f0-map-f18" ], "evidence": [ "20261013-024700-3f62-map-f18/evidence/F18.secret-key-rotation/read-under-new-key.json", "20261013-024700-3f62-map-f18/evidence/F18.secret-key-rotation/settings-under-new-key.json", "20261013-024700-3f62-map-f18/evidence/F18.secret-key-rotation/control-rotated-back.json", "20261013-024700-3f62-map-f18/evidence/F18.secret-key-rotation/put-under-new-key.json", "20261013-024700-3f62-map-f18/evidence/F18.secret-key-rotation/patch-under-new-key.json", "20261013-024700-3f62-map-f18/evidence/map-run/F18-settings-and-secrets-07-cipher-key-rotation-f18-secret-key-rotat.json", "20261013-025117-43f0-map-f18/evidence/F18.secret-key-rotation/read-under-new-key.json", "20261013-025117-43f0-map-f18/evidence/F18.secret-key-rotation/settings-under-new-key.json", "20261013-025117-43f0-map-f18/evidence/F18.secret-key-rotation/control-rotated-back.json", "20261013-025117-43f0-map-f18/evidence/F18.secret-key-rotation/put-under-new-key.json", "20261013-025117-43f0-map-f18/evidence/F18.secret-key-rotation/patch-under-new-key.json", "20261013-025117-43f0-map-f18/evidence/map-run/F18-settings-and-secrets-07-cipher-key-rotation-f18-secret-key-rotat.json" ], "notes": "The decisive post-restore persisted-null assertion is inside the map-run transcript, not a separate HTTP exchange. Each recipe stops at that first failing secrets.json assertion. The later settings.json ciphertext assertion and final post-restore GET checks were not executed, so permanent LLM-key loss and final HTTP reads are not claimed as independently confirmed here. Not filed: this task is read-only saved-evidence extraction; no issue lookup or external action was performed. Both fresh runs used checkout SHA acca181af1f353975965bcf6486e1d0a58f444c0. Reproduction commands are provided for review and were not rerun during extraction." }, { "id": "F17-workspace-registry-noop-writes", "title": "Workspace registry no-op mutations unnecessarily create or rewrite the persistence file", "severity": "low", "feature_ids": [ "F17.noop-delete-no-file", "F17.noop-write-untouched" ], "expected": "A refused DELETE on a fresh directory must leave workspaces.json absent; re-posting identical registered workspaces must preserve the existing file inode and modification time, as GET does.", "actual": "Both runs begin with no workspaces.json, return 404 for the two unknown-path deletes, keep both lists empty, then find a newly created file. Later an identical re-add returns 200 with the same registry body as before, but the inode/modification-time equality assertion fails; the preceding GET leaves that equality intact.", "repro_command": "control-agent-server map run --file F17 --only F17.noop-delete-no-file,F17.add-workspaces,F17.wire-format,F17.idempotent-add,F17.delete-workspace,F17.noop-write-untouched --fresh --repeat 2 --record --keep-going --checkout /workspace/agent-sdk-verify", "runs": [ "/workspace/verification-artifacts/runs/20261013-025957-13e4-map-f17", "/workspace/verification-artifacts/runs/20261013-030107-4090-map-f17" ], "evidence": [ "20261013-025957-13e4-map-f17/evidence/F17.noop-delete-no-file/delete.json", "20261013-025957-13e4-map-f17/evidence/map-run/F17-workspaces-registry-01-a-refused-delete-on-a-fresh-directory-f1.json", "20261013-025957-13e4-map-f17/evidence/F17.delete-workspace/get.json", "20261013-025957-13e4-map-f17/evidence/F17.noop-write-untouched/readd.json", "20261013-025957-13e4-map-f17/evidence/map-run/F17-workspaces-registry-06-no-op-writes-leave-the-file-alone-f17-no.json", "20261013-030107-4090-map-f17/evidence/F17.noop-delete-no-file/delete.json", "20261013-030107-4090-map-f17/evidence/map-run/F17-workspaces-registry-01-a-refused-delete-on-a-fresh-directory-f1.json", "20261013-030107-4090-map-f17/evidence/F17.delete-workspace/get.json", "20261013-030107-4090-map-f17/evidence/F17.noop-write-untouched/readd.json", "20261013-030107-4090-map-f17/evidence/map-run/F17-workspaces-registry-06-no-op-writes-leave-the-file-alone-f17-no.json" ], "notes": "Grouped file creation and identical-content replacement as one persistence no-op defect. The re-add branch stops at its first failed stamp comparison; subsequent unknown workspace/parent DELETE stamp checks are not reached. F17.noop-write-readonly is blocked by missing chattr and is not included as a verified symptom. Explicit arrange IDs are included because F17 recipes lack Requires declarations. Confirmed on two fresh runs at checkout SHA acca181af1f353975965bcf6486e1d0a58f444c0. Not filed: read-only saved-evidence extraction performed no issue lookup, rerun, or external action." }, { "id": "F19-empty-mcp-server-accepted", "title": "MCP settings accept a server with neither command nor URL", "severity": "low", "feature_ids": [ "F19.settings-empty-server" ], "expected": "POST /api/settings/mcp/qa-empty with {} must return 422 and leave no stored server.", "actual": "Both fresh runs confirm the key is initially absent, then POST {} returns 201 with qa-empty={enabled:true} in the returned MCP settings. The recipe removes the accepted entry and fails its explicit false guard.", "repro_command": "control-agent-server map run --file F19 --only F19.settings-empty-server --fresh --repeat 2 --record --keep-going --checkout /workspace/agent-sdk-verify", "runs": [ "/workspace/verification-artifacts/runs/20261013-030153-d95d-map-f19", "/workspace/verification-artifacts/runs/20261013-030348-f557-map-f19" ], "evidence": [ "20261013-030153-d95d-map-f19/evidence/F19.settings-empty-server/create.json", "20261013-030153-d95d-map-f19/evidence/map-run/F19-mcp-01-server-with-neither-command-nor-url-f19-.json", "20261013-030348-f557-map-f19/evidence/F19.settings-empty-server/create.json", "20261013-030348-f557-map-f19/evidence/map-run/F19-mcp-01-server-with-neither-command-nor-url-f19-.json" ], "notes": "Only malformed configuration acceptance is proved. The frozen prose claims this can remove all MCP tools from future conversations, but these focused repeats do not independently drive that downstream behavior; no tool-loss severity claim is made. Confirmed on two fresh runs at checkout SHA acca181af1f353975965bcf6486e1d0a58f444c0. Not filed: read-only saved-evidence extraction performed no issue lookup, rerun, or external action." }, { "id": "F19-oauth-callback-replay-500", "title": "Replaying a completed MCP OAuth callback returns an unhandled server error", "severity": "low", "feature_ids": [ "F19.oauth-callback-replay" ], "expected": "Submitting the callback again for a succeeded local OAuth job must return a handled result (200 or a 4xx such as 409).", "actual": "Both runs complete the local OAuth flow and show the job succeeded/callback_ready=true immediately before replay. The repeated callback POST returns 500 with exception All connection attempts failed.", "repro_command": "control-agent-server map run --file F19 --only F19.oauth-callback-replay --fresh --repeat 2 --record --keep-going --checkout /workspace/agent-sdk-verify", "runs": [ "/workspace/verification-artifacts/runs/20261013-030153-d95d-map-f19", "/workspace/verification-artifacts/runs/20261013-030348-f557-map-f19" ], "evidence": [ "20261013-030153-d95d-map-f19/evidence/F19.oauth-flow/succeeded.json", "20261013-030153-d95d-map-f19/evidence/F19.oauth-callback-replay/replay.json", "20261013-030153-d95d-map-f19/evidence/map-run/F19-mcp-04-replayed-callback-f19-oauth-callback-rep.json", "20261013-030348-f557-map-f19/evidence/F19.oauth-flow/succeeded.json", "20261013-030348-f557-map-f19/evidence/F19.oauth-callback-replay/replay.json", "20261013-030348-f557-map-f19/evidence/map-run/F19-mcp-04-replayed-callback-f19-oauth-callback-rep.json" ], "notes": "The assertion stops at the replay response; the later status-unchanged GET is not reached. The frozen map claim about callbacks after consent timeout is not independently exercised. Only the local saved OAuth fixture was involved. Confirmed on two fresh runs at checkout SHA acca181af1f353975965bcf6486e1d0a58f444c0. Not filed: read-only saved-evidence extraction performed no issue lookup, rerun, or external action." }, { "id": "F19-oauth-callback-port-out-of-range-500", "title": "An out-of-range OAuth callback port returns 500 instead of a validation error", "severity": "low", "feature_ids": [ "F19.oauth-callback-bad-port" ], "expected": "A callback URL with port 99999 must return 400, as an incorrect in-range port does.", "actual": "Both runs arrange a local OAuth job awaiting consent, accept the ready-state check, and return 400 Unexpected OAuth callback URL for port 1. Posting port 99999 returns 500 with exception Port out of range 0-65535.", "repro_command": "control-agent-server map run --file F19 --only F19.oauth-callback-bad-port --fresh --repeat 2 --record --keep-going --checkout /workspace/agent-sdk-verify", "runs": [ "/workspace/verification-artifacts/runs/20261013-030153-d95d-map-f19", "/workspace/verification-artifacts/runs/20261013-030348-f557-map-f19" ], "evidence": [ "20261013-030153-d95d-map-f19/evidence/F19.oauth-callback-bad-port/port-99999.json", "20261013-030153-d95d-map-f19/evidence/map-run/F19-mcp-05-port-out-of-range-f19-oauth-callback-bad.json", "20261013-030348-f557-map-f19/evidence/F19.oauth-callback-bad-port/port-99999.json", "20261013-030348-f557-map-f19/evidence/map-run/F19-mcp-05-port-out-of-range-f19-oauth-callback-bad.json" ], "notes": "The recipe stops at the port-99999 failure. The subsequent nonnumeric port abc and final still-authorizing check are not reached, so neither is claimed as confirmed. Confirmed on two fresh runs at checkout SHA acca181af1f353975965bcf6486e1d0a58f444c0. Not filed: read-only saved-evidence extraction performed no issue lookup, rerun, or external action." }, { "id": "F16-stopped-vscode-url", "title": "A stopped VS Code editor is still advertised with a usable-looking URL", "severity": "low", "feature_ids": [ "F16.vscode-url-not-running" ], "expected": "With enabled=true and running=false, both URL routes return url=null.", "actual": "Both fresh keyed servers report running=false but the global URL returns a localhost editor URL carrying the redacted session token. The scoped URL-shape controls also return editor URLs.", "repro_command": "control-agent-server map run --file F16 --only F16.vscode-enabled-status,F16.vscode-url-shape,F16.vscode-url-not-running --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-025955-5a37-map-f16", "/workspace/verification-artifacts/runs/20261013-030422-3050-map-f16" ], "evidence": [ "20261013-025955-5a37-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-03-not-running-still-advertised-f16-vscode-.json", "20261013-025955-5a37-map-f16/evidence/F16.vscode-url-not-running/global.json", "20261013-030422-3050-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-03-not-running-still-advertised-f16-vscode-.json", "20261013-030422-3050-map-f16/evidence/F16.vscode-url-not-running/global.json" ], "notes": "The keyless and deferred-init controls return null in both fresh runs. This is a dead-link/metadata defect; the caller already holds the token. The dedicated scoped null assertion follows the failed global assertion and is not reached. Not filed: parent owns external reporting; no issue was filed by this worker." }, { "id": "F16-vscode-folder-url-encoding", "title": "VS Code URLs do not encode folder names containing ampersand or hash", "severity": "medium", "feature_ids": [ "F16.vscode-folder-encoding" ], "expected": "Parsing the URL yields exactly the requested folder and no fragment.", "actual": "The saved scoped URL pastes qa f16&x=1#frag literally: a URL parser reads a truncated folder, an extra x parameter, and a fragment. The normal-folder control passes in both runs.", "repro_command": "control-agent-server map run --file F16 --only F16.vscode-enabled-status,F16.vscode-url-shape,F16.vscode-folder-encoding --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-025955-5a37-map-f16", "/workspace/verification-artifacts/runs/20261013-030422-3050-map-f16" ], "evidence": [ "20261013-025955-5a37-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-04-folder-with-and-f16-vscode-folder-encodi.json", "20261013-025955-5a37-map-f16/evidence/F16.vscode-folder-encoding/odd-folder.json", "20261013-030422-3050-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-04-folder-with-and-f16-vscode-folder-encodi.json", "20261013-030422-3050-map-f16/evidence/F16.vscode-folder-encoding/odd-folder.json" ], "notes": "No editor binary is installed. This confirms the URL returned by the real API and its browser parsing, not opening the editor. The URL is observable through the separately confirmed stopped-editor advertisement defect. Not filed: parent owns external reporting; no issue was filed by this worker." }, { "id": "F16-docker-slashless-workspace-cookie", "title": "Cookie authentication fails on the slashless Docker workspace root", "severity": "medium", "feature_ids": [ "F16.docker-root-cookie" ], "expected": "With the workspace-session cookie, slash and slashless routes both reach the conversation lookup (404 for the chosen unknown UUID).", "actual": "In both fresh Docker-mode subprocesses the slashed route returns 404 with the cookie, but the slashless route returns 401 Unauthorized. Header-auth controls reach the conversation lookup for both forms.", "repro_command": "control-agent-server map run --file F16 --only F16.docker-root-cookie --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-025955-5a37-map-f16", "/workspace/verification-artifacts/runs/20261013-030422-3050-map-f16" ], "evidence": [ "20261013-025955-5a37-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-07-docker-mode-root-and-the-cookie-f16-dock.json", "20261013-030329-0721-f16-docker/evidence/F16.docker-root-cookie/slash.json", "20261013-030329-0721-f16-docker/evidence/F16.docker-root-cookie/slashless.json", "20261013-030422-3050-map-f16/evidence/map-run/F16-workspace-serving-and-vscode-07-docker-mode-root-and-the-cookie-f16-dock.json", "20261013-030655-92b6-f16-docker/evidence/F16.docker-root-cookie/slash.json", "20261013-030655-92b6-f16-docker/evidence/F16.docker-root-cookie/slashless.json" ], "notes": "This proves startup route/auth dispatch with OH_CONVERSATION_RUNTIME=docker; a Docker conversation/container is not used. The fresh parent map runs each launch and stop their own isolated Docker-mode server. Not filed: parent owns external reporting; no issue was filed by this worker." }, { "id": "F21-agent-profile-store-stalls-server", "title": "Waiting for the agent-profile store lock stalls unrelated server requests", "severity": "high", "feature_ids": [ "F21.busy-blocks-server" ], "expected": "GET /alive must answer within 2000ms while an agent-profile request waits for the store lock.", "actual": "No-lock /alive controls pass, but while the profile request waits for its store lock, /alive returns 200 only after 25,097ms and 25,063ms in the two fresh targeted runs.", "repro_command": "control-agent-server map run --file F21 --only F21.save-create,F21.acp-kind,F21.busy-blocks-server --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-030758-8dc4-map-f21", "/workspace/verification-artifacts/runs/20261013-030902-92d4-map-f21" ], "evidence": [ "20261013-030758-8dc4-map-f21/evidence/map-run/F21-agent-profiles-03-lock-wait-stalls-the-server-f21-busy-blo.json", "20261013-030758-8dc4-map-f21/evidence/F21.busy-blocks-server/alive.json", "20261013-030902-92d4-map-f21/evidence/map-run/F21-agent-profiles-03-lock-wait-stalls-the-server-f21-busy-blo.json", "20261013-030902-92d4-map-f21/evidence/F21.busy-blocks-server/alive.json" ], "notes": "This matches the already-confirmed F20 LLM-profile-store stall family, but this agent-profile variant currently has only one clean-run proof. The recipe stops at the timing assertion; shell exit releases the held lock. Two further fresh targeted confirmations now reproduce the exact marked failures. Not filed: read-only evidence extraction performed no issue lookup or external action." }, { "id": "F21-typescript-agent-profile-payload-types", "title": "Exported TypeScript agent-profile types reject live server payload fields", "severity": "medium", "feature_ids": [ "F21.ts-payload-types" ], "expected": "The exported OpenHandsAgentProfile, ACPAgentProfile, and AgentProfileDiagnostics types must accept all fields of live payloads accepted by the generated server contract.", "actual": "The generated-contract shape file type-checked. The exported shape file failed tsc with TS2353 for disabled_skills on OpenHandsAgentProfile, acp_startup_timeout on ACPAgentProfile, and resolved_mcp_config_keys on AgentProfileDiagnostics; the summary type produced no error. Both cited fresh targeted repeats reproduce this outcome; the exact per-attempt values are in the saved exchanges/transcripts.", "repro_command": "control-agent-server map run --file F21 --only F21.save-create,F21.acp-kind,F21.ts-payload-types --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-030758-8dc4-map-f21", "/workspace/verification-artifacts/runs/20261013-030902-92d4-map-f21" ], "evidence": [ "20261013-030758-8dc4-map-f21/evidence/map-run/F21-agent-profiles-04-typescript-payload-types-f21-ts-payload-.json", "20261013-030902-92d4-map-f21/evidence/map-run/F21-agent-profiles-04-typescript-payload-types-f21-ts-payload-.json" ], "notes": "Use the full-family command to ensure qa-f21-coder and qa-f21-acp fixture profiles exist; the selected recipe has no Requires declarations for those arrange steps. The compiler errors and payload snapshots are local saved evidence; no rebuild or network operation was performed during extraction. Two further fresh targeted confirmations now reproduce the exact marked failures. Not filed: read-only evidence extraction performed no issue lookup or external action." }, { "id": "F21-json-suffix-overwrite-identity", "title": "Overwriting an agent profile whose name ends in .json changes its identity", "severity": "medium", "feature_ids": [ "F21.json-suffix-identity" ], "expected": "A repeated save of the same accepted name must retain the original profile id and increment revision to 1, or the .json-suffixed name must be rejected with 422 and nothing stored.", "actual": "The accepted profile was activated under id 25f93991-4f33-4a2d-a179-bcff203c003b. A second save updated the LLM reference to deepseek-pro but returned a profile with different id be7b87d6-80df-4dd7-9f2c-62d429d7224e and revision 0. The saved identity/revision assertion failed. Both cited fresh targeted repeats reproduce this outcome; the exact per-attempt values are in the saved exchanges/transcripts.", "repro_command": "control-agent-server map run --file F21 --only F21.save-create,F21.acp-kind,F21.json-suffix-identity --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-030758-8dc4-map-f21", "/workspace/verification-artifacts/runs/20261013-030902-92d4-map-f21" ], "evidence": [ "20261013-030758-8dc4-map-f21/evidence/map-run/F21-agent-profiles-05-json-names-lose-their-identity-f21-json-.json", "20261013-030758-8dc4-map-f21/evidence/F21.json-suffix-identity/after-overwrite.json", "20261013-030902-92d4-map-f21/evidence/map-run/F21-agent-profiles-05-json-names-lose-their-identity-f21-json-.json", "20261013-030902-92d4-map-f21/evidence/F21.json-suffix-identity/after-overwrite.json" ], "notes": "The recipe stops at its first identity/revision failure. The later DELETE and GET /api/settings pointer-clear assertion were not reached, so delete behavior and the final active pointer are not independently claimed. The original id is proven by the saved activation exchange inside the transcript. Two further fresh targeted confirmations now reproduce the exact marked failures. Not filed: read-only evidence extraction performed no issue lookup or external action." }, { "id": "F20-deepseek-auth-error-type", "title": "DeepSeek authentication failure is classified as a bad request", "severity": "low", "feature_ids": [ "F20.validate-bad-key-type" ], "expected": "A provider authentication_error should be reported as LLMAuthenticationError.", "actual": "POST /api/profiles/qa-draft/validate returned 200 valid=false with error.type=LLMBadRequestError, although its redacted provider message contains authentication_error. The saved jq assertion expecting LLMAuthenticationError failed. Both cited fresh targeted repeats reproduce this outcome; the exact per-attempt values are in the saved exchanges/transcripts.", "repro_command": "control-agent-server map run --file F20 --only F20.validate-bad-key-type --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-030609-7ab0-map-f20", "/workspace/verification-artifacts/runs/20261013-030955-5180-map-f20" ], "evidence": [ "20261013-030609-7ab0-map-f20/evidence/map-run/F20-llm-profiles-01-deepseek-401-type-f20-validate-bad-key-t.json", "20261013-030609-7ab0-map-f20/evidence/F20.validate-bad-key-type/deepseek-401.json", "20261013-030955-5180-map-f20/evidence/map-run/F20-llm-profiles-01-deepseek-401-type-f20-validate-bad-key-t.json", "20261013-030955-5180-map-f20/evidence/F20.validate-bad-key-type/deepseek-401.json" ], "notes": "The observed failure is error classification; validation honestly returns valid=false and the posted credential is absent from the provider message. The saved exchange does not include a separate upstream HTTP-status capture. Two further fresh targeted confirmations now reproduce the exact marked failures. Not filed: read-only evidence extraction performed no issue lookup or external action." }, { "id": "F20-validation-defaults-exceed-bound", "title": "Profile validation with default retries and timeout exceeds its 30-second bound", "severity": "medium", "feature_ids": [ "F20.validate-unreachable-bounded", "F20.validate-hang-bounded" ], "expected": "Validation should return a verdict within 30 seconds for a refused connection or a provider that never responds, with the default retry and timeout settings.", "actual": "A refused localhost connection returned valid=false/LLMServiceUnavailableError after 120,553ms and 120,482ms. In both fresh runs the hanging fixture received the provider request, the CLI reached its30-second ReadTimeout, /alive stayed responsive, and a hard restart cancelled the in-flight validation. The exact marked bound assertions failed.", "repro_command": "control-agent-server map run --file F20 --only F20.validate-unreachable-bounded,F20.validate-hang-bounded --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-030609-7ab0-map-f20", "/workspace/verification-artifacts/runs/20261013-030955-5180-map-f20" ], "evidence": [ "20261013-030609-7ab0-map-f20/evidence/map-run/F20-llm-profiles-02-unreachable-endpoint-is-slow-f20-validat.json", "20261013-030609-7ab0-map-f20/evidence/F20.validate-unreachable-bounded/default-retries.json", "20261013-030609-7ab0-map-f20/evidence/map-run/F20-llm-profiles-03-hanging-provider-is-slower-f20-validate-.json", "20261013-030609-7ab0-map-f20/evidence/F20.validate-hang-bounded/stub-request.json", "20261013-030955-5180-map-f20/evidence/map-run/F20-llm-profiles-02-unreachable-endpoint-is-slow-f20-validat.json", "20261013-030955-5180-map-f20/evidence/F20.validate-unreachable-bounded/default-retries.json", "20261013-030955-5180-map-f20/evidence/map-run/F20-llm-profiles-03-hanging-provider-is-slower-f20-validate-.json", "20261013-030955-5180-map-f20/evidence/F20.validate-hang-bounded/stub-request.json" ], "notes": "Grouped the two variants because profile validation inherits the LLM default retry/timeout policy without a bounded preflight budget. No 27-minute terminal duration or eventual hanging-provider verdict is claimed: the focused recipe intentionally cancels it after proving the 30-second timeout. Two further fresh targeted confirmations now reproduce the exact marked failures. Not filed: read-only evidence extraction performed no issue lookup or external action." }, { "id": "F20-json-alias-reference-guard", "title": "Deleting an LLM profile through its .json alias bypasses the reference guard", "severity": "medium", "feature_ids": [ "F20.json-alias-guard" ], "expected": "Deleting the .json alias of a referenced profile must return 409 and preserve the profile, as deleting the plain name does.", "actual": "The plain-name DELETE returned 409 naming the agent referrer and GET through the alias read the profile. Alias DELETE returned 200; the following plain-name GET returned 404 while the referencing agent profile still existed. Cleanup completed before the saved 409 assertion failed. Both cited fresh targeted repeats reproduce this outcome; the exact per-attempt values are in the saved exchanges/transcripts.", "repro_command": "control-agent-server map run --file F20 --only F20.json-alias-guard --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-030609-7ab0-map-f20", "/workspace/verification-artifacts/runs/20261013-030955-5180-map-f20" ], "evidence": [ "20261013-030609-7ab0-map-f20/evidence/map-run/F20-llm-profiles-04-delete-through-the-json-alias-f20-json-a.json", "20261013-030609-7ab0-map-f20/evidence/F20.json-alias-guard/delete.json", "20261013-030609-7ab0-map-f20/evidence/F20.json-alias-guard/after.json", "20261013-030955-5180-map-f20/evidence/map-run/F20-llm-profiles-04-delete-through-the-json-alias-f20-json-a.json", "20261013-030955-5180-map-f20/evidence/F20.json-alias-guard/delete.json", "20261013-030955-5180-map-f20/evidence/F20.json-alias-guard/after.json" ], "notes": "Alias rename and active-profile pointer aftermath mentioned in frozen prose were not independently driven by this recipe and are excluded. Two further fresh targeted confirmations now reproduce the exact marked failures. Not filed: read-only evidence extraction performed no issue lookup or external action." }, { "id": "F20-keyless-profile-activation-after-rotation", "title": "Profile activation accepts an undecryptable stored key after cipher rotation", "severity": "medium", "feature_ids": [ "F20.secret-key-rotation" ], "expected": "Activation must refuse a profile whose encrypted key cannot be decrypted, preserving the previous active profile and settings ciphertext.", "actual": "After rotation the profile file retained Fernet ciphertext, profile detail returned config.api_key=null/api_key_set=false, while the list reported api_key_set=true. Activation returned 200 and llm_applied=true instead of 4xx, failing the exact activation assertion. Both cited fresh targeted repeats reproduce this outcome; the exact per-attempt values are in the saved exchanges/transcripts.", "repro_command": "control-agent-server map run --file F20 --only F20.secret-key-rotation --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going --timeout 420", "runs": [ "/workspace/verification-artifacts/runs/20261013-030609-7ab0-map-f20", "/workspace/verification-artifacts/runs/20261013-030955-5180-map-f20" ], "evidence": [ "20261013-030609-7ab0-map-f20/evidence/map-run/F20-llm-profiles-05-secret-key-rotation-f20-secret-key-rotat.json", "20261013-030609-7ab0-map-f20/evidence/F20.secret-key-rotation/detail.json", "20261013-030609-7ab0-map-f20/evidence/F20.secret-key-rotation/list.json", "20261013-030609-7ab0-map-f20/evidence/F20.secret-key-rotation/activate.json", "20261013-030955-5180-map-f20/evidence/map-run/F20-llm-profiles-05-secret-key-rotation-f20-secret-key-rotat.json", "20261013-030955-5180-map-f20/evidence/F20.secret-key-rotation/detail.json", "20261013-030955-5180-map-f20/evidence/F20.secret-key-rotation/list.json", "20261013-030955-5180-map-f20/evidence/F20.secret-key-rotation/activate.json" ], "notes": "The recipe stops at activation. Later GET /api/settings and settings.json ciphertext checks were not reached; persisted settings destruction, a changed active pointer, and a subsequent conversation failure are not independently confirmed by this saved attempt. Two further fresh targeted confirmations now reproduce the exact marked failures. Not filed: read-only evidence extraction performed no issue lookup or external action." }, { "id": "F22.store-busy-responsive", "title": "Synchronous profile-store locks freeze the server", "severity": "high", "feature_ids": [ "F22.store-busy-responsive", "F23.store-busy-responsive" ], "expected": "GET /alive responds within 3000 ms while a meta-profile save waits on the store lock.", "actual": "F22 meta-profile save lock waits stall /alive for 29,087 and 27,731 ms; F23 provider-connection list lock waits also stall /alive while idle controls remain fast.", "repro_command": "control-agent-server map run --file F22 --only F22.save-classes,F22.store-busy-responsive --fresh --repeat 2 --record --keep-going\n.venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F23-llm-catalog-and-connections.md --only F23.models-no-bedrock,F23.conn-create,F23.conn-create-blank-key,F23.conn-patch-blank-key,F23.validate-linked-draft,F23.store-busy-responsive --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-022801-881f-f22-lock-confirm-1", "/workspace/verification-artifacts/runs/20261013-023058-b5d0-f22-lock-confirm-2", "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23" ], "evidence": [ "/workspace/verification-artifacts/family-results/F22-lock-confirm-1-doctor-after.json", "/workspace/verification-artifacts/family-results/F22-lock-confirm-1-doctor-before.json", "/workspace/verification-artifacts/family-results/F22-lock-confirm-2-doctor-after.json", "/workspace/verification-artifacts/family-results/F22-lock-confirm-2-doctor-before-retry.json", "/workspace/verification-artifacts/runs/20261013-022801-881f-f22-lock-confirm-1/evidence/F22.store-busy-responsive/alive-idle.json", "/workspace/verification-artifacts/runs/20261013-022801-881f-f22-lock-confirm-1/evidence/F22.store-busy-responsive/alive.json", "/workspace/verification-artifacts/runs/20261013-022801-881f-f22-lock-confirm-1/evidence/map-run/F22-meta-profiles-02-lock-wait-blocks-the-server-f22-store-bu.json", "/workspace/verification-artifacts/runs/20261013-023058-b5d0-f22-lock-confirm-2/evidence/F22.store-busy-responsive/alive-idle.json", "/workspace/verification-artifacts/runs/20261013-023058-b5d0-f22-lock-confirm-2/evidence/F22.store-busy-responsive/alive.json", "/workspace/verification-artifacts/runs/20261013-023058-b5d0-f22-lock-confirm-2/evidence/map-run/F22-meta-profiles-02-lock-wait-blocks-the-server-f22-store-bu.json", "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/F23.store-busy-responsive/alive.json", "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/F23.store-busy-responsive/list.json", "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-23-lock-wait-blocks-the-server-f23-store-bu.json", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/F23.store-busy-responsive/alive.json", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/F23.store-busy-responsive/list.json", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/map-run/F23-llm-catalog-and-connections-06-lock-wait-blocks-the-server-f23-store-bu.json" ], "notes": "The async meta-profile router calls the synchronous file-locked MetaProfileStore on the event loop. The same root cause may be grouped with F20/F23 store lock stalls, if each endpoint's proof and counting convention are retained. Both clean confirmations use explicit fresh launch, successful doctor before map driving, doctor after recovery, and targeted stop. Second run required an initial OpenAPI warmup retry before driving. F22.save-classes supplies the undeclared store-directory prerequisite. No model inference occurs in the selected recipe. Grouped F22/F23 synchronous store operations on async request paths. Each endpoint has its own two fresh proofs. Root may group the separately verified F20 route with this mechanism, preserving every endpoint proof." }, { "id": "F28.prepare-corrupt-archive", "title": "Matching-checksum non-gzip backend archive returns unhandled 500", "severity": "low", "feature_ids": [ "F28.prepare-corrupt-archive" ], "expected": "POST backend/prepare rejects invalid archive format with HTTP 409 and records no approval.", "actual": "Both fresh runs return HTTP 500 with exception not a gzip file after checksum validation.", "repro_command": "control-agent-server map run --file F28 --only F28.prepare-corrupt-archive --fresh --repeat 2 --record", "runs": [ "/workspace/verification-artifacts/runs/20261013-025047-4da6-map-f28", "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025047-4da6-map-f28/evidence/F28.prepare-corrupt-archive/prepare.json", "/workspace/verification-artifacts/runs/20261013-025047-4da6-map-f28/evidence/map-run/F28-canvas-extensions-01-non-gzip-artifact-f28-prepare-corrupt-ar.json", "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28/evidence/F28.prepare-corrupt-archive/prepare.json", "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28/evidence/map-run/F28-canvas-extensions-01-non-gzip-artifact-f28-prepare-corrupt-ar.json" ], "notes": "tarfile.ReadError escapes the router ValueError mapping. The case stops at this invalid-format assertion; later no-write checks in the recipe are not reached." }, { "id": "F28.install-ref-cached", "title": "Cached installs silently accept a nonexistent git ref", "severity": "medium", "feature_ids": [ "F25.install-unknown-ref", "F28.install-ref-cached" ], "expected": "An unknown ref returns HTTP 400 and leaves the existing pinned install untouched.", "actual": "Both Canvas and skill endpoints accept the nonexistent ref with 200 when the source is cached and install the cached checkout.", "repro_command": "control-agent-server map run --file F28 --only F28.install-ref-cached --fresh --repeat 2 --record\n.venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F25-skills.md --only F25.marketplace-public,F25.install-bad-frontmatter,F25.install-documented-sources,F25.install-unknown-ref,F25.refresh-unfetchable,F25.sync-offline,F25.marketplace-stale-cache --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-025047-4da6-map-f28", "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28", "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025047-4da6-map-f28/evidence/F28.install-ref-cached/unknown-ref.json", "/workspace/verification-artifacts/runs/20261013-025047-4da6-map-f28/evidence/map-run/F28-canvas-extensions-02-unknown-ref-on-a-cached-source-f28-insta.json", "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28/evidence/F28.install-ref-cached/unknown-ref.json", "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28/evidence/map-run/F28-canvas-extensions-02-unknown-ref-on-a-cached-source-f28-insta.json", "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/F25.install-unknown-ref/cached.json", "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/F25.install-unknown-ref/first-clone.json", "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-19-unknown-ref-f25-install-unknown-ref-know.json", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/F25.install-unknown-ref/cached.json", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/F25.install-unknown-ref/first-clone.json", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/map-run/F25-skills-07-unknown-ref-f25-install-unknown-ref-know.json" ], "notes": "The first pinned install is the positive control; the cache falls back after checkout failure. This shares a generic cached-repository cause with F25 but each driving bullet is counted separately unless grouping is explained. Grouped F25/F28 shared cached-repository fallback defect; each endpoint has two fresh proofs. No duplicate bug count." }, { "id": "F28.status-probe-latch", "title": "One slow health probe permanently latches a recovered backend unhealthy", "severity": "medium", "feature_ids": [ "F28.status-probe-latch" ], "expected": "After direct health returns ok again, backend status returns ready without manual stop/start.", "actual": "Both fresh runs keep reporting unhealthy during the five-second recovery poll, despite direct health ok and a live serving process.", "repro_command": "control-agent-server map run --file F28 --only F28.status-probe-latch --fresh --repeat 2 --record", "runs": [ "/workspace/verification-artifacts/runs/20261013-025047-4da6-map-f28", "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025047-4da6-map-f28/evidence/F28.status-probe-latch/after-recovery.json", "/workspace/verification-artifacts/runs/20261013-025047-4da6-map-f28/evidence/map-run/F28-canvas-extensions-03-health-probe-latch-f28-status-probe-latc.json", "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28/evidence/F28.status-probe-latch/after-recovery.json", "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28/evidence/map-run/F28-canvas-extensions-03-health-probe-latch-f28-status-probe-latc.json" ], "notes": "The slow probe has a one-second timeout. Once unhealthy, later status calls do not probe again. No timing-only model assertions are involved." }, { "id": "F28.hard-restart-orphan", "title": "A live backend survives the server crash while status says stopped", "severity": "high", "feature_ids": [ "F28.hard-restart-orphan" ], "expected": "No backend process of the dead server survives its hard restart.", "actual": "Both fresh runs report stopped with null pid after restart while pgrep of the run-private artifact path still finds the live backend.", "repro_command": "control-agent-server map run --file F28 --only F28.hard-restart-orphan --fresh --repeat 2 --record", "runs": [ "/workspace/verification-artifacts/runs/20261013-025047-4da6-map-f28", "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025047-4da6-map-f28/evidence/F28.hard-restart-orphan/status-after-crash.json", "/workspace/verification-artifacts/runs/20261013-025047-4da6-map-f28/evidence/map-run/F28-canvas-extensions-04-backend-outlives-a-crash-f28-hard-restar.json", "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28/evidence/F28.hard-restart-orphan/status-after-crash.json", "/workspace/verification-artifacts/runs/20261013-025217-c43a-map-f28/evidence/map-run/F28-canvas-extensions-04-backend-outlives-a-crash-f28-hard-restar.json" ], "notes": "The backend starts a new session and survives SIGKILL with parent pid 1. Matching the artifact command path excludes an empty-cmdline zombie; this is separate from the unreaped-child observation limit in backend-stop/status-exit-hang. Automatic targeted CLI stop reaps owned orphans." }, { "id": "F22.empty-router", "title": "A meta-profile with no routing mode is accepted", "severity": "low", "feature_ids": [ "F22.empty-router" ], "expected": "Reject a router with neither classes nor direct prompt with 422.", "actual": "POST with only classifier_model returns 201 and persists an unusable routing configuration.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F22-meta-profiles.md --only F22.save-classes,F22.save-direct,F22.empty-router,F22.activate-acp,F22.corrupt-settings,F22.persistence-dir --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/F22.empty-router/save.json", "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-05-router-with-no-route-f22-empty-router-kn.json", "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22/evidence/F22.empty-router/save.json", "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22/evidence/map-run/F22-meta-profiles-03-router-with-no-route-f22-empty-router-kn.json" ], "notes": "Validation defect; the model validator documents requiring one routing mode. Trap removes the test profile." }, { "id": "F22.activate-acp", "title": "Meta-profile activation reports success while ACP drops it", "severity": "medium", "feature_ids": [ "F22.activate-acp" ], "expected": "Refuse activation or explicitly report that it was not applied.", "actual": "Activation returns 200 activated while both settings and the list retain active_meta_profile=null on an ACP agent.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F22-meta-profiles.md --only F22.save-classes,F22.save-direct,F22.empty-router,F22.activate-acp,F22.corrupt-settings,F22.persistence-dir --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/F22.activate-acp/activate.json", "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/F22.activate-acp/list.json", "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-17-activation-on-acp-settings-f22-activate-.json", "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22/evidence/F22.activate-acp/activate.json", "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22/evidence/F22.activate-acp/list.json", "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22/evidence/map-run/F22-meta-profiles-04-activation-on-acp-settings-f22-activate-.json" ], "notes": "Both runs verify the ACP state and restore OpenHands settings afterwards; the copied recipe arranges the previously undeclared qa-direct activation." }, { "id": "F22.corrupt-settings", "title": "Corrupted settings produce an unmapped meta-profile activation error", "severity": "low", "feature_ids": [ "F22.corrupt-settings" ], "expected": "Return 409 like LLM-profile activation and retain the corrupted file.", "actual": "The LLM-profile control returns 409; meta-profile activation returns 500 for the same corrupted file. The file remains unchanged and is restored.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F22-meta-profiles.md --only F22.save-classes,F22.save-direct,F22.empty-router,F22.activate-acp,F22.corrupt-settings,F22.persistence-dir --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/F22.corrupt-settings/activate.json", "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/F22.corrupt-settings/llm-profile-activate.json", "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-19-unreadable-settings-f22-corrupt-settings.json", "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22/evidence/F22.corrupt-settings/activate.json", "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22/evidence/F22.corrupt-settings/llm-profile-activate.json", "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22/evidence/map-run/F22-meta-profiles-05-unreadable-settings-f22-corrupt-settings.json" ], "notes": "Status/error mapping only; no additional data loss is claimed." }, { "id": "F22.persistence-dir", "title": "Meta-profile persistence does not expand the configured tilde path", "severity": "medium", "feature_ids": [ "F22.persistence-dir" ], "expected": "Write meta-profiles under the expanded OH_PERSISTENCE_DIR beside settings.json.", "actual": "The configured home/qa-f22-tilde/settings.json exists, but its adjacent meta-profile is absent; the meta-profile uses the unexpanded path.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F22-meta-profiles.md --only F22.save-classes,F22.save-direct,F22.empty-router,F22.activate-acp,F22.corrupt-settings,F22.persistence-dir --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22", "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-24-persistence-directory-with-f22-persisten.json", "/workspace/verification-artifacts/runs/20261013-022816-f88c-f22-tilde/evidence/F22.persistence-dir/save.json", "/workspace/verification-artifacts/runs/20261013-034317-5724-map-f22/evidence/map-run/F22-meta-profiles-06-persistence-directory-with-f22-persisten.json", "/workspace/verification-artifacts/runs/20261013-034339-4b8d-f22-tilde/evidence/F22.persistence-dir/save.json" ], "notes": "Nested isolated servers are stopped by the recipe; saved response and state-location transcript prove both fresh attempts." }, { "id": "F23.models-no-bedrock", "title": "Provider model lists include identifiers from Bedrock", "severity": "low", "feature_ids": [ "F23.models-no-bedrock" ], "expected": "A DeepSeek provider list excludes Bedrock-only deepseek.* identifiers.", "actual": "Both provider lists contain deepseek.r1-v1:0, deepseek.v3-v1:0, and deepseek.v3.2.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F23-llm-catalog-and-connections.md --only F23.models-no-bedrock,F23.conn-create,F23.conn-create-blank-key,F23.conn-patch-blank-key,F23.validate-linked-draft,F23.store-busy-responsive --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/F23.models-no-bedrock/anthropic.json", "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/F23.models-no-bedrock/deepseek.json", "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-05-no-bedrock-ids-under-a-provider-f23-mode.json", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/F23.models-no-bedrock/anthropic.json", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/F23.models-no-bedrock/deepseek.json", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/map-run/F23-llm-catalog-and-connections-01-no-bedrock-ids-under-a-provider-f23-mode.json" ], "notes": "The same recipe fetches Anthropic as a control, but its later negative assertion is not reached after the DeepSeek failure." }, { "id": "provider-connection-blank-keys", "title": "Blank and redacted provider keys are accepted on creation and update", "severity": "medium", "feature_ids": [ "F23.conn-create-blank-key", "F23.conn-patch-blank-key" ], "expected": "Reject blank, whitespace-only, and redacted placeholder keys and retain the previous shared key.", "actual": "Whitespace and placeholder creates return 201 with api_key_set=false. PATCH accepts empty, whitespace, and placeholder values; subsequent linked-profile activation applies null, null, and the literal placeholder.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F23-llm-catalog-and-connections.md --only F23.models-no-bedrock,F23.conn-create,F23.conn-create-blank-key,F23.conn-patch-blank-key,F23.validate-linked-draft,F23.store-busy-responsive --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/F23.conn-create-blank-key/list.json", "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/F23.conn-create-blank-key/placeholder.json", "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/F23.conn-create-blank-key/space.json", "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/F23.conn-patch-blank-key/patch-1.json", "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/F23.conn-patch-blank-key/patch-2.json", "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/F23.conn-patch-blank-key/patch-3.json", "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-12-blank-keys-on-create-f23-conn-create-bla.json", "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-16-blank-keys-on-update-f23-conn-patch-blan.json", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/F23.conn-create-blank-key/list.json", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/F23.conn-create-blank-key/placeholder.json", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/F23.conn-create-blank-key/space.json", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/F23.conn-patch-blank-key/patch-1.json", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/F23.conn-patch-blank-key/patch-2.json", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/F23.conn-patch-blank-key/patch-3.json", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/map-run/F23-llm-catalog-and-connections-03-blank-keys-on-create-f23-conn-create-bla.json", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/map-run/F23-llm-catalog-and-connections-04-blank-keys-on-update-f23-conn-patch-blan.json" ], "notes": "One grouped validation defect across create/update; both endpoint branches independently reproduced twice. All negative-test keys are synthetic." }, { "id": "F23.validate-linked-draft", "title": "Draft validation does not resolve its provider connection key", "severity": "medium", "feature_ids": [ "F23.validate-linked-draft" ], "expected": "The same DeepSeek draft validates whether its key is inline or linked through provider_connection_id.", "actual": "The inline-key positive control validates successfully; the linked draft returns valid=false with LLMAuthenticationError.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F23-llm-catalog-and-connections.md --only F23.models-no-bedrock,F23.conn-create,F23.conn-create-blank-key,F23.conn-patch-blank-key,F23.validate-linked-draft,F23.store-busy-responsive --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/F23.validate-linked-draft/linked.json", "/workspace/verification-artifacts/runs/20261013-024155-0956-map-f23/evidence/map-run/F23-llm-catalog-and-connections-18-validate-a-linked-draft-f23-validate-lin.json", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/F23.validate-linked-draft/linked.json", "/workspace/verification-artifacts/runs/20261013-034323-e6ea-map-f23/evidence/map-run/F23-llm-catalog-and-connections-05-validate-a-linked-draft-f23-validate-lin.json" ], "notes": "Tiny real DeepSeek calls distinguish credential resolution from provider/network availability." }, { "id": "F24.status-offline", "title": "Expired subscription status crashes when token refresh is offline", "severity": "medium", "feature_ids": [ "F24.status-offline" ], "expected": "Status returns connected=false after an unreachable token refresh, as it does for a rejected refresh.", "actual": "An unexpired synthetic login reports connected=true; after expiry with the same unreachable proxy, status returns 500 All connection attempts failed.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F24-openai-subscription.md --only F24.status-offline,F24.device-start-offline --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "/workspace/verification-artifacts/runs/20261013-034748-623a-map-f24" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-18-status-with-an-expired-login-offline-f24.json", "/workspace/verification-artifacts/runs/20261013-025510-6f5e-f24-offline/evidence/F24.status-offline/status.json", "/workspace/verification-artifacts/runs/20261013-034748-623a-map-f24/evidence/map-run/F24-openai-subscription-01-status-with-an-expired-login-offline-f24.json", "/workspace/verification-artifacts/runs/20261013-034812-81fc-f24-offline/evidence/F24.status-offline/status.json" ], "notes": "No human login or real account is completed; fixtures contain only explicitly fake access/refresh tokens." }, { "id": "F24.device-start-offline", "title": "Offline device-login start returns an unhandled 500", "severity": "low", "feature_ids": [ "F24.device-start-offline" ], "expected": "Return a gateway failure such as 502/503/504 when OpenAI is unreachable.", "actual": "A real throwaway device code is created and confirmed pending; after its loopback CONNECT relay stops, another device/start returns 500 All connection attempts failed.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F24-openai-subscription.md --only F24.status-offline,F24.device-start-offline --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24", "/workspace/verification-artifacts/runs/20261013-034748-623a-map-f24" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025101-09f6-map-f24/evidence/map-run/F24-openai-subscription-19-start-and-poll-a-login-offline-f24-devic.json", "/workspace/verification-artifacts/runs/20261013-025551-44ef-f24-outage/evidence/F24.device-start-offline/start.json", "/workspace/verification-artifacts/runs/20261013-034748-623a-map-f24/evidence/map-run/F24-openai-subscription-02-start-and-poll-a-login-offline-f24-devic.json", "/workspace/verification-artifacts/runs/20261013-034843-3ace-f24-outage/evidence/F24.device-start-offline/start.json" ], "notes": "No account login was completed. The later offline poll assertion is not reached and is listed as not-run." }, { "id": "F25.marketplace-public", "title": "The public skills marketplace misses the current extensions manifest", "severity": "medium", "feature_ids": [ "F25.marketplace-public" ], "expected": "Return the skills from the actual public extensions manifest.", "actual": "A real extensions checkout is available, but GET /api/skills/marketplace returns 200 with skills=[].", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F25-skills.md --only F25.marketplace-public,F25.install-bad-frontmatter,F25.install-documented-sources,F25.install-unknown-ref,F25.refresh-unfetchable,F25.sync-offline,F25.marketplace-stale-cache --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/F25.marketplace-public/catalog.json", "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-03-public-marketplace-catalog-f25-marketpla.json", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/F25.marketplace-public/catalog.json", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/map-run/F25-skills-02-public-marketplace-catalog-f25-marketpla.json" ], "notes": "The current repository uses .plugin/marketplace.json while the service default expects marketplaces/default.json." }, { "id": "F25.install-bad-frontmatter", "title": "Malformed skill frontmatter produces an unhandled server error", "severity": "low", "feature_ids": [ "F25.install-bad-frontmatter" ], "expected": "Reject malformed YAML frontmatter with a client error.", "actual": "Skill install returns 500 with the YAML parser exception instead of 422.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F25-skills.md --only F25.marketplace-public,F25.install-bad-frontmatter,F25.install-documented-sources,F25.install-unknown-ref,F25.refresh-unfetchable,F25.sync-offline,F25.marketplace-stale-cache --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/F25.install-bad-frontmatter/install.json", "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-12-malformed-frontmatter-f25-install-bad-fr.json", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/F25.install-bad-frontmatter/install.json", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/map-run/F25-skills-03-malformed-frontmatter-f25-install-bad-fr.json" ], "notes": "Synthetic local package; bad-input error mapping only." }, { "id": "documented-extension-source-unusable", "title": "Documented skill and plugin source examples cannot be parsed", "severity": "medium", "feature_ids": [ "F25.install-documented-sources", "F26.install-documented-source" ], "expected": "The source syntax advertised by the current OpenAPI descriptions installs successfully.", "actual": "The supported repository plus repo_path forms install from GitHub, while the advertised github:owner/repo/subdirectory forms return 400 Failed to fetch source.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F25-skills.md --only F25.marketplace-public,F25.install-bad-frontmatter,F25.install-documented-sources,F25.install-unknown-ref,F25.refresh-unfetchable,F25.sync-offline,F25.marketplace-stale-cache --fresh --record --keep-going --timeout 360\n.venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F26-plugins.md --only F26.install-repo-root-name,F26.install-documented-source,F26.refresh-unfetchable,F26.conversation-resume-disabled --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25", "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/F25.install-documented-sources/documented.json", "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-16-documented-install-sources-f25-install-d.json", "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/F26.install-documented-source/documented.json", "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/F26.install-documented-source/working-form.json", "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-11-documented-install-source-f26-install-do.json", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/F25.install-documented-sources/documented.json", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/map-run/F25-skills-06-documented-install-sources-f25-install-d.json", "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26/evidence/F26.install-documented-source/documented.json", "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26/evidence/F26.install-documented-source/working-form.json", "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26/evidence/map-run/F26-plugins-02-documented-install-source-f26-install-do.json" ], "notes": "Grouped documentation/parser contract across skills and plugins. Working GitHub installs are live positive controls. The later skill /tree/main URL variant is not reached." }, { "id": "extension-refresh-missing-source", "title": "Refreshing a vanished skill or plugin source returns an unmapped 500", "severity": "low", "feature_ids": [ "F25.refresh-unfetchable", "F26.refresh-unfetchable" ], "expected": "Refuse an unfetchable source with 400/422 and retain the prior install.", "actual": "After a local installed source is moved away, both refresh endpoints return 500 with ExtensionFetchError.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F25-skills.md --only F25.marketplace-public,F25.install-bad-frontmatter,F25.install-documented-sources,F25.install-unknown-ref,F25.refresh-unfetchable,F25.sync-offline,F25.marketplace-stale-cache --fresh --record --keep-going --timeout 360\n.venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F26-plugins.md --only F26.install-repo-root-name,F26.install-documented-source,F26.refresh-unfetchable,F26.conversation-resume-disabled --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25", "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/F25.refresh-unfetchable/refresh.json", "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-20-refresh-of-a-vanished-source-f25-refresh.json", "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/F26.refresh-unfetchable/vanished.json", "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-16-unfetchable-refresh-f26-refresh-unfetcha.json", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/F25.refresh-unfetchable/refresh.json", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/map-run/F25-skills-08-refresh-of-a-vanished-source-f25-refresh.json", "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26/evidence/F26.refresh-unfetchable/vanished.json", "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26/evidence/map-run/F26-plugins-03-unfetchable-refresh-f26-refresh-unfetcha.json" ], "notes": "Grouped shared fetch-error mapping; each endpoint independently reproduced twice. Later preservation assertions are not reached after the status failure." }, { "id": "F25.sync-offline", "title": "Skills sync reports success after its git fetch fails", "severity": "low", "feature_ids": [ "F25.sync-offline" ], "expected": "Report an unsuccessful forced refresh when its origin cannot be fetched.", "actual": "A direct git fetch proves the configured origin is unreachable, but POST /api/skills/sync returns status=success and Skills repository synced successfully.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F25-skills.md --only F25.marketplace-public,F25.install-bad-frontmatter,F25.install-documented-sources,F25.install-unknown-ref,F25.refresh-unfetchable,F25.sync-offline,F25.marketplace-stale-cache --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/F25.sync-offline/sync.json", "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-24-sync-without-a-reachable-origin-f25-sync.json", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/F25.sync-offline/sync.json", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/map-run/F25-skills-10-sync-without-a-reachable-origin-f25-sync.json" ], "notes": "Response metadata defect; retaining the cached clone is permissible, but the forced sync result is inaccurate." }, { "id": "F25.marketplace-stale-cache", "title": "Skills sync does not refresh the marketplace catalog cache", "severity": "medium", "feature_ids": [ "F25.marketplace-stale-cache" ], "expected": "After sync brings a newly listed skill into the clone, marketplace catalog lists that entry.", "actual": "Sync succeeds and the cloned skill and manifest entry exist, but the catalog still omits qa-f25-pub-extra; one warmed catalog retains its prior entry, and the second retains a cached empty result.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F25-skills.md --only F25.marketplace-public,F25.install-bad-frontmatter,F25.install-documented-sources,F25.install-unknown-ref,F25.refresh-unfetchable,F25.sync-offline,F25.marketplace-stale-cache --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/F25.marketplace-stale-cache/after-sync.json", "/workspace/verification-artifacts/runs/20261013-025747-be38-map-f25/evidence/map-run/F25-skills-32-catalog-after-a-sync-f25-marketplace-sta.json", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/F25.marketplace-stale-cache/after-sync.json", "/workspace/verification-artifacts/runs/20261013-034422-49fa-map-f25/evidence/map-run/F25-skills-11-catalog-after-a-sync-f25-marketplace-sta.json" ], "notes": "Independent from the public manifest-path issue: the local stand-in has a valid marketplaces/default.json. Sync invalidates only public-skills cache, leaving the catalog TTL cache." }, { "id": "F26.install-repo-root-name", "title": "Manifestless repository plugins inherit the cache hash in their name", "severity": "low", "feature_ids": [ "F26.install-repo-root-name" ], "expected": "A manifestless repository installs under its repository directory name.", "actual": "The source qa-f26-src installs as qa-f26-src-<16 hex>, derived from its URL-hashed cache directory.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F26-plugins.md --only F26.install-repo-root-name,F26.install-documented-source,F26.refresh-unfetchable,F26.conversation-resume-disabled --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/F26.install-repo-root-name/install.json", "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-07-repository-root-name-f26-install-repo-ro.json", "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26/evidence/F26.install-repo-root-name/install.json", "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26/evidence/map-run/F26-plugins-01-repository-root-name-f26-install-repo-ro.json" ], "notes": "Metadata/naming defect; fixture is removed before the assertion." }, { "id": "F26.conversation-resume-disabled", "title": "A resumed conversation still activates a disabled plugin skill", "severity": "medium", "feature_ids": [ "F26.conversation-resume-disabled" ], "expected": "A resumed turn follows the plugin enabled state on disk.", "actual": "After disabling the plugin and restarting, its hook no longer runs, but its first slash command still has activated_skills=[qa-f26-inst:qa-hello] on the user message.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F26-plugins.md --only F26.install-repo-root-name,F26.install-documented-source,F26.refresh-unfetchable,F26.conversation-resume-disabled --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26", "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/F26.conversation-resume-disabled/frames.json", "/workspace/verification-artifacts/runs/20261013-030021-e114-map-f26/evidence/map-run/F26-plugins-20-resume-after-disabling-f26-conversation-.json", "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26/evidence/F26.conversation-resume-disabled/frames.json", "/workspace/verification-artifacts/runs/20261013-034505-0581-map-f26/evidence/map-run/F26-plugins-04-resume-after-disabling-f26-conversation-.json" ], "notes": "Two real DeepSeek turns per proof. The later include_skills GET assertion is not reached; the live socket activation itself is proven twice." }, { "id": "F27.subagents-bad-skill", "title": "A sub-agent with an absent configured skill causes message POST 500", "severity": "low", "feature_ids": [ "F27.subagents-bad-skill" ], "expected": "Reject invalid agent configuration or return a client error when it is used.", "actual": "The conversation is created, but appending a user message returns 500 Skill ... not found but was given to agent ...", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F27-hooks-subagents-tools.md --only F27.subagents-bad-skill,F27.custom-tool-modules,F27.custom-tool-missing-on-resume --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "/workspace/verification-artifacts/runs/20261013-034502-5a13-map-f27" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/F27.subagents-bad-skill/send-control.json", "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/F27.subagents-bad-skill/send.json", "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-15-agent-with-a-missing-skill-f27-subagents.json", "/workspace/verification-artifacts/runs/20261013-034502-5a13-map-f27/evidence/F27.subagents-bad-skill/send-control.json", "/workspace/verification-artifacts/runs/20261013-034502-5a13-map-f27/evidence/F27.subagents-bad-skill/send.json", "/workspace/verification-artifacts/runs/20261013-034502-5a13-map-f27/evidence/map-run/F27-hooks-subagents-tools-01-agent-with-a-missing-skill-f27-subagents.json" ], "notes": "Bad configuration/error-mapping defect, not a model failure." }, { "id": "F27.custom-tool-missing-on-resume", "title": "Resuming without the registered custom tool module returns an unhandled 500", "severity": "low", "feature_ids": [ "F27.custom-tool-missing-on-resume" ], "expected": "Report a missing custom-module prerequisite with a controlled client error.", "actual": "After the real custom-tool turn persists its observation, restart without its module makes message POST return 500 because the observation discriminator is not registered.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F27-hooks-subagents-tools.md --only F27.subagents-bad-skill,F27.custom-tool-modules,F27.custom-tool-missing-on-resume --fresh --record --keep-going --timeout 360", "runs": [ "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "/workspace/verification-artifacts/runs/20261013-034502-5a13-map-f27" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/F27.custom-tool-missing-on-resume/send.json", "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-24-tool-module-gone-on-resume-f27-custom-to.json", "/workspace/verification-artifacts/runs/20261013-034502-5a13-map-f27/evidence/F27.custom-tool-missing-on-resume/send.json", "/workspace/verification-artifacts/runs/20261013-034502-5a13-map-f27/evidence/map-run/F27-hooks-subagents-tools-03-tool-module-gone-on-resume-f27-custom-to.json" ], "notes": "The initial custom tool executes successfully on real DeepSeek in both proofs; missing-module errors are not claimed as successful restores." }, { "id": "F27.server-info-browser-disabled", "title": "Server info advertises the browser despite enable_browser=false", "severity": "low", "feature_ids": [ "F27.server-info-browser-disabled" ], "expected": "The server-info usable_tools agrees with the config-aware catalog.", "actual": "Catalog browser usability is false on the disabled server, but server_info still includes browser_tool_set.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F27-browser-config.md --fresh --record --keep-going --timeout 240", "runs": [ "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "/workspace/verification-artifacts/runs/20261013-035121-8592-map-f27" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/F27.server-info-browser-disabled/server-info.json", "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-29-usable-tools-with-the-browser-disabled-f.json", "/workspace/verification-artifacts/runs/20261013-035121-8592-map-f27/evidence/F27.server-info-browser-disabled/server-info.json", "/workspace/verification-artifacts/runs/20261013-035121-8592-map-f27/evidence/map-run/F27-browser-config-02-server-info-browser-disabled-f27-server-.json" ], "notes": "Metadata inconsistency. The fresh supplement retains the real host Chromium rather than requiring an unnecessary cache symlink." }, { "id": "F27.subagents-browser-disabled", "title": "Disabled-browser servers still list the browser-only built-in agent", "severity": "low", "feature_ids": [ "F27.subagents-browser-disabled" ], "expected": "Exclude the browser-only web-researcher when enable_browser=false.", "actual": "The catalog marks browser_tool_set unusable, but built-in discovery still returns web-researcher.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/run-map.py map run --file F27-browser-config.md --fresh --record --keep-going --timeout 240", "runs": [ "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27", "/workspace/verification-artifacts/runs/20261013-035121-8592-map-f27" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/F27.subagents-browser-disabled/builtins.json", "/workspace/verification-artifacts/runs/20261013-030559-0256-map-f27/evidence/map-run/F27-hooks-subagents-tools-30-browser-only-sub-agent-with-the-browser-.json", "/workspace/verification-artifacts/runs/20261013-035121-8592-map-f27/evidence/F27.subagents-browser-disabled/builtins.json", "/workspace/verification-artifacts/runs/20261013-035121-8592-map-f27/evidence/map-run/F27-browser-config-03-subagents-browser-disabled-f27-subagents.json" ], "notes": "Only discovery was driven; no browser delegation or configuration bypass is claimed." }, { "id": "F27.browser-playwright-path", "title": "The browser availability probe ignores PLAYWRIGHT_BROWSERS_PATH", "severity": "medium", "feature_ids": [ "F27.browser-playwright-path" ], "expected": "Find the real Chromium binary installed solely under PLAYWRIGHT_BROWSERS_PATH.", "actual": "Two fresh Chromium-free layout-image containers report the browser unusable with the configured Playwright path. Linking the same binary into HOME/.cache/ms-playwright makes the catalog usable; removing that link reproduces the failure.", "repro_command": ".venv/bin/python /workspace/verification-artifacts/takeover-f22-f28/browser_container.py", "runs": [ "/workspace/verification-artifacts/runs/20261013-040445-9999-qa-f27-browser-path-1", "/workspace/verification-artifacts/runs/20261013-040554-b24c-qa-f27-browser-path-2" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-040445-9999-qa-f27-browser-path-1/evidence/F27.browser-playwright-path/catalog.json", "/workspace/verification-artifacts/runs/20261013-040445-9999-qa-f27-browser-path-1/evidence/F27.browser-playwright-path/layout.json", "/workspace/verification-artifacts/runs/20261013-040445-9999-qa-f27-browser-path-1/evidence/F27.browser-probe/before.json", "/workspace/verification-artifacts/runs/20261013-040445-9999-qa-f27-browser-path-1/evidence/F27.browser-probe/home-control.json", "/workspace/verification-artifacts/runs/20261013-040445-9999-qa-f27-browser-path-1/evidence/map-run/F27-browser-container-00-preconditions.json", "/workspace/verification-artifacts/runs/20261013-040445-9999-qa-f27-browser-path-1/evidence/map-run/F27-browser-container-01-home-cache-positive-control-f27-browser-.json", "/workspace/verification-artifacts/runs/20261013-040445-9999-qa-f27-browser-path-1/evidence/map-run/F27-browser-container-02-playwright-override-f27-browser-playwrig.json", "/workspace/verification-artifacts/runs/20261013-040554-b24c-qa-f27-browser-path-2/evidence/F27.browser-playwright-path/catalog.json", "/workspace/verification-artifacts/runs/20261013-040554-b24c-qa-f27-browser-path-2/evidence/F27.browser-playwright-path/layout.json", "/workspace/verification-artifacts/runs/20261013-040554-b24c-qa-f27-browser-path-2/evidence/F27.browser-probe/before.json", "/workspace/verification-artifacts/runs/20261013-040554-b24c-qa-f27-browser-path-2/evidence/F27.browser-probe/home-control.json", "/workspace/verification-artifacts/runs/20261013-040554-b24c-qa-f27-browser-path-2/evidence/map-run/F27-browser-container-00-preconditions.json", "/workspace/verification-artifacts/runs/20261013-040554-b24c-qa-f27-browser-path-2/evidence/map-run/F27-browser-container-01-home-cache-positive-control-f27-browser-.json", "/workspace/verification-artifacts/runs/20261013-040554-b24c-qa-f27-browser-path-2/evidence/map-run/F27-browser-container-02-playwright-override-f27-browser-playwrig.json" ], "notes": "The real host Chromium binary is mounted read-only; only probe/catalog usability is asserted, not a navigation test. Each container is removed after its attached-run proof." }, { "id": "F29.session-forwarded-proto", "title": "Canvas bridge trusts a forwarded scheme despite its setting", "severity": "low", "feature_ids": [ "F29.session-forwarded-proto" ], "expected": "With trust_forwarded_headers disabled, X-Forwarded-Proto should not alter an otherwise valid request.", "actual": "The same session bootstrap succeeds without the header and returns 421 when X-Forwarded-Proto: https is supplied.", "repro_command": "control-agent-server map run --file F29 --only F29.session-forwarded-proto --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29", "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29/evidence/map-run/F29-app-backend-bridge-01-forwarded-proto-f29-session-forwarded-pr.json", "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-01-forwarded-proto-f29-session-forwarded-pr.json" ], "notes": [] }, { "id": "F29.proxy-root-path-query", "title": "Canvas bare-root query parameter changes the proxied path", "severity": "low", "feature_ids": [ "F29.proxy-root-path-query" ], "expected": "?path=echo on the bare app root should reach the root with the query unchanged.", "actual": "The proxy sends the request to /echo, whereas the trailing-slash root preserves /.", "repro_command": "control-agent-server map run --file F29 --only F29.proxy-root-path-query --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29", "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29/evidence/map-run/F29-app-backend-bridge-02-root-path-parameter-f29-proxy-root-path-.json", "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-02-root-path-parameter-f29-proxy-root-path-.json" ], "notes": [] }, { "id": "F29.proxy-response-headers", "title": "Canvas proxy sends duplicate Date and Server headers", "severity": "low", "feature_ids": [ "F29.proxy-response-headers" ], "expected": "A proxied response should contain one Date and one Server header.", "actual": "Uvicorn adds its headers alongside copied backend headers, producing comma-joined duplicate values.", "repro_command": "control-agent-server map run --file F29 --only F29.proxy-response-headers --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29", "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29/evidence/map-run/F29-app-backend-bridge-04-one-date-and-server-header-f29-proxy-res.json", "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-04-one-date-and-server-header-f29-proxy-res.json" ], "notes": [] }, { "id": "F29.ws-subprotocol", "title": "Canvas WebSocket bridge loses requested subprotocol", "severity": "medium", "feature_ids": [ "F29.ws-subprotocol" ], "expected": "The bridge should negotiate qa.v1 exactly as the directly contacted backend does.", "actual": "The direct socket selects qa.v1; the proxied socket selects no protocol and reports protocol none, breaking browser clients that require a subprotocol.", "repro_command": "control-agent-server map run --file F29 --only F29.ws-subprotocol --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29", "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29/evidence/map-run/F29-app-backend-bridge-05-subprotocol-f29-ws-subprotocol-known-bug.json", "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-05-subprotocol-f29-ws-subprotocol-known-bug.json" ], "notes": [] }, { "id": "F29.backend-crash-detected", "title": "Canvas bridge keeps crashed backend ready until status polling", "severity": "low", "feature_ids": [ "F29.backend-crash-detected" ], "expected": "After the backend process exits, requests should return 503 backend not ready.", "actual": "Without a status poll, requests return 502 Conversation container unreachable for the entire 15-second observation.", "repro_command": "control-agent-server map run --file F29 --only F29.backend-crash-detected --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29", "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-031857-df62-map-f29/evidence/map-run/F29-app-backend-bridge-06-crash-seen-by-the-bridge-f29-backend-cra.json", "/workspace/verification-artifacts/runs/20261013-031947-956c-map-f29/evidence/map-run/F29-app-backend-bridge-06-crash-seen-by-the-bridge-f29-backend-cra.json" ], "notes": [] }, { "id": "F30.chat-usage-per-request", "title": "OpenAI continuation usage includes previous turns", "severity": "medium", "feature_ids": [ "F30.chat-usage-per-request" ], "expected": "The second request should report its own 3 prompt, 2 completion and 5 total tokens.", "actual": "It reports accumulated 13 prompt, 7 completion and 20 total tokens although per-call usage is available.", "repro_command": "control-agent-server map run --file F30 --only F30.chat-usage-per-request --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30", "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30/evidence/map-run/F30-openai-gateway-03-usage-per-request-f30-chat-usage-per-req.json", "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-03-usage-per-request-f30-chat-usage-per-req.json" ], "notes": [] }, { "id": "F30.chat-reuse-concurrent", "title": "Overlapping OpenAI requests receive the later turn's answer", "severity": "medium", "feature_ids": [ "F30.chat-reuse-concurrent" ], "expected": "Request X should receive its stored QA_F30_RACE_B answer; request Y should receive QA_F30_RACE_C.", "actual": "Both requests receive QA_F30_RACE_C although the server stores the correct separate B and C agent messages.", "repro_command": "control-agent-server map run --file F30 --only F30.chat-reuse-concurrent --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30", "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30/evidence/map-run/F30-openai-gateway-04-overlapping-turns-on-one-conversation-f3.json", "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-04-overlapping-turns-on-one-conversation-f3.json" ], "notes": [] }, { "id": "F30.chat-reuse-confirmation", "title": "OpenAI continuation waits 120 seconds at confirmation", "severity": "medium", "feature_ids": [ "F30.chat-reuse-confirmation" ], "expected": "A continued conversation awaiting confirmation should return 409 promptly, like a new one.", "actual": "It remains waiting_for_confirmation with the proposed command unrun, then returns 504 after the fixed 120-second gateway timeout.", "repro_command": "control-agent-server map run --file F30 --only F30.chat-reuse-confirmation --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30", "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30/evidence/map-run/F30-openai-gateway-05-confirmation-on-a-continued-conversation.json", "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-05-confirmation-on-a-continued-conversation.json" ], "notes": [] }, { "id": "F30.chat-reuse-run-limit-429", "title": "Rejected OpenAI continuations persist duplicate user turns", "severity": "medium", "feature_ids": [ "F30.chat-reuse-run-limit-429" ], "expected": "A 429 should leave the conversation unchanged so an OpenAI retry is safe.", "actual": "Each refused request appends the user message; two 429 responses leave two copies.", "repro_command": "control-agent-server map run --file F30 --only F30.chat-reuse-run-limit-429 --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30", "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-032042-3ef3-map-f30/evidence/map-run/F30-openai-gateway-06-run-limit-on-a-continued-conversation-f3.json", "/workspace/verification-artifacts/runs/20261013-032720-3de9-map-f30/evidence/map-run/F30-openai-gateway-06-run-limit-on-a-continued-conversation-f3.json" ], "notes": [] }, { "id": "F31.takeover-keeps-metadata", "title": "Second-server takeover overwrites newer conversation metadata", "severity": "high", "feature_ids": [ "F31.takeover-keeps-metadata" ], "expected": "The title last persisted by the first owner should survive takeover.", "actual": "The second server rewrites meta.json with its startup catalog's older title, losing the owner's later rename.", "repro_command": "/workspace/verification-artifacts/harness-f29-f34/.agents/skills/verify-agent-server/scripts/control-agent-server map run --file F31 --only F31.takeover-keeps-metadata --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31", "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31/evidence/map-run/F31-persistence-and-limits-01-takeover-keeps-newer-metadata-f31-takeov.json", "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-01-takeover-keeps-newer-metadata-f31-takeov.json" ], "notes": [ "Isolated copied recipe adjusts only exited-zombie setup observation; frozen product source and bug assertion unchanged. See F31-zombie-adaptation.md." ] }, { "id": "F31.fenced-write-no-trace", "title": "Rejected stale-owner write changes its GET state and returns 500", "severity": "medium", "feature_ids": [ "F31.fenced-write-no-trace" ], "expected": "A lost-ownership PATCH should give a client error and leave visible state unchanged.", "actual": "The guarded save refuses the write and disk remains correct, but the stale owner's GET reports the refused title; PATCH returns 500.", "repro_command": "control-agent-server map run --file F31 --only F31.fenced-write-no-trace --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31", "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31/evidence/map-run/F31-persistence-and-limits-03-refused-write-leaves-no-trace-f31-fenced.json", "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-03-refused-write-leaves-no-trace-f31-fenced.json" ], "notes": [] }, { "id": "F31.config-null-override", "title": "Explicit environment null cannot clear a configured value", "severity": "medium", "feature_ids": [ "F31.config-null-override" ], "expected": "OH_APP_BACKEND_PUBLIC_URL_IS_NONE=1 should clear the URL and capability even when the config file sets it.", "actual": "The file's URL remains advertised because merge keeps an earlier value when the environment supplies None. The same behavior prevents environment-based disabling of a configured idle TTL.", "repro_command": "control-agent-server map run --file F31 --only F31.config-null-override --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31", "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024316-e532-map-f31/evidence/map-run/F31-persistence-and-limits-09-null-from-the-environment-f31-config-nul.json", "/workspace/verification-artifacts/runs/20261013-025937-a7eb-map-f31/evidence/map-run/F31-persistence-and-limits-09-null-from-the-environment-f31-config-nul.json" ], "notes": [] }, { "id": "F31.tmux-dir-cleanup", "title": "Server-selected per-process tmux directory survives shutdown", "severity": "low", "feature_ids": [ "F31.tmux-dir-cleanup" ], "expected": "The per-process TMUX_TMPDIR selected by the server should disappear after graceful stop.", "actual": "The directory remains; the recipe's cleanup trap removes it after recording the failed assertion.", "repro_command": "/workspace/verification-artifacts/harness-f29-f34/.agents/skills/verify-agent-server/scripts/control-agent-server map run --file F31 --only F31.tmux-dir-cleanup --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-031657-8509-map-f31", "/workspace/verification-artifacts/runs/20261013-031758-c007-map-f31" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-031657-8509-map-f31/evidence/map-run/F31-persistence-and-limits-01-per-process-tmux-directory-removed-on-st.json", "/workspace/verification-artifacts/runs/20261013-031758-c007-map-f31/evidence/map-run/F31-persistence-and-limits-01-per-process-tmux-directory-removed-on-st.json" ], "notes": [ "Isolated copied recipe adjusts only exited-zombie setup observation; frozen product source and bug assertion unchanged. See F31-zombie-adaptation.md." ] }, { "id": "F32.buffer-stranded", "title": "Full webhook buffer waits behind an in-flight batch", "severity": "low", "feature_ids": [ "F32.buffer-stranded" ], "expected": "The queued user message should reach the sink within ten seconds once the buffer fills.", "actual": "The first batch is posted, while a second full buffer waits for the long flush timer or a later close.", "repro_command": "control-agent-server map run --file F32 --only F32.buffer-stranded --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-02-events-stranded-behind-an-in-flight-batc.json", "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-02-events-stranded-behind-an-in-flight-batc.json" ], "notes": [] }, { "id": "F32.webhook-spec-docs", "title": "Webhook schema documents the wrong event path and timer", "severity": "low", "feature_ids": [ "F32.webhook-spec-docs" ], "expected": "OpenAPI should describe the per-conversation event path and a timer started by the first event.", "actual": "It says events go to /events and the timer resets with every new event; live behavior differs.", "repro_command": "control-agent-server map run --file F32 --only F32.webhook-spec-docs --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-04-webhook-field-descriptions-f32-webhook-s.json", "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-04-webhook-field-descriptions-f32-webhook-s.json" ], "notes": [] }, { "id": "F32.slow-receiver", "title": "Slow webhook causes 409 after the conversation reports finished", "severity": "medium", "feature_ids": [ "F32.slow-receiver" ], "expected": "A finished conversation should accept a new run regardless of receiver latency.", "actual": "A fast receiver permits POST /run; a 15-second receiver makes it return 409 Conversation already running while GET says finished.", "repro_command": "control-agent-server map run --file F32 --only F32.slow-receiver --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-05-a-slow-receiver-blocks-the-next-run-f32-.json", "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-05-a-slow-receiver-blocks-the-next-run-f32-.json" ], "notes": [] }, { "id": "F32.telemetry-usage-buckets", "title": "Conversation outcome telemetry misses real token and cost usage", "severity": "low", "feature_ids": [ "F32.telemetry-usage-buckets" ], "expected": "A real model run's outcome should contain measured usage buckets.", "actual": "The conversation has positive token usage but outcome total_tokens_bucket and cost_bucket are unknown.", "repro_command": "control-agent-server map run --file F32 --only F32.telemetry-usage-buckets --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-10-token-and-cost-buckets-from-real-usage-f.json", "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-10-token-and-cost-buckets-from-real-usage-f.json" ], "notes": [] }, { "id": "F32.telemetry-salt-docs", "title": "Telemetry schema promises a random fallback instead of secret key", "severity": "low", "feature_ids": [ "F32.telemetry-salt-docs" ], "expected": "The salt description should state that the configured secret key is the first fallback.", "actual": "It promises per-process random salt and unlinkability across runs although the secret-key-derived reference remains stable across restarts.", "repro_command": "control-agent-server map run --file F32 --only F32.telemetry-salt-docs --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32", "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-031700-aa28-map-f32/evidence/map-run/F32-webhooks-and-telemetry-16-salt-description-f32-telemetry-salt-docs.json", "/workspace/verification-artifacts/runs/20261013-032530-35b4-map-f32/evidence/map-run/F32-webhooks-and-telemetry-16-salt-description-f32-telemetry-salt-docs.json" ], "notes": [] }, { "id": "F33.conversation-token-callbacks", "title": "Python remote conversations ignore token callbacks", "severity": "medium", "feature_ids": [ "F33.conversation-token-callbacks" ], "expected": "Token callbacks should receive streamed deltas supplied to the remote conversation.", "actual": "Event callbacks see deltas, but the token callback is never called.", "repro_command": "control-agent-server map run --file F33 --only F33.conversation-token-callbacks --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-03-token-callbacks-on-a-remote-conversation.json", "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-03-token-callbacks-on-a-remote-conversation.json" ], "notes": [] }, { "id": "F33.conversation-streaming-state", "title": "Python remote state stores transient token deltas", "severity": "medium", "feature_ids": [ "F33.conversation-streaming-state" ], "expected": "state.events should exclude transient StreamingDeltaEvent frames and match stored server history.", "actual": "The client adds streamed deltas to state.events although the server persists none.", "repro_command": "control-agent-server map run --file F33 --only F33.conversation-streaming-state --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-04-deltas-kept-out-of-state-events-f33-conv.json", "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-04-deltas-kept-out-of-state-events-f33-conv.json" ], "notes": [] }, { "id": "F33.conversation-state-reread", "title": "Reading Python remote state twice corrupts its cached values", "severity": "medium", "feature_ids": [ "F33.conversation-state-reread" ], "expected": "Repeated reads of confirmation policy, analyzer, agent and confirmation-mode flag should remain stable.", "actual": "First reads are correct; second policy/agent reads raise ValidationError, analyzer becomes None and confirmation mode changes true to false.", "repro_command": "control-agent-server map run --file F33 --only F33.conversation-state-reread --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-08-read-the-state-twice-f33-conversation-st.json", "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-08-read-the-state-twice-f33-conversation-st.json" ], "notes": [] }, { "id": "F33.conversation-run-join-long-tool", "title": "Python run join times out during a healthy long tool call", "severity": "medium", "feature_ids": [ "F33.conversation-run-join-long-tool" ], "expected": "A second run() should join the 45-second tool call and return finished.", "actual": "It raises ConversationRunError wrapping ReadTimeout after 30 seconds while the original run continues, writes its file and finishes.", "repro_command": "control-agent-server map run --file F33 --only F33.conversation-run-join-long-tool --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-17-join-a-run-during-a-long-tool-call-f33-c.json", "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-17-join-a-run-during-a-long-tool-call-f33-c.json" ], "notes": [] }, { "id": "F33.conversation-reconnect-callbacks", "title": "Python reconnect reconciles history but skips missed callbacks", "severity": "medium", "feature_ids": [ "F33.conversation-reconnect-callbacks" ], "expected": "A user message recorded during the socket outage should reach callbacks after reconnection.", "actual": "Both messages appear in server history and state.events; only the connected message reaches the callback.", "repro_command": "control-agent-server map run --file F33 --only F33.conversation-reconnect-callbacks --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-23-callbacks-after-a-reconnect-f33-conversa.json", "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-23-callbacks-after-a-reconnect-f33-conversa.json" ], "notes": [] }, { "id": "F33.custom-tool-module-unimportable", "title": "Unimportable custom tool produces an unhandled server error", "severity": "low", "feature_ids": [ "F33.custom-tool-module-unimportable" ], "expected": "A client tool the server cannot import should result in a 4xx or a recorded conversation error.", "actual": "Creation succeeds with 201, then sending the message fails with 500 for the unregistered tool.", "repro_command": "control-agent-server map run --file F33 --only F33.custom-tool-module-unimportable --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33", "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-033440-1239-map-f33/evidence/map-run/F33-sdk-remote-client-26-an-agent-tool-the-server-cannot-import-f.json", "/workspace/verification-artifacts/runs/20261013-034709-4e02-map-f33/evidence/map-run/F33-sdk-remote-client-26-an-agent-tool-the-server-cannot-import-f.json" ], "notes": [] }, { "id": "F34.workspace-binary-download", "title": "TypeScript RemoteWorkspace corrupts binary downloads", "severity": "medium", "feature_ids": [ "F34.workspace-binary-download" ], "expected": "The five-byte file ff 00 80 fe 41 should download byte for byte with file_size 5.", "actual": "FileClient returns the exact bytes; RemoteWorkspace produces eleven bytes ef bf bd 00 ef bf bd ef bf bd 41 and file_size 11.", "repro_command": "control-agent-server map run --file F34 --only F34.workspace-binary-download --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-024310-2602-map-f34", "/workspace/verification-artifacts/runs/20261013-024359-a3ff-map-f34" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024310-2602-map-f34/evidence/map-run/F34-typescript-client-01-binary-download-through-remoteworkspace-.json", "/workspace/verification-artifacts/runs/20261013-024359-a3ff-map-f34/evidence/map-run/F34-typescript-client-01-binary-download-through-remoteworkspace-.json" ], "notes": [] }, { "id": "F34.workspace-git-types", "title": "TypeScript GitChange status type disagrees with server values", "severity": "medium", "feature_ids": [ "F34.workspace-git-types" ], "expected": "Statuses returned by gitChanges should type-check against exported GitChange.status.", "actual": "The server values UPDATED and ADDED type-check against generated GitChangeStatus but fail against the exported lowercase union.", "repro_command": "control-agent-server map run --file F34 --only F34.workspace-git-types --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-01-git-status-type-f34-workspace-git-types-.json", "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-01-git-status-type-f34-workspace-git-types-.json" ], "notes": [] }, { "id": "F34.bash-websocket", "title": "TypeScript BashWebSocketClient cannot open a Node socket", "severity": "medium", "feature_ids": [ "F34.bash-websocket", "F34.ws-callback-client" ], "expected": "The Node callback should receive the BashOutput that the CLI replays on the same socket.", "actual": "No frames arrive and onError says WebSocket implementation not available because the ESM module attempts require('ws').", "repro_command": "control-agent-server map run --file F34 --only F34.bash-websocket,F34.ws-callback-client --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-02-bash-events-socket-from-node-f34-bash-we.json", "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-02-bash-events-socket-from-node-f34-bash-we.json", "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-05-conversation-callback-socket-from-node-f.json", "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-05-conversation-callback-socket-from-node-f.json" ], "notes": [ "Two driving bullets share the ESM WebSocket constructor discovery failure and are counted as one defect." ] }, { "id": "F34.manager-update-result", "title": "TypeScript updateConversation return type promises missing fields", "severity": "medium", "feature_ids": [ "F34.manager-update-result" ], "expected": "The declared ConversationInfo result should include the conversation id and stored new title.", "actual": "The title is stored, but the promise resolves with success:true and its id/title are undefined.", "repro_command": "control-agent-server map run --file F34 --only F34.manager-update-result --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-06-update-result-f34-manager-update-result-.json", "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-06-update-result-f34-manager-update-result-.json" ], "notes": [] }, { "id": "F34.manager-create-options", "title": "TypeScript manager ignores conversation execution options", "severity": "medium", "feature_ids": [ "F34.manager-create-options" ], "expected": "maxIterations 7 and stuckDetection false should be preserved, like direct RemoteConversation.start.", "actual": "The manager-created conversation has max_iterations 500 and stuck_detection true.", "repro_command": "control-agent-server map run --file F34 --only F34.manager-create-options --checkout /workspace/agent-sdk-verify --fresh --repeat 2 --record --keep-going", "runs": [ "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34", "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024756-c705-map-f34/evidence/map-run/F34-typescript-client-07-create-options-f34-manager-create-option.json", "/workspace/verification-artifacts/runs/20261013-025335-6a42-map-f34/evidence/map-run/F34-typescript-client-07-create-options-f34-manager-create-option.json" ], "notes": [] } ], "harness_map_issues": [ { "title": "Some F05/F06 isolated recipe selections omit state prerequisites", "actual": "Requires clauses omit earlier recipes whose variables or conversation state are consumed. Repro selections explicitly include the prerequisite IDs; shared files were not changed." }, { "title": "Cold OpenAPI generation exceeded doctor timeout under concurrent load", "evidence": [ "/workspace/verification-artifacts/family-results/F02-repro-doctor.json", "/workspace/verification-artifacts/family-results/F02-repro-doctor-retry.json" ], "actual": "The first doctor had only the OpenAPI check fail with ReadTimeout. After cache warmup the identical doctor was clean, with health/auth/build checks and no log tracebacks. Treat as transient timing; not counted as a product bug." }, { "title": "Shared overlay filled during verification", "actual": "ENOSPC interrupted first primary F02 and first F04 repro save_evidence; empty F04-F07 repro summaries followed. Existing F01, F02-repro and F03-repro summaries were valid. After space restoration the incomplete families were restarted, preserving prior evidence. No bug conclusion uses ENOSPC attempts.", "runs": [ "/workspace/verification-artifacts/runs/20261013-022642-a649-map-f02", "/workspace/verification-artifacts/runs/20261013-023639-adff-map-f04" ] }, { "title": "Hard-restart lease checks encounter unreaped PID-1 zombies", "feature_ids": [ "F05.restart-hard-running", "F05.pause-running-crash" ], "actual": "Original crash recipes fail before the target pause assertion because os.kill(owner_pid, 0) still succeeds for the killed zombie process. The recipe copy waits the recorded lease TTL and restarts; ordinary running recovery then passes and pause-loss reproduces.", "adaptation": "/workspace/verification-artifacts/adapted-f05/feature-map/F05-run-pause-interrupt.md", "evidence": [ "/workspace/verification-artifacts/family-results/F05-lease-adapted-repro.json" ] }, { "id": "cold-doctor-timeout", "title": "Cold OpenAPI doctor probe times out under CPU contention", "actual": "F08.secrets-restart, F10.restart-persist and F14 upload-atomic/restart stop at the required OpenAPI read while health/auth/SHA/socket checks pass.", "adaptation": "Isolated map copies retry doctor once; original timeout evidence is preserved.", "evidence": [ "/workspace/verification-artifacts/family-results/F08-doctor-after-surprises.json" ], "notes": "Original doctor probes use http_client(10); no product code changed." }, { "id": "F09-subscription-race", "title": "Mid-run recipes start before socket subscription", "feature_ids": [ "F09.switch-llm-mid-run", "F09.switch-llm-mid-run-get" ], "actual": "Primary capture contains the sleep observation but misses its earlier ActionEvent.", "adaptation": "Create with --no-run, wait for socket full_state, then POST /run; same real DeepSeek prompt and assertions." }, { "id": "missing-selected-dependencies", "title": "Selected restart recipes omit state-arranging dependencies", "feature_ids": [ "F10.restart-persist", "F14.restart" ], "adaptation": "Added F10.fork-independent/navigate-empty and F14.upload-download/scoped-transfer in isolated copies." }, { "id": "F12-blocked-label", "title": "Docker recipe remains unconditionally blocked on a capable host", "feature_ids": [ "F12.docker-runtime" ], "adaptation": "Removed blocked label only in isolated copy, selected frozen-source image, added own-container safe inspection and missing lifecycle assertions." }, { "id": "F12-no-daemon-precondition", "title": "Docker error-path recipe assumes no daemon without arranging it", "feature_ids": [ "F12.docker-runtime-contract" ], "adaptation": "Explicit missing DOCKER_HOST Unix socket ensures intended no-daemon path." }, { "id": "unknown-runtime-credential-redaction", "severity": "high", "confirmed": true, "title": "Harness does not automatically redact newly issued container session credentials", "actual": "Redactor recognizes run keys and selected env values, rather than the session_api_key response field.", "adaptation": "Capture raw output directly, reveal only boolean presence proof, register key via recognized OPENHANDS_API_KEY env before recording exchanges.", "notes": "Two fresh compatible-image observations safely printed only the boolean marker for an unmasked newly issued runtime key. The captured key was registered under OPENHANDS_API_KEY before evidence/attach. No credential value was printed or published. This is a harness gap, excluded from product-defect counts.", "runs": [ "/workspace/verification-artifacts/runs/20261013-035149-9cf7-map-f12", "/workspace/verification-artifacts/runs/20261013-035150-c7fa-map-f12" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-035149-9cf7-map-f12/evidence/map-run/F12-runtime-and-credentials-01-docker-runtime-with-a-daemon-f12-docker-.json", "/workspace/verification-artifacts/runs/20261013-035150-c7fa-map-f12/evidence/map-run/F12-runtime-and-credentials-01-docker-runtime-with-a-daemon-f12-docker-.json" ] }, { "id": "disk-exhaustion", "title": "Docker build filled 32GB overlay and aborted evidence writes", "feature_ids": [ "F12.managed-key-refresh", "F12.managed-key-scope", "F12.managed-key-switch", "F12.docker-runtime-contract", "F14.trajectory-init-path" ], "actual": "ENOSPC left primary F12.json/F14.json empty, with 27/29 feature ledger rows intact.", "recovery": "Root removed only owned disposable Docker build caches; 26GB free; affected tails replayed separately.", "evidence": [ "/workspace/verification-artifacts/family-results/F12-primary-aborted.json", "/workspace/verification-artifacts/family-results/F14-primary-aborted.json" ] }, { "id": "Docker-layout-and-owner-permissions", "feature_ids": [ "F12.docker-runtime" ], "classification": "harness image setup", "detail": "Earlier image stored source/venv under the runtime-mounted /workspace. Relocation fixed the vanished entrypoint, then copied mode 700 root-owned virtualenv bin blocked UID 1000. Both failure variants are preserved as harness evidence. Local-uid artifact relocates public source/deps and grants read/traverse permissions; actual UID 1000 plus empty-/workspace import proof passes.", "evidence": [ "/workspace/verification-artifacts/docker/runtime-layout-permissions.txt", "/workspace/verification-artifacts/docker/runtime-uid-imports-proof.json" ] }, { "id": "Docker-vfs-transport-deadline", "feature_ids": [ "F12.docker-runtime" ], "classification": "environment/harness timing", "detail": "Two compatible-image creates returned 201 in 86,055 ms and 82,971 ms. CLI default 30-second transport deadline interrupted earlier local vfs provisioning. Create/reprovision now allow 180 seconds. The configured child health deadline remains unchanged." }, { "id": "Docker-persisted-history-route", "feature_ids": [ "F12.docker-runtime" ], "classification": "map drift", "detail": "Frozen registry deliberately mounts outer persisted event reads before the runtime proxy. Events/search with X-Expose-Secrets returns 200; the refusal guard belongs to proxy_http. Copied recipe records persisted-history200 and tests 422 on an actual proxied scoped bash read." }, { "feature_ids": [ "F15.commits" ], "classification": "map drift", "detail": "Copied recipe now accepts valid ISO 8601 UTC Z timestamps and declares Requires: F15.changes-base for the feature-branch fixture. The first supplemental selective history attempt lacked that fixture and is discarded as a harness failure. Two further fresh corrected history attempts pass every assertion. Product checkout unchanged." }, { "feature_ids": [ "F15.ts-client", "F17" ], "classification": "environment and shared build race", "detail": "Initial npm cache/build race attempts discarded. Parent centrally rebuilt TypeScript. F15.ts-client passes in both fresh bug-reproduction runs; clean F17 full map has22 passes,2 expected failures and one chattr environment failure." }, { "feature_ids": [ "F15.repo-search-unreachable", "F16.vscode-enabled-status" ], "classification": "load transient/harness cascading state", "detail": "Initial cold OpenAPI doctor exceeded its fixed10-second timeout under shared load. Fresh F16 enabled-status/doctor and dependent URL-shape/no-key/deferred-init controls pass twice. F15 recovery warms OpenAPI before the unchanged doctor in a copied recipe." }, { "feature_ids": [ "F17.noop-write-untouched" ], "classification": "missing Requires", "detail": "Selective reproduction must explicitly include add-workspaces, wire-format, idempotent-add, delete-workspace prerequisites because family contains no Requires declarations." }, { "feature_ids": [ "F20.sdk-get-llm-linked", "F20.no-cipher", "F21.ts-client" ], "classification": "environment ENOSPC", "detail": "ENOSPC interrupted initial runs at02:39. Evidence preserved. Fresh complete replacement F20 has30 pass/6 xfail and F21 has29 pass/4 xfail. No disk-incident failure is counted as a product defect." }, { "feature_ids": [ "F21.dangling-llm-report" ], "classification": "source/map documentation drift", "status": "excluded-from-product-bugs", "expected": "The private seed-helper prose should describe materialize diagnostics using the publicly declared llm_profile_ref and llm_profile_resolved fields.", "actual": "The saved materialize body conforms to the public response model: valid=false, llm_profile_ref=qa-f21-missing, llm_profile_resolved=false, and an errors entry naming the missing profile. It has no dangling_llm_profile_ref field, and the public AgentProfileDiagnostics model and generated schema do not declare that field. The private seed helper docstring alone promises it; the recipe conditionally asserts it after grepping that docstring.", "runs": [ "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21" ], "evidence": [ "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/F21.dangling-llm-report/materialize.json", "/workspace/verification-artifacts/runs/20261013-024523-a1a8-map-f21/evidence/map-run/F21-agent-profiles-30-how-a-dangling-llm-reference-is-reported.json" ], "source_references": [ "/workspace/agent-sdk-verify/openhands-agent-server/openhands/agent_server/agent_profiles_router.py:181", "/workspace/agent-sdk-verify/openhands-agent-server/openhands/agent_server/agent_profiles_router.py:545", "/workspace/agent-sdk-verify/openhands-sdk/openhands/sdk/profiles/resolver.py:86", "/workspace/agent-sdk-verify/clients/typescript/src/generated/agent-server-schema.ts:1975" ], "notes": "No missing public-contract field or dishonest diagnostics response is established. Fix the seed docstring/map expectation rather than require a new redundant response field; no shared source or map was changed during extraction." }, { "feature_ids": [ "F18.secret-name-contract" ], "classification": "map drift", "detail": "Frozen prose says PUT documents400, but saved OpenAPI already documents422 for PUT. Only GET documentation/status mismatch is confirmed; DELETE branch was not reached." }, { "feature_ids": [ "F15.repo-search-github" ], "classification": "Old capability blocker removed; live positive checks verified; negative control limited by environment", "detail": "An isolated frozen skill copy removes only the predeclared blocked marker. QA_GITHUB_TOKEN is privately assigned from the supplied credential and registered as OPENHANDS_API_KEY for CLI redaction. Product source remains unchanged. Only the public target identity should remain in exported repository names and URLs.", "evidence": [ "/workspace/verification-artifacts/F15-github-live-adaptation.md", "/workspace/verification-artifacts/F15-github-live-adaptation.patch" ], "harness_copy": "/workspace/verification-artifacts/github-live-harness" }, { "feature_ids": [ "F23", "F28" ], "kind": "environment_disk_exhaustion", "actual": "The 32 GB overlay filled during unrelated Docker VFS verification. F23/F28 replay aborted with ENOSPC while saving evidence and arranging fixtures; these attempts are excluded from product confirmations and are being rerun after 26 GB was freed.", "runs": [ "/workspace/verification-artifacts/runs/20261013-022431-19cb-map-f23", "/workspace/verification-artifacts/runs/20261013-023125-701d-map-f28", "/workspace/verification-artifacts/runs/20261013-023753-d1da-map-f22" ], "notes": "All three affected run process groups were stopped through their explicit CLI --run selection. Partial ledgers are preserved. The last F22 directory was an unstarted confirmation attempt with zero evidence files." }, { "feature_ids": [ "F22.store-busy" ], "kind": "harness_timing_fragility", "actual": "Full replay failed shell elapsed <=33s, while the saved HTTP exchange correctly returned 503 after 30360 ms. CLI startup overhead under concurrent load is included in shell SECONDS.", "evidence": [ "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/map-run/F22-meta-profiles-20-store-busy-f22-store-busy.json", "/workspace/verification-artifacts/runs/20261013-021742-7d44-map-f22/evidence/F22.store-busy/save.json" ], "notes": "The subsequent DELETE timeout step was not reached in this raw bullet. This failure does not count as a product bug." }, { "feature_ids": [ "F22.activate-acp", "F22.corrupt-settings", "F22.store-busy-responsive" ], "kind": "undeclared_state_prerequisites", "actual": "Several --only recipes depend on earlier state without declaring Requires. Activation and corrupted-settings need qa-direct active; responsive lock needs an existing meta-profile store directory.", "notes": "Explicit prerequisite IDs or CLI setup restore reviewable reproduction. A discarded exploratory lock subset used save-direct alone, which saved the needed directory but failed its list-count assertion; neither exploratory attempt is used in the confirmed-bug count." }, { "feature_ids": [ "F28.restart-persistence", "F28.uninstall" ], "kind": "map_count_drift", "expected": "Unrelated installed-app counts 6 before uninstall and5 after.", "actual": "New early qa-f28-flaky fixture installed by backend-stop makes counts7 and6; behavior checks after the obsolete count assert were skipped.", "runs": [ "/workspace/verification-artifacts/runs/20261013-024207-5ac2-map-f28" ], "notes": "Fresh independent core persistence/uninstall supplement planned; no shared recipe edit." }, { "feature_ids": [ "F24.status-offline", "F24.device-start-offline" ], "kind": "discarded_arrangement_attempt", "actual": "The first copied precondition had incorrect indentation and did not carry its fixture variables into selected bullets. No bug assertion was reached; the corrected fresh replay reaches both markers.", "evidence": [ "/workspace/verification-artifacts/family-results/F24-repro-arrangement-invalid.json", "/workspace/verification-artifacts/family-results/F24-repro.json" ] }, { "feature_ids": [ "F27.browser-disabled-launch", "F27.browser-disabled", "F27.catalog-docker-runtime" ], "kind": "host_chromium_layout_assumption", "actual": "The host already has standard Chromium. Primary negative-probe checks and cache-symlink assertions fail as preconditions, not product behavior. The focused config/catalog supplement uses the actual host layout and proves the remaining ordinary branches.", "evidence": [ "/workspace/verification-artifacts/family-results/F27-browser-config-repro.json" ] }, { "feature_ids": [ "F28.backend-stop", "F28.status-exit-hang" ], "kind": "pid1_zombie_observation_limit", "actual": "The parent/child stop APIs and status transitions return their expected values, but kill -0 continues to see unreaped zombies. A group-stop timing bound also exceeds seven seconds in this environment. No additional live orphan is inferred from those PID checks; the separately confirmed hard-restart orphan has a nonempty artifact command line.", "evidence": [ "/workspace/verification-artifacts/family-results/F28.json" ] }, { "feature_ids": [ "F28.restart-persistence", "F28.uninstall" ], "kind": "focused_core_replay", "actual": "The focused fresh fixture proves persistence, restart with an existing approval, uninstall, retained data/approval, repeated DELETE, and data removal without unrelated global install counts.", "evidence": [ "/workspace/verification-artifacts/family-results/F28-core-supplement.json" ] }, { "id": "ENOSPC", "reason": "Task-owned Docker build exhausted shared filesystem during original F30\u2013F34 pass; all original ledgers preserved; root removed only task build caches and npm download cache. Recovery supplements replace environment failures.", "evidence": "/workspace/verification-artifacts/F29-F34-enospc-incident.json" }, { "id": "PID1-zombies", "reason": "PID1 does not reap exited server children; kill(pid,0) and CLI alive/stop status report zombies alive. Original instant takeover checks pass twice under own-scope subreaper with unchanged assertions,16 adopted exited children reaped,0 remain,0signals.", "evidence": [ "/workspace/verification-artifacts/family-results/F31-takeover-supervised-repro.json", "/workspace/verification-artifacts/F31-takeover-supervisor-report.json", "/workspace/verification-artifacts/F31-zombie-adaptation.md" ] }, { "id": "F32-shell-timer", "reason": "Separate CLI imports crossed6s window under load. One-process real HTTP probe observes flush6.106s and6.269s twice. First own adaptation omitted workspace and correctly422; those failures preserved.", "evidence": [ "/workspace/verification-artifacts/family-results/F32-timer-corrected-repro.json", "/workspace/verification-artifacts/F32-timer-adaptation.md" ] }, { "id": "F34-reconnect-window", "reason": "Original60s watcher window overlaps15sstop+~25sstartup+backoff;120s observation passes twice on ws-backed client and native Node24. Native knownmarker xpass is runtime drift.", "evidence": "/workspace/verification-artifacts/family-results/F34-reconnect-adapted-repro.json" }, { "id": "F30-missing-Requires", "reason": "Reuse-run-limit recipe consumes QA_F30_SECOND created by reuse-header but declares chat-basic only. Supplement explicitly includes reuse-header; limit recovery passes twice." }, { "id": "source-verification-image", "reason": "Docker became available, so static capability blockers are adapted. Published canonical image packaging remains outside this proof; exact frozen source image requires relocated source/UID1000 traversal permissions.", "evidence": "/workspace/verification-artifacts/F31-F33-docker-capability-adaptation.md" }, { "id": "F33-provider-refusal", "reason": "Unmodified real DeepSeek prompt refused hashing synthetic QA strings and produced no ObservationEvent. Explicit local synthetic fixture purpose passes real DeepSeek and unchanged assertions.", "evidence": "/workspace/verification-artifacts/family-results/F33-secrets-synthetic-prompt.json" } ], "blocked": [ { "feature_ids": [ "F02.default-bind" ], "prerequisite": "Writable /proc/self/uid_map and unprivileged user/network namespaces for unshare -rn", "attempted": "python -m openhands.agent_server default host inside a private namespace", "actual": "unshare: cannot open /proc/self/uid_map: Read-only file system. It failed before launching the server.", "evidence": [ "/workspace/verification-artifacts/runs/20261013-024124-a231-map-f02/evidence/F02.default-bind/no-key.json" ], "notes": "This is blocked by the execution environment, not an agent-server bug. The primary JSON retains the raw fail; the evidence ledger has a later blocked judgment." }, { "feature_ids": [ "F05.restart-hard-running" ], "prerequisite": "A host that reaps the killed server PID before the immediate recovery assertion", "attempted": "Original immediate SIGKILL/restart dead-owner takeover recipe", "actual": "The container leaves the owner PID as a zombie and os.kill(pid, 0) treats it as alive. Recovery after recorded lease expiry and a further restart is proven twice; instantaneous takeover is not proven.", "evidence": [ "/workspace/verification-artifacts/family-results/F05-repro.json", "/workspace/verification-artifacts/family-results/F05-lease-adapted-repro.json" ] }, { "title": "Running Codex ACP credential write-back", "prerequisite": "Codex ACP binary plus OpenAI login not provided", "route": "External credential source PUT with expected_version/value", "notes": "F12 activation, scrubbing, binding scope and reload guard are driven; provider-driven credential refresh synchronization is not reached.", "reason": "Codex ACP binary plus OpenAI login not provided", "feature_ids": [ "F12.binding-live-acp" ] }, { "id": "canonical-docker-packaging", "feature_ids": [ "F12.docker-runtime" ], "route": "Canonical Dockerfile-built runtime image", "reason": "Canonical build exhausted the 32 GB overlay and was aborted. The compatible local image uses frozen source plus the locked host virtualenv; local lifecycle coverage does not verify canonical packaging." }, { "feature_ids": [ "F16.vscode-running" ], "reason": "No installed OpenVSCode Server binary; map hardcoded blocked." }, { "feature_ids": [ "F17.noop-write-readonly" ], "reason": "chattr executable missing (exit127); immutable-attribute capability and filesystem support also unproven. API assertions were not reached." }, { "feature_ids": [ "F15.repo-search-github" ], "route": "Invalid-token 401 branch only", "reason": "Live GitHub listing returned 200; pagination and case-insensitive AGENT-SDK search passed. The 401 negative control branch is inconclusive because api.github.com/user/repos itself returns 200 for the same fake token through the inherited egress/proxy environment. The local persisted token equals the fake fixture, so no backend token-selection/cache defect is established.", "evidence": [ "/workspace/verification-artifacts/runs/20261013-041559-37e6-map-f15/evidence/F15.repo-search-github/bad-token.json", "/workspace/verification-artifacts/runs/20261013-041559-37e6-map-f15/evidence/F15.repo-search-github/direct-negative-control.json" ], "positive_branch_evidence": [ "/workspace/verification-artifacts/runs/20261013-041559-37e6-map-f15/evidence/F15.repo-search-github/page-1.json", "/workspace/verification-artifacts/runs/20261013-041559-37e6-map-f15/evidence/F15.repo-search-github/page-2.json", "/workspace/verification-artifacts/runs/20261013-041559-37e6-map-f15/evidence/map-run/F15-git-01-live-github-listing-f15-repo-search-gith.json" ], "harness_copy": "/workspace/verification-artifacts/github-live-harness/.agents/skills/verify-agent-server/references/feature-map/F15-git.md" }, { "feature_ids": [ "F24.login-complete" ], "prerequisite": "A human completing a real ChatGPT account login", "actual": "Not attempted; throwaway device codes and fake persisted tokens test the credential-free branches without signing anyone in." }, { "feature_ids": [ "F28.backend-stop", "F28.status-exit-hang" ], "prerequisite": "A host that reaps terminated descendant PIDs before kill -0 process-absence checks", "actual": "Ordinary stop/status APIs succeeded, but the final process-absence/timing bounds are inconclusive for unreaped zombies. No extra product bug counted.", "evidence": [ "/workspace/verification-artifacts/family-results/F28.json" ] } ], "inconclusive": [ { "feature_ids": [ "F15.repo-search-github" ], "expectation": "A deliberately invalid GitHub token should produce401.", "actual": "Both the Agent Server route and direct api.github.com negative control return 200. The local store contains the fake fixture token. Egress authentication or response handling prevents an unauthenticated upstream control.", "classification": "Environment negative control limitation; no product bug", "evidence": [ "/workspace/verification-artifacts/runs/20261013-041559-37e6-map-f15/evidence/F15.repo-search-github/bad-token.json", "/workspace/verification-artifacts/runs/20261013-041559-37e6-map-f15/evidence/F15.repo-search-github/direct-negative-control.json" ] }, { "id": "F29.proxy-request-framing", "reason": "Valid HTTP rechunking changes Transfer-Encoding; GETbody/status correct and adjacent POST exact-byte control passes. No supported backend failure shown; exclude productbug count." }, { "id": "F32.deferred-init-flag", "reason": "Effective deferred_init false after /api/init is intentional and asserted by unit tests. Map assumes booted-deferred semantics without documentedcontract. Exclude productbug count." } ] }