{
  "file": "F31-persistence-and-limits.md",
  "bullet": "Crash during a tool call (`F31.crash-tool-call`).",
  "ids": [
    "F31.crash-tool-call"
  ],
  "line": 191,
  "script": "control-agent-server conversation send \"$T\" --text 'hi' --no-run\ncontrol-agent-server api POST \"<redacted:session-cookie>/$T/run\" --expect 200 --check success eq true\ncontrol-agent-server api GET \"<redacted:session-cookie>/$T/events/count\" --query kind=$ACTION_KIND --check . eq 1 --until-ok 30\nTCALL=$(control-agent-server api GET \"<redacted:session-cookie>/$T/events/search\" --query kind=$ACTION_KIND --field items.0.tool_call_id)\ncontrol-agent-server state cat base_state.json --conversation \"$T\" --check execution_status eq running\ncontrol-agent-server restart --hard\nPIDT=$(jq -r .pid \"$AGENT_SERVER_VERIFY_RUN/run.json\")\ncontrol-agent-server state cat owner_lease.json --conversation \"$T\" --check owner_pid eq \"$PIDT\"\ncontrol-agent-server api GET \"<redacted:session-cookie>/$T\" --check execution_status eq error --quiet --save F31.crash-tool-call/get\ncontrol-agent-server api GET \"<redacted:session-cookie>/$T/events/search\" --query kind=$AGENT_ERROR_KIND --check items len-eq 1 \\\n  --check items.0.tool_call_id eq \"$TCALL\" --check items.0.error contains 'A restart occurred while this tool was in progress' \\\n  --save F31.crash-tool-call/agent-error\ncontrol-agent-server api GET \"<redacted:session-cookie>/$T/events/count\" --query kind=$OBSERVATION_KIND --check . eq 0\ncontrol-agent-server fixture llm-stub --name qa-f31-tool --step 'tool:finish:{\"message\":\"QA_F31_RESUMED\"}'\ncontrol-agent-server api POST \"<redacted:session-cookie>/$T/run\" --expect 200 --check success eq true\ncontrol-agent-server conversation wait \"$T\" --until finished --timeout 60\ncontrol-agent-server api GET \"<redacted:session-cookie>/$T/agent_final_response\" --check response eq QA_F31_RESUMED\ncontrol-agent-server sink read --name qa-f31-tool --contains 'A restart occurred while this tool was in progress' --expect-min 1 \\\n  --save F31.crash-tool-call/resumed-request\n",
  "exit_code": 0,
  "elapsed_ms": 80353,
  "failed_line": null,
  "failed_command": null,
  "stdout": "{\n  \"ok\": true,\n  \"status\": 200,\n  \"body\": {\n    \"success\": true\n  }\n}\n{\n  \"ok\": true,\n  \"status\": 200,\n  \"request\": {\n    \"method\": \"POST\",\n    \"path\": \"<redacted:session-cookie>/39cbc9e6-8928-4497-80da-b59b1c8ee9f9/run\",\n    \"query\": [],\n    \"auth\": \"header\",\n    \"body\": null,\n    \"files\": []\n  },\n  \"response\": {\n    \"status\": 200,\n    \"headers\": {\n      \"content-length\": \"16\",\n      \"content-type\": \"application/json\"\n    },\n    \"body\": {\n      \"success\": true\n    },\n    \"truncated\": false\n  },\n  \"elapsed_ms\": 319,\n  \"checks\": [\n    {\n      \"check\": \"success eq true\",\n      \"ok\": true,\n      \"actual\": \"true\"\n    }\n  ]\n}\n{\n  \"ok\": true,\n  \"status\": 200,\n  \"request\": {\n    \"method\": \"GET\",\n    \"path\": \"<redacted:session-cookie>/39cbc9e6-8928-4497-80da-b59b1c8ee9f9/events/count\",\n    \"query\": [\n      [\n        \"kind\",\n        \"openhands.sdk.event.llm_convertible.action.ActionEvent\"\n      ]\n    ],\n    \"auth\": \"header\",\n    \"body\": null,\n    \"files\": []\n  },\n  \"response\": {\n    \"status\": 200,\n    \"headers\": {\n      \"content-length\": \"1\",\n      \"content-type\": \"application/json\"\n    },\n    \"body\": 1,\n    \"truncated\": false\n  },\n  \"elapsed_ms\": 344,\n  \"checks\": [\n    {\n      \"check\": \". eq 1\",\n      \"ok\": true,\n      \"actual\": \"1\"\n    }\n  ],\n  \"attempts\": 1\n}\n{\n  \"ok\": true,\n  \"path\": \"base_state.json\",\n  \"bytes\": 2323,\n  \"mode\": \"0o600\",\n  \"content\": {\n    \"id\": \"39cbc9e6-8928-4497-80da-b59b1c8ee9f9\",\n    \"agent\": {\n      \"llm\": {\n        \"model\": \"openai/qa-stub\",\n        \"api_key\": \"<redacted:encrypted>\",\n        \"auth_type\": \"api_key\",\n        \"base_url\": \"http://127.0.0.1:50219/v1\",\n        \"openrouter_site_url\": \"https://docs.all-hands.dev/\",\n        \"openrouter_app_name\": \"OpenHands\",\n        \"num_retries\": 0,\n        \"retry_multiplier\": 8.0,\n        \"retry_min_wait\": 8,\n        \"retry_max_wait\": 64,\n        \"timeout\": 300,\n        \"stream_idle_timeout\": 300.0,\n        \"max_message_chars\": 30000,\n        \"api_mode\": \"auto\",\n        \"capability_overrides\": {},\n        \"stream\": false,\n        \"drop_params\": true,\n        \"disable_stop_word\": false,\n        \"caching_prompt\": true,\n        \"log_completions\": false,\n        \"log_completions_folder\": \"logs/completions\",\n        \"native_tool_calling\": true,\n        \"reasoning_effort\": \"high\",\n        \"enable_encrypted_reasoning\": true,\n        \"prompt_cache_retention\": \"24h\",\n        \"extended_thinking_budget\": 200000,\n        \"usage_id\": \"default\",\n        \"litellm_extra_body\": {},\n        \"is_subscription\": false\n      },\n      \"tools\": [\n        {\n          \"name\": \"terminal\",\n          \"params\": {}\n        }\n      ],\n      \"mcp_config\": {},\n      \"include_default_tools\": [\n        \"FinishTool\",\n        \"ThinkTool\"\n      ],\n      \"system_prompt_filename\": \"system_prompt.j2\",\n      \"security_policy_filename\": \"security_policy.j2\",\n      \"system_prompt_kwargs\": {\n        \"llm_security_analyzer\": true\n      },\n      \"tool_concurrency_limit\": 1,\n      \"kind\": \"Agent\"\n    },\n    \"workspace\": {\n      \"working_dir\": \"/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/fixtures/workspaces/021905-232714\",\n      \"kind\": \"LocalWorkspace\"\n    },\n    \"persistence_dir\": \"/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/server/workspace/conversations/39cbc9e68928449780dab59b1c8ee9f9\",\n    \"max_iterations\": 500,\n    \"stuck_detection\": true,\n    \"execution_status\": \"running\",\n    \"confirmation_policy\": {\n      \"kind\": \"NeverConfirm\"\n    },\n    \"activated_knowledge_skills\": [],\n    \"activated_path_rules\": [],\n    \"invoked_skills\": [],\n    \"blocked_actions\": {},\n    \"blocked_messages\": {},\n    \"last_user_message_id\": \"b4942625-73fe-42cc-8e30-2f7ce5862047\",\n    \"leaf_event_id\": \"b4942625-73fe-42cc-8e30-2f7ce5862047\",\n    \"head_is_empty\": false,\n    \"stats\": {\n      \"usage_to_metrics\": {\n        \"default\": {\n          \"model_name\": \"openai/qa-stub\",\n          \"accumulated_cost\": 0.0,\n          \"accumulated_token_usage\": {\n            \"model\": \"openai/qa-stub\",\n            \"prompt_tokens\": 0,\n            \"completion_tokens\": 0,\n            \"cache_read_tokens\": 0,\n            \"cache_write_tokens\": 0,\n            \"reasoning_tokens\": 0,\n            \"context_window\": 0,\n            \"per_turn_token\": 0,\n            \"response_id\": \"\"\n          },\n          \"costs\": [],\n          \"response_latencies\": [],\n          \"token_usages\": []\n        }\n      }\n    },\n    \"secret_registry\": {\n      \"secret_sources\": {}\n    },\n    \"tags\": {},\n    \"agent_state\": {}\n  },\n  \"checks\": [\n    {\n      \"check\": \"execution_status eq running\",\n      \"ok\": true,\n      \"actual\": \"running\"\n    }\n  ]\n}\n{\n  \"ok\": true,\n  \"run\": \"/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31\",\n  \"url\": \"http://127.0.0.1:43785\",\n  \"rotated_key\": false,\n  \"hard\": true,\n  \"rotated_secret_key\": false,\n  \"ready_ms\": 44541,\n  \"ready_path\": \"/ready\"\n}\n{\n  \"ok\": true,\n  \"path\": \"owner_lease.json\",\n  \"bytes\": 158,\n  \"mode\": \"0o600\",\n  \"content\": {\n    \"owner_instance_id\": \"56286cf1703a43518ce599f3b3f8198d\",\n    \"generation\": 2,\n    \"expires_at\": 1791858167.1804273,\n    \"owner_host\": \"e7ee1a756865\",\n    \"owner_pid\": 12077\n  },\n  \"checks\": [\n    {\n      \"check\": \"owner_pid eq 12077\",\n      \"ok\": true,\n      \"actual\": \"12077\"\n    }\n  ]\n}\n{\n  \"ok\": true,\n  \"status\": 200,\n  \"request\": {\n    \"method\": \"GET\",\n    \"path\": \"<redacted:session-cookie>/39cbc9e6-8928-4497-80da-b59b1c8ee9f9\",\n    \"query\": [],\n    \"auth\": \"header\",\n    \"files\": []\n  },\n  \"response\": {\n    \"status\": 200,\n    \"headers\": {\n      \"content-length\": \"3530\",\n      \"content-type\": \"application/json\"\n    },\n    \"truncated\": false\n  },\n  \"elapsed_ms\": 439,\n  \"checks\": [\n    {\n      \"check\": \"execution_status eq error\",\n      \"ok\": true,\n      \"actual\": \"error\"\n    }\n  ],\n  \"saved\": \"/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/F31.crash-tool-call/get.json\"\n}\n{\n  \"ok\": true,\n  \"status\": 200,\n  \"request\": {\n    \"method\": \"GET\",\n    \"path\": \"<redacted:session-cookie>/39cbc9e6-8928-4497-80da-b59b1c8ee9f9/events/search\",\n    \"query\": [\n      [\n        \"kind\",\n        \"openhands.sdk.event.llm_convertible.observation.AgentErrorEvent\"\n      ]\n    ],\n    \"auth\": \"header\",\n    \"body\": null,\n    \"files\": []\n  },\n  \"response\": {\n    \"status\": 200,\n    \"headers\": {\n      \"content-length\": \"519\",\n      \"content-type\": \"application/json\"\n    },\n    \"body\": {\n      \"items\": [\n        {\n          \"id\": \"33f75209-2179-4e5a-be6e-6247d635f545\",\n          \"timestamp\": \"2026-10-13T02:22:02.254167\",\n          \"source\": \"agent\",\n          \"parent_id\": \"6ccfa4fa-9277-4eaf-b4b5-50f4064e51a1\",\n          \"tool_name\": \"terminal\",\n          \"tool_call_id\": \"call_qa_2770c653\",\n          \"error\": \"A restart occurred while this tool was in progress. This may indicate a fatal memory error or system crash. The tool execution was interrupted and did not complete.\",\n          \"classification\": {\n            \"kind\": \"internal\",\n            \"retryable\": false,\n            \"user_action\": \"none\"\n          },\n          \"kind\": \"AgentErrorEvent\"\n        }\n      ],\n      \"next_page_id\": null\n    },\n    \"truncated\": false\n  },\n  \"elapsed_ms\": 420,\n  \"checks\": [\n    {\n      \"check\": \"items len-eq 1\",\n      \"ok\": true,\n      \"actual\": \"[{\\\"id\\\": \\\"33f75209-2179-4e5a-be6e-6247d635f545\\\", \\\"timestamp\\\": \\\"2026-10-13T02:22:02.254167\\\", \\\"source\\\": \\\"agent\\\", \\\"parent_id\\\": \\\"6ccfa4fa-9277-4eaf-b4b5-50f4064e51a1\\\", \\\"tool_name\\\": \\\"terminal\\\", \\\"tool_call_i\\u2026\"\n    },\n    {\n      \"check\": \"items.0.tool_call_id eq call_qa_2770c653\",\n      \"ok\": true,\n      \"actual\": \"call_qa_2770c653\"\n    },\n    {\n      \"check\": \"items.0.error contains A restart occurred while this tool was in progress\",\n      \"ok\": true,\n      \"actual\": \"A restart occurred while this tool was in progress. This may indicate a fatal memory error or system crash. The tool execution was interrupted and did not complete.\"\n    }\n  ],\n  \"saved\": \"/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/F31.crash-tool-call/agent-error.json\"\n}\n{\n  \"ok\": true,\n  \"status\": 200,\n  \"request\": {\n    \"method\": \"GET\",\n    \"path\": \"<redacted:session-cookie>/39cbc9e6-8928-4497-80da-b59b1c8ee9f9/events/count\",\n    \"query\": [\n      [\n        \"kind\",\n        \"openhands.sdk.event.llm_convertible.observation.ObservationEvent\"\n      ]\n    ],\n    \"auth\": \"header\",\n    \"body\": null,\n    \"files\": []\n  },\n  \"response\": {\n    \"status\": 200,\n    \"headers\": {\n      \"content-length\": \"1\",\n      \"content-type\": \"application/json\"\n    },\n    \"body\": 0,\n    \"truncated\": false\n  },\n  \"elapsed_ms\": 439,\n  \"checks\": [\n    {\n      \"check\": \". eq 0\",\n      \"ok\": true,\n      \"actual\": \"0\"\n    }\n  ]\n}\n{\n  \"ok\": true,\n  \"kind\": \"llm-stub\",\n  \"path\": \"/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/fixtures/llm-stub/qa-f31-tool.json\",\n  \"url\": \"http://127.0.0.1:50219\",\n  \"steps\": 1,\n  \"already_running\": true,\n  \"requests_log\": \"/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/fixtures/qa-f31-tool.jsonl\"\n}\n{\n  \"ok\": true,\n  \"status\": 200,\n  \"request\": {\n    \"method\": \"POST\",\n    \"path\": \"<redacted:session-cookie>/39cbc9e6-8928-4497-80da-b59b1c8ee9f9/run\",\n    \"query\": [],\n    \"auth\": \"header\",\n    \"body\": null,\n    \"files\": []\n  },\n  \"response\": {\n    \"status\": 200,\n    \"headers\": {\n      \"content-length\": \"16\",\n      \"content-type\": \"application/json\"\n    },\n    \"body\": {\n      \"success\": true\n    },\n    \"truncated\": false\n  },\n  \"elapsed_ms\": 379,\n  \"checks\": [\n    {\n      \"check\": \"success eq true\",\n      \"ok\": true,\n      \"actual\": \"true\"\n    }\n  ]\n}\n{\n  \"id\": \"39cbc9e6-8928-4497-80da-b59b1c8ee9f9\",\n  \"ok\": true,\n  \"status\": \"finished\",\n  \"transitions\": [\n    \"running\",\n    \"finished\"\n  ],\n  \"waited_ms\": 1655\n}\n{\n  \"ok\": true,\n  \"status\": 200,\n  \"request\": {\n    \"method\": \"GET\",\n    \"path\": \"<redacted:session-cookie>/39cbc9e6-8928-4497-80da-b59b1c8ee9f9/agent_final_response\",\n    \"query\": [],\n    \"auth\": \"header\",\n    \"body\": null,\n    \"files\": []\n  },\n  \"response\": {\n    \"status\": 200,\n    \"headers\": {\n      \"content-length\": \"29\",\n      \"content-type\": \"application/json\"\n    },\n    \"body\": {\n      \"response\": \"QA_F31_RESUMED\"\n    },\n    \"truncated\": false\n  },\n  \"elapsed_ms\": 464,\n  \"checks\": [\n    {\n      \"check\": \"response eq QA_F31_RESUMED\",\n      \"ok\": true,\n      \"actual\": \"QA_F31_RESUMED\"\n    }\n  ]\n}\n{\n  \"ok\": true,\n  \"requests\": 1,\n  \"paths\": {\n    \"/v1/chat/completions\": 1\n  },\n  \"last\": [\n    \"{\\\"at\\\": \\\"2026-10-13T02:22:18Z\\\", \\\"method\\\": \\\"POST\\\", \\\"path\\\": \\\"/v1/chat/completions\\\", \\\"call\\\": 1, \\\"step\\\": {\\\"tool\\\": \\\"finish\\\", \\\"args\\\": {\\\"message\\\": \\\"QA_F31_RESUMED\\\"}}, \\\"body\\\": {\\\"messages\\\": [{\\\"content\\\": [{\\\"type\\\": \\\"text\\\", \\\"text\\\": \\\"<SOUL>\\\\nYou are OpenHands agent, a helpful AI assistant that can interact with a computer to solve tasks.\\\\n</SOUL>\\\\n\\\\n<ROLE>\\\\n* Your primary role is to assist users by executing co\\u2026\"\n  ],\n  \"saved\": \"/workspace/verification-artifacts/runs/20261013-021830-e72b-map-f31/evidence/F31.crash-tool-call/resumed-request.json\"\n}\n",
  "stderr": ""
}