From 0771d394bd46a5bc315f7d1631b87b3460db29ba Mon Sep 17 00:00:00 2001
From: Felix Faerber <ffaerber@gmail.com>
Date: Mon, 6 Jul 2026 17:41:49 +0300
Subject: [PATCH] =?UTF-8?q?agents:=20hard=20timeout=20on=20agent=20runs=20?=
 =?UTF-8?q?=E2=80=94=20a=20hung=20model=20must=20not=20block=20the=20runne?=
 =?UTF-8?q?r?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A stalled local-ollama generate on a trivial @intern question held the single
runner slot for ~1h, queueing every agent run instance-wide (homelab run 869).

- run-agent.sh: each opencode attempt wrapped in `timeout -k 30 $AGENT_TIMEOUT`
  (default 1200s). rc=124 is not retried — a hung backend stays hung; fail fast
  and free the runner.
- agent.yml: job-level timeout-minutes: 45 as backstop.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .gitea/workflows/agent.yml            |  3 +++
 .gitea/workflows/scripts/run-agent.sh | 11 +++++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/.gitea/workflows/agent.yml b/.gitea/workflows/agent.yml
index aebc4b8..0345229 100644
--- a/.gitea/workflows/agent.yml
+++ b/.gitea/workflows/agent.yml
@@ -36,6 +36,9 @@ jobs:
         contains(github.event.comment.body, '@ops') ||
         contains(github.event.comment.body, '@intern')))
     runs-on: ci-runner
+    # Job-level backstop (the per-attempt `timeout` in run-agent.sh is the primary guard): a wedged
+    # job must never hold the single runner slot for hours.
+    timeout-minutes: 45
     steps:
       - name: Acknowledge with 👀
         env:
diff --git a/.gitea/workflows/scripts/run-agent.sh b/.gitea/workflows/scripts/run-agent.sh
index 38a2f49..993adc6 100755
--- a/.gitea/workflows/scripts/run-agent.sh
+++ b/.gitea/workflows/scripts/run-agent.sh
@@ -152,14 +152,21 @@ echo "opencode version: $(opencode --version 2>&1)"
 # keeps reading agent_out.md exactly as before. Success is exit code 0: the agent may
 # make tool-only changes with no text summary, so DO NOT treat empty output as failure.
 rc=1
+# HARD per-attempt timeout: there is exactly ONE runner slot, and a hung model (a stalled local
+# ollama generate on a trivial @intern question) once held it for ~1h, queueing every agent run
+# instance-wide. 20 min is far above any legitimate attempt. timeout SIGTERMs, then SIGKILLs 30s
+# later. rc=124 (timed out) is NOT retried — a hung backend stays hung; fail fast, free the runner.
+AGENT_TIMEOUT="${AGENT_TIMEOUT:-1200}"
 for attempt in 1 2 3; do
-  echo "opencode attempt $attempt/3 for @$NAME ($MODEL)"
+  echo "opencode attempt $attempt/3 for @$NAME ($MODEL, timeout ${AGENT_TIMEOUT}s)"
   rc=0
-  opencode run --model "$MODEL" --auto --format json "$PROMPT" ${FILES:-} \
+  timeout -k 30 "$AGENT_TIMEOUT" \
+    opencode run --model "$MODEL" --auto --format json "$PROMPT" ${FILES:-} \
     >/tmp/events.jsonl 2>/tmp/agent_err.log || rc=$?
   echo "rc=$rc"; echo "--- events ($(wc -l < /tmp/events.jsonl 2>/dev/null || echo 0) lines) ---"
   echo "--- stderr (trace) ---"; cat /tmp/agent_err.log
   [ $rc -eq 0 ] && break
+  if [ $rc -eq 124 ]; then echo "attempt timed out after ${AGENT_TIMEOUT}s — backend hung, not retrying"; break; fi
   if grep -qiE 'overloaded|429|529|rate.?limit|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN' /tmp/events.jsonl /tmp/agent_err.log; then
     echo "transient error — backing off $((attempt*20))s"; sleep $((attempt * 20)); continue
   fi
-- 
2.54.0