Merge pull request 'agents: hard timeout on agent runs (hung model blocked the runner ~1h)' (#95) from fix/agent-run-timeout into main

.gitea/workflows/agent.yml

@@ -36,6 +36,9 @@ jobs:
         contains(github.event.comment.body, '@ops') ||
         contains(github.event.comment.body, '@intern')))
     runs-on: ci-runner
+    # Job-level backstop (the per-attempt `timeout` in run-agent.sh is the primary guard): a wedged
+    # job must never hold the single runner slot for hours.
+    timeout-minutes: 45
     steps:
       - name: Acknowledge with 👀
         env:

.gitea/workflows/scripts/run-agent.sh

@@ -152,14 +152,21 @@ echo "opencode version: $(opencode --version 2>&1)"
 # keeps reading agent_out.md exactly as before. Success is exit code 0: the agent may
 # make tool-only changes with no text summary, so DO NOT treat empty output as failure.
 rc=1
+# HARD per-attempt timeout: there is exactly ONE runner slot, and a hung model (a stalled local
+# ollama generate on a trivial @intern question) once held it for ~1h, queueing every agent run
+# instance-wide. 20 min is far above any legitimate attempt. timeout SIGTERMs, then SIGKILLs 30s
+# later. rc=124 (timed out) is NOT retried — a hung backend stays hung; fail fast, free the runner.
+AGENT_TIMEOUT="${AGENT_TIMEOUT:-1200}"
 for attempt in 1 2 3; do
-  echo "opencode attempt $attempt/3 for @$NAME ($MODEL)"
+  echo "opencode attempt $attempt/3 for @$NAME ($MODEL, timeout ${AGENT_TIMEOUT}s)"
   rc=0
+  timeout -k 30 "$AGENT_TIMEOUT" \
     opencode run --model "$MODEL" --auto --format json "$PROMPT" ${FILES:-} \
     >/tmp/events.jsonl 2>/tmp/agent_err.log || rc=$?
   echo "rc=$rc"; echo "--- events ($(wc -l < /tmp/events.jsonl 2>/dev/null || echo 0) lines) ---"
   echo "--- stderr (trace) ---"; cat /tmp/agent_err.log
   [ $rc -eq 0 ] && break
+  if [ $rc -eq 124 ]; then echo "attempt timed out after ${AGENT_TIMEOUT}s — backend hung, not retrying"; break; fi
   if grep -qiE 'overloaded|429|529|rate.?limit|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN' /tmp/events.jsonl /tmp/agent_err.log; then
     echo "transient error — backing off $((attempt*20))s"; sleep $((attempt * 20)); continue
   fi