gitea/agents
8
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

agents: hard timeout on agent runs (hung model blocked the runner ~1h) #95

Merged
ffaerber merged 1 commits from fix/agent-run-timeout into main 2026-07-06 16:58:20 +02:00
Conversation 0 Commits 1 Files Changed 2
+12 -2
2 changed files with 12 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitea/workflows/agent.yml
+3
View File
@@ -36,6 +36,9 @@ jobs:
contains(github.event.comment.body, '@ops') || contains(github.event.comment.body, '@ops') ||
contains(github.event.comment.body, '@intern'))) contains(github.event.comment.body, '@intern')))
runs-on: ci-runner runs-on: ci-runner
# Job-level backstop (the per-attempt `timeout` in run-agent.sh is the primary guard): a wedged
# job must never hold the single runner slot for hours.
timeout-minutes: 45
steps: steps:
- name: Acknowledge with 👀 - name: Acknowledge with 👀
env: env:
.gitea/workflows/scripts/run-agent.sh
+8 -1
View File
@@ -152,14 +152,21 @@ echo "opencode version: $(opencode --version 2>&1)"
# keeps reading agent_out.md exactly as before. Success is exit code 0: the agent may # keeps reading agent_out.md exactly as before. Success is exit code 0: the agent may
# make tool-only changes with no text summary, so DO NOT treat empty output as failure. # make tool-only changes with no text summary, so DO NOT treat empty output as failure.
rc=1 rc=1
# HARD per-attempt timeout: there is exactly ONE runner slot, and a hung model (a stalled local
# ollama generate on a trivial @intern question) once held it for ~1h, queueing every agent run
# instance-wide. 20 min is far above any legitimate attempt. timeout SIGTERMs, then SIGKILLs 30s
# later. rc=124 (timed out) is NOT retried — a hung backend stays hung; fail fast, free the runner.
AGENT_TIMEOUT="${AGENT_TIMEOUT:-1200}"
for attempt in 1 2 3; do for attempt in 1 2 3; do
echo "opencode attempt $attempt/3 for @$NAME ($MODEL)" echo "opencode attempt $attempt/3 for @$NAME ($MODEL, timeout ${AGENT_TIMEOUT}s)"
rc=0 rc=0
timeout -k 30 "$AGENT_TIMEOUT" \
opencode run --model "$MODEL" --auto --format json "$PROMPT" ${FILES:-} \ opencode run --model "$MODEL" --auto --format json "$PROMPT" ${FILES:-} \
>/tmp/events.jsonl 2>/tmp/agent_err.log || rc=$? >/tmp/events.jsonl 2>/tmp/agent_err.log || rc=$?
echo "rc=$rc"; echo "--- events ($(wc -l < /tmp/events.jsonl 2>/dev/null || echo 0) lines) ---" echo "rc=$rc"; echo "--- events ($(wc -l < /tmp/events.jsonl 2>/dev/null || echo 0) lines) ---"
echo "--- stderr (trace) ---"; cat /tmp/agent_err.log echo "--- stderr (trace) ---"; cat /tmp/agent_err.log
[ $rc -eq 0 ] && break [ $rc -eq 0 ] && break
if [ $rc -eq 124 ]; then echo "attempt timed out after ${AGENT_TIMEOUT}s — backend hung, not retrying"; break; fi
if grep -qiE 'overloaded|429|529|rate.?limit|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN' /tmp/events.jsonl /tmp/agent_err.log; then if grep -qiE 'overloaded|429|529|rate.?limit|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN' /tmp/events.jsonl /tmp/agent_err.log; then
echo "transient error — backing off $((attempt*20))s"; sleep $((attempt * 20)); continue echo "transient error — backing off $((attempt*20))s"; sleep $((attempt * 20)); continue
fi fi