diff --git a/.gitea/workflows/agent.yml b/.gitea/workflows/agent.yml index aebc4b8..0345229 100644 --- a/.gitea/workflows/agent.yml +++ b/.gitea/workflows/agent.yml @@ -36,6 +36,9 @@ jobs: contains(github.event.comment.body, '@ops') || contains(github.event.comment.body, '@intern'))) runs-on: ci-runner + # Job-level backstop (the per-attempt `timeout` in run-agent.sh is the primary guard): a wedged + # job must never hold the single runner slot for hours. + timeout-minutes: 45 steps: - name: Acknowledge with 👀 env: diff --git a/.gitea/workflows/scripts/run-agent.sh b/.gitea/workflows/scripts/run-agent.sh index 38a2f49..993adc6 100755 --- a/.gitea/workflows/scripts/run-agent.sh +++ b/.gitea/workflows/scripts/run-agent.sh @@ -152,14 +152,21 @@ echo "opencode version: $(opencode --version 2>&1)" # keeps reading agent_out.md exactly as before. Success is exit code 0: the agent may # make tool-only changes with no text summary, so DO NOT treat empty output as failure. rc=1 +# HARD per-attempt timeout: there is exactly ONE runner slot, and a hung model (a stalled local +# ollama generate on a trivial @intern question) once held it for ~1h, queueing every agent run +# instance-wide. 20 min is far above any legitimate attempt. timeout SIGTERMs, then SIGKILLs 30s +# later. rc=124 (timed out) is NOT retried — a hung backend stays hung; fail fast, free the runner. +AGENT_TIMEOUT="${AGENT_TIMEOUT:-1200}" for attempt in 1 2 3; do - echo "opencode attempt $attempt/3 for @$NAME ($MODEL)" + echo "opencode attempt $attempt/3 for @$NAME ($MODEL, timeout ${AGENT_TIMEOUT}s)" rc=0 - opencode run --model "$MODEL" --auto --format json "$PROMPT" ${FILES:-} \ + timeout -k 30 "$AGENT_TIMEOUT" \ + opencode run --model "$MODEL" --auto --format json "$PROMPT" ${FILES:-} \ >/tmp/events.jsonl 2>/tmp/agent_err.log || rc=$? echo "rc=$rc"; echo "--- events ($(wc -l < /tmp/events.jsonl 2>/dev/null || echo 0) lines) ---" echo "--- stderr (trace) ---"; cat /tmp/agent_err.log [ $rc -eq 0 ] && break + if [ $rc -eq 124 ]; then echo "attempt timed out after ${AGENT_TIMEOUT}s — backend hung, not retrying"; break; fi if grep -qiE 'overloaded|429|529|rate.?limit|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN' /tmp/events.jsonl /tmp/agent_err.log; then echo "transient error — backing off $((attempt*20))s"; sleep $((attempt * 20)); continue fi