From bbdb200f32b90c3b59cded806ce75619a3adfc85 Mon Sep 17 00:00:00 2001 From: hermes Date: Thu, 30 Jul 2026 16:25:46 +0300 Subject: [PATCH 1/2] fix(agents): route pm/junior/senior/qa to xAI while Ollama Cloud is exhausted Ollama Cloud usage is at 100% so ollama-cloud/* agents fail. Move pm, junior, senior, qa to xai-oc (grok-4.5 / grok-4.3). leave intern on local ollama/ornith:35b; lead/ops already on xAI. --- .gitea/workflows/scripts/agents.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitea/workflows/scripts/agents.json b/.gitea/workflows/scripts/agents.json index 3d56dba..e83daaf 100644 --- a/.gitea/workflows/scripts/agents.json +++ b/.gitea/workflows/scripts/agents.json @@ -1,28 +1,28 @@ { "pm": { - "model": "ollama-cloud/minimax-m3:cloud", + "model": "xai-oc/grok-4.5", "vision": true, "mode": "comment", "skills": [ "gitea-api" ], - "desc": "Product manager & orchestrator — plans and picks the dev, hands each finished PR to @qa for review, and reports back to the issue creator (in autopilot it merges approved PRs itself). Works from the issue thread only — comments only, never edits files, never reads the PR diff." + "desc": "Product manager & orchestrator — plans and picks the dev, hands each finished PR to @qa for review, and reports back to the issue creator (in autopilot it merges approved PRs itself). Works from the issue thread only — comments only, never edits files, never reads the PR diff. Model: grok-4.5 via xAI (Ollama Cloud quota exhausted)." }, "junior": { - "model": "ollama-cloud/kimi-k2.7-code:cloud", + "model": "xai-oc/grok-4.3", "vision": false, "mode": "pr", "skills": [], - "desc": "Junior dev — small, low-risk changes (mostly YAML/compose/config). Text-only, cannot read images. Defers complex or image tasks to @senior or @lead." + "desc": "Junior dev — small, low-risk changes (mostly YAML/compose/config). Text-only, cannot read images. Defers complex or image tasks to @senior or @lead. Model: grok-4.3 via xAI." }, "senior": { - "model": "ollama-cloud/glm-5.2:cloud", + "model": "xai-oc/grok-4.5", "vision": false, "mode": "pr", "skills": [ "gitea-api" ], - "desc": "Senior dev — complex, multi-file implementation (GLM-5.2 via Ollama Cloud, text-only)." + "desc": "Senior dev — complex, multi-file implementation (text-only). Model: grok-4.5 via xAI (moved off Ollama Cloud)." }, "lead": { "model": "xai-oc/grok-4.5", @@ -34,13 +34,13 @@ "desc": "Tech lead — the hardest problems, architecture, and final calls." }, "qa": { - "model": "ollama-cloud/minimax-m3:cloud", + "model": "xai-oc/grok-4.5", "vision": true, "mode": "comment", "skills": [ "gitea-api" ], - "desc": "QA / reviewer — reviews PRs: reads the diff, drives a headless browser (Playwright) to verify behavior, posts specific recommendations on the PR and the pass/fail verdict on the issue. Never edits code, never merges." + "desc": "QA / reviewer — reviews PRs: reads the diff, drives a headless browser (Playwright) to verify behavior, posts specific recommendations on the PR and the pass/fail verdict on the issue. Never edits code, never merges. Model: grok-4.5 via xAI." }, "ops": { "model": "xai-oc/grok-4.5", From 0ddde87b40201405abaa6ef677e72ee4343820f9 Mon Sep 17 00:00:00 2001 From: hermes Date: Thu, 30 Jul 2026 16:28:30 +0300 Subject: [PATCH 2/2] feat(agents): automatic model fallback when Ollama Cloud is exhausted Keep ollama-cloud/* as primary. Each agent may declare fallback in agents.json (xai-oc/grok-*). On quota/auth/provider failure, run-agent switches once to the fallback and continues retries. - agents.json: fallback fields for pm/junior/senior/qa/intern - route.sh: emit fallback to GITHUB_OUTPUT - agent.yml: pass FALLBACK env into run-agent - install-opencode.sh: register primary+fallback models in provider maps - run-agent.sh: failover on quota/credit/429/capacity-class errors --- .gitea/workflows/agent.yml | 1 + .gitea/workflows/scripts/agents.json | 23 ++++++++++++-------- .gitea/workflows/scripts/install-opencode.sh | 8 ++++--- .gitea/workflows/scripts/route.sh | 8 ++++--- .gitea/workflows/scripts/run-agent.sh | 18 +++++++++++++++ 5 files changed, 43 insertions(+), 15 deletions(-) diff --git a/.gitea/workflows/agent.yml b/.gitea/workflows/agent.yml index aebcd5d..7ce220e 100644 --- a/.gitea/workflows/agent.yml +++ b/.gitea/workflows/agent.yml @@ -222,6 +222,7 @@ jobs: SELF_TOKEN: ${{ steps.prep.outputs.name == 'pm' && secrets.TOKEN_PM || steps.prep.outputs.name == 'junior' && secrets.TOKEN_JUNIOR || steps.prep.outputs.name == 'senior' && secrets.TOKEN_SENIOR || steps.prep.outputs.name == 'lead' && secrets.TOKEN_LEAD || steps.prep.outputs.name == 'qa' && secrets.TOKEN_QA || steps.prep.outputs.name == 'ops' && secrets.TOKEN_OPS || steps.prep.outputs.name == 'intern' && secrets.TOKEN_INTERN || '' }} NAME: ${{ steps.prep.outputs.name }} MODEL: ${{ steps.prep.outputs.model }} + FALLBACK: ${{ steps.prep.outputs.fallback }} VISION: ${{ steps.prep.outputs.vision }} MODE: ${{ steps.prep.outputs.mode }} WORKMODE: ${{ steps.prep.outputs.workmode }} # build | discuss (devs consulted in-thread) diff --git a/.gitea/workflows/scripts/agents.json b/.gitea/workflows/scripts/agents.json index e83daaf..735fc3d 100644 --- a/.gitea/workflows/scripts/agents.json +++ b/.gitea/workflows/scripts/agents.json @@ -1,28 +1,31 @@ { "pm": { - "model": "xai-oc/grok-4.5", + "model": "ollama-cloud/minimax-m3:cloud", + "fallback": "xai-oc/grok-4.5", "vision": true, "mode": "comment", "skills": [ "gitea-api" ], - "desc": "Product manager & orchestrator — plans and picks the dev, hands each finished PR to @qa for review, and reports back to the issue creator (in autopilot it merges approved PRs itself). Works from the issue thread only — comments only, never edits files, never reads the PR diff. Model: grok-4.5 via xAI (Ollama Cloud quota exhausted)." + "desc": "Product manager & orchestrator — plans and picks the dev, hands each finished PR to @qa for review, and reports back to the issue creator (in autopilot it merges approved PRs itself). Works from the issue thread only — comments only, never edits files, never reads the PR diff. Falls back to xAI grok-4.5 if Ollama Cloud is unavailable/quota-exhausted." }, "junior": { - "model": "xai-oc/grok-4.3", + "model": "ollama-cloud/kimi-k2.7-code:cloud", + "fallback": "xai-oc/grok-4.3", "vision": false, "mode": "pr", "skills": [], - "desc": "Junior dev — small, low-risk changes (mostly YAML/compose/config). Text-only, cannot read images. Defers complex or image tasks to @senior or @lead. Model: grok-4.3 via xAI." + "desc": "Junior dev — small, low-risk changes (mostly YAML/compose/config). Text-only, cannot read images. Defers complex or image tasks to @senior or @lead. Falls back to xAI grok-4.3 if Ollama Cloud fails." }, "senior": { - "model": "xai-oc/grok-4.5", + "model": "ollama-cloud/glm-5.2:cloud", + "fallback": "xai-oc/grok-4.5", "vision": false, "mode": "pr", "skills": [ "gitea-api" ], - "desc": "Senior dev — complex, multi-file implementation (text-only). Model: grok-4.5 via xAI (moved off Ollama Cloud)." + "desc": "Senior dev — complex, multi-file implementation (GLM-5.2 via Ollama Cloud, text-only). Falls back to xAI grok-4.5 if Ollama Cloud fails." }, "lead": { "model": "xai-oc/grok-4.5", @@ -34,13 +37,14 @@ "desc": "Tech lead — the hardest problems, architecture, and final calls." }, "qa": { - "model": "xai-oc/grok-4.5", + "model": "ollama-cloud/minimax-m3:cloud", + "fallback": "xai-oc/grok-4.5", "vision": true, "mode": "comment", "skills": [ "gitea-api" ], - "desc": "QA / reviewer — reviews PRs: reads the diff, drives a headless browser (Playwright) to verify behavior, posts specific recommendations on the PR and the pass/fail verdict on the issue. Never edits code, never merges. Model: grok-4.5 via xAI." + "desc": "QA / reviewer — reviews PRs: reads the diff, drives a headless browser (Playwright) to verify behavior, posts specific recommendations on the PR and the pass/fail verdict on the issue. Never edits code, never merges. Falls back to xAI grok-4.5 if Ollama Cloud fails." }, "ops": { "model": "xai-oc/grok-4.5", @@ -53,9 +57,10 @@ }, "intern": { "model": "ollama/ornith:35b", + "fallback": "xai-oc/grok-4.3", "vision": false, "mode": "pr", "skills": [], - "desc": "Intern — very basic tasks only, routed to the local Ollama model (ornith:35b). Text-only, cannot read images. Escalates anything non-trivial to @junior, @senior or @lead." + "desc": "Intern — very basic tasks only, routed to the local Ollama model (ornith:35b). Text-only, cannot read images. Escalates anything non-trivial to @junior, @senior or @lead. Falls back to xAI grok-4.3 if local Ollama is down." } } diff --git a/.gitea/workflows/scripts/install-opencode.sh b/.gitea/workflows/scripts/install-opencode.sh index b6d705f..b6189cc 100755 --- a/.gitea/workflows/scripts/install-opencode.sh +++ b/.gitea/workflows/scripts/install-opencode.sh @@ -54,9 +54,11 @@ PERM=$(jq -nc --argjson s "$SKILLS" ' # 1.17.13 — renaming the key to `xai-oc` avoids the collision entirely. See issue #118. # See issue #31. AGENTS_JSON="${SCRIPTS:-$(dirname -- "$0")}/agents.json" -CLOUD_MODELS=$(jq -r '[.[] | .model | select(startswith("ollama-cloud/")) | sub("^ollama-cloud/";"")] | map({(.):{}}) | add // {}' "$AGENTS_JSON") -LOCAL_MODELS=$(jq -r '[.[] | .model | select(startswith("ollama/")) | sub("^ollama/";"")] | map({(.):{}}) | add // {"ornith:35b":{}}' "$AGENTS_JSON") -XAI_MODELS=$(jq -r '[.[] | .model | select(startswith("xai-oc/")) | sub("^xai-oc/";"")] | map({(.):{}}) | add // {}' "$AGENTS_JSON") +# Include primary `.model` AND optional `.fallback` so failover models are always registered +# in opencode provider maps (issue: Ollama Cloud quota → xAI). +CLOUD_MODELS=$(jq -r '[.[] | (.model, .fallback) | select(type=="string" and startswith("ollama-cloud/")) | sub("^ollama-cloud/";"")] | unique | map({(.):{}}) | add // {}' "$AGENTS_JSON") +LOCAL_MODELS=$(jq -r '[.[] | (.model, .fallback) | select(type=="string" and startswith("ollama/")) | sub("^ollama/";"")] | unique | map({(.):{}}) | add // {"ornith:35b":{}}' "$AGENTS_JSON") +XAI_MODELS=$(jq -r '[.[] | (.model, .fallback) | select(type=="string" and startswith("xai-oc/")) | sub("^xai-oc/";"")] | unique | map({(.):{}}) | add // {}' "$AGENTS_JSON") jq -n --argjson mcp "$MCP" --argjson perm "$PERM" --argjson cloud "$CLOUD_MODELS" --argjson local "$LOCAL_MODELS" --argjson xai "$XAI_MODELS" --arg url "$OLLAMA_URL" --arg ckey "$OLLAMA_CLOUD_API_KEY" --arg xkey "$XAI_API_KEY" '{ provider: { ollama: {npm:"@ai-sdk/openai-compatible", options:{baseURL:($url+"/v1")}, models:$local}, diff --git a/.gitea/workflows/scripts/route.sh b/.gitea/workflows/scripts/route.sh index 9da4248..d15ead3 100755 --- a/.gitea/workflows/scripts/route.sh +++ b/.gitea/workflows/scripts/route.sh @@ -43,12 +43,14 @@ if [ -z "$name" ]; then # Not an agent task (e.g. the gate's contains() matched "@internal"). Skip GRACEFULLY: emit # mode=skip so every later step no-ops — a red run for a non-agent comment is just noise. echo "no known agent mentioned (word-boundary) — skipping run" - { echo "name=none"; echo "model=none"; echo "vision=false"; echo "mode=skip"; echo "skills=[]"; + { echo "name=none"; echo "model=none"; echo "fallback="; echo "vision=false"; echo "mode=skip"; echo "skills=[]"; echo "branch=main"; echo "new=false"; echo "autopilot=false"; echo "issnum=$NUM"; } >> "$GITHUB_OUTPUT" exit 0 fi fi model=$(jq -r --arg a "$name" '.[$a].model' /tmp/agents.json) +# Optional provider failover when primary is quota/network-dead (e.g. Ollama Cloud → xAI). +fallback=$(jq -r --arg a "$name" '.[$a].fallback // empty' /tmp/agents.json) vision=$(jq -r --arg a "$name" '.[$a].vision' /tmp/agents.json) mode=$(jq -r --arg a "$name" '.[$a].mode' /tmp/agents.json) # Compact JSON array of the skills this agent may load (scopes permission.skill in install-opencode.sh). @@ -70,8 +72,8 @@ if [ "$mode" = "pr" ] && [ -z "$IS_PR" ]; then workmode=discuss fi fi -echo "Routing to @$name (model=$model vision=$vision mode=$mode workmode=$workmode skills=$skills)" -{ echo "name=$name"; echo "model=$model"; echo "vision=$vision"; echo "mode=$mode"; echo "workmode=$workmode"; echo "skills=$skills"; } >> "$GITHUB_OUTPUT" +echo "Routing to @$name (model=$model fallback=${fallback:-none} vision=$vision mode=$mode workmode=$workmode skills=$skills)" +{ echo "name=$name"; echo "model=$model"; echo "fallback=$fallback"; echo "vision=$vision"; echo "mode=$mode"; echo "workmode=$workmode"; echo "skills=$skills"; } >> "$GITHUB_OUTPUT" # Act as the agent's own Gitea user when its token is set; else the built-in bot. case "$name" in diff --git a/.gitea/workflows/scripts/run-agent.sh b/.gitea/workflows/scripts/run-agent.sh index c5a7c9b..1c4eb24 100755 --- a/.gitea/workflows/scripts/run-agent.sh +++ b/.gitea/workflows/scripts/run-agent.sh @@ -186,6 +186,16 @@ rc=1 # instance-wide. 20 min is far above any legitimate attempt. timeout SIGTERMs, then SIGKILLs 30s # later. rc=124 (timed out) is NOT retried — a hung backend stays hung; fail fast, free the runner. AGENT_TIMEOUT="${AGENT_TIMEOUT:-1200}" +# Provider failover: when primary model dies on quota/auth/provider errors, switch once to +# FALLBACK (from agents.json) and continue the retry loop. Transient rate-limits still back off +# on the current model first. +FALLBACK_MODEL="${FALLBACK:-}" +fallback_used=0 +is_failover_error() { + # Ollama Cloud exhausted / provider hard-fail — switch to fallback rather than thrash. + grep -qiE 'overloaded|429|529|rate.?limit|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN|quota|credit|balance|usage.?limit|limit.?exceed|402|403|401|insufficient|out of credits|payment.?required|model_not_found|not found|Unavailable|capacity|ENOTFOUND|ECONNREFUSED' \ + /tmp/events.jsonl /tmp/agent_err.log 2>/dev/null +} for attempt in 1 2 3; do echo "opencode attempt $attempt/3 for @$NAME ($MODEL, timeout ${AGENT_TIMEOUT}s)" rc=0 @@ -196,6 +206,14 @@ for attempt in 1 2 3; do echo "--- stderr (trace) ---"; cat /tmp/agent_err.log [ $rc -eq 0 ] && break if [ $rc -eq 124 ]; then echo "attempt timed out after ${AGENT_TIMEOUT}s — backend hung, not retrying"; break; fi + if [ $fallback_used -eq 0 ] && [ -n "$FALLBACK_MODEL" ] && [ "$FALLBACK_MODEL" != "$MODEL" ] && is_failover_error; then + echo "primary model failed — failing over to fallback: $FALLBACK_MODEL" + MODEL="$FALLBACK_MODEL" + fallback_used=1 + # short pause then use next attempt slot on the fallback provider + sleep 2 + continue + fi if grep -qiE 'overloaded|429|529|rate.?limit|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN' /tmp/events.jsonl /tmp/agent_err.log; then echo "transient error — backing off $((attempt*20))s"; sleep $((attempt * 20)); continue fi