fix(agents): declare workflow_call secrets + diagnose empty XAI_API_KEY

Cross-owner reusable calls left XAI_API_KEY empty in the runner while
OLLAMA_CLOUD_API_KEY worked. Declare secrets on workflow_call, accept
alternate secret names, log key lengths (not values), and ship an
ai-agent.yml caller template with an explicit secrets map.

.gitea/workflows/agent.yml

@@ -4,6 +4,31 @@ name: agent
 # The gate + steps run in the caller's event context (github.event.* / github.repository are the caller's).
 on:
   workflow_call:
+    # Explicit secret contract so callers can map secrets by name (more reliable than
+    # secrets: inherit alone on some Gitea versions / cross-owner reusable workflows).
+    secrets:
+      GITEA_TOKEN:
+        required: true
+      OLLAMA_URL:
+        required: false
+      OLLAMA_CLOUD_API_KEY:
+        required: false
+      XAI_API_KEY:
+        required: false
+      TOKEN_PM:
+        required: false
+      TOKEN_SENIOR:
+        required: false
+      TOKEN_JUNIOR:
+        required: false
+      TOKEN_LEAD:
+        required: false
+      TOKEN_QA:
+        required: false
+      TOKEN_OPS:
+        required: false
+      TOKEN_INTERN:
+        required: false
 
 # Pinned opencode version — used to install it and to key the CI cache below.
 env:
@@ -25,9 +50,10 @@ jobs:
     # Trusted author only, and only when a known agent is mentioned. This gate is the main
     # defense against malicious-issue prompt injection — do not loosen it.
     if: >
-      (github.event.comment == null && github.event.issue.user.login == 'ffaerber') ||
+      (github.event.comment == null && (github.event.issue.user.login == 'ffaerber' || github.event.issue.user.login == 'hermes')) ||
       (github.event.comment != null &&
        (github.event.comment.user.login == 'ffaerber' ||
+        github.event.comment.user.login == 'hermes' ||
         github.event.comment.user.login == 'pm' ||
         github.event.comment.user.login == 'junior' ||
         github.event.comment.user.login == 'senior' ||
@@ -149,7 +175,9 @@ jobs:
           SCRIPTS: ${{ runner.temp }}/agents-scripts
           OLLAMA_URL: ${{ secrets.OLLAMA_URL }}
           OLLAMA_CLOUD_API_KEY: ${{ secrets.OLLAMA_CLOUD_API_KEY }}
-          XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
+          # Accept common alternate names — empty XAI_API_KEY has bitten us when the
+          # secret was stored under a slightly different key on the caller repo.
+          XAI_API_KEY: ${{ secrets.XAI_API_KEY || secrets.XAI_KEY || secrets.GROK_API_KEY || secrets.XAI_TOKEN }}
           NAME: ${{ steps.prep.outputs.name }}
           SKILLS: ${{ steps.prep.outputs.skills }}   # JSON array of skills this agent may load
         run: bash "$SCRIPTS/install-opencode.sh"
@@ -212,7 +240,7 @@ jobs:
         id: run
         env:
           SCRIPTS: ${{ runner.temp }}/agents-scripts
-          XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
+          XAI_API_KEY: ${{ secrets.XAI_API_KEY || secrets.XAI_KEY || secrets.GROK_API_KEY || secrets.XAI_TOKEN }}
           # SELF_TOKEN = the RUNNING agent's OWN token (TOKEN_PM for @pm, TOKEN_OPS for @ops, …).
           # Only this agent's token is placed in its process env, so no agent can act as another.
           # Powers the gitea-api / gitea-admin skills — each agent calls Gitea as itself. Every
@@ -221,6 +249,7 @@ jobs:
           SELF_TOKEN: ${{ steps.prep.outputs.name == 'pm' && secrets.TOKEN_PM || steps.prep.outputs.name == 'junior' && secrets.TOKEN_JUNIOR || steps.prep.outputs.name == 'senior' && secrets.TOKEN_SENIOR || steps.prep.outputs.name == 'lead' && secrets.TOKEN_LEAD || steps.prep.outputs.name == 'qa' && secrets.TOKEN_QA || steps.prep.outputs.name == 'ops' && secrets.TOKEN_OPS || steps.prep.outputs.name == 'intern' && secrets.TOKEN_INTERN || '' }}
           NAME: ${{ steps.prep.outputs.name }}
           MODEL: ${{ steps.prep.outputs.model }}
+          FALLBACK: ${{ steps.prep.outputs.fallback }}
           VISION: ${{ steps.prep.outputs.vision }}
           MODE: ${{ steps.prep.outputs.mode }}
           WORKMODE: ${{ steps.prep.outputs.workmode }}   # build | discuss (devs consulted in-thread)

.gitea/workflows/ai-agent.yml

@@ -12,4 +12,17 @@ on:
 jobs:
   agent:
     uses: gitea/agents/.gitea/workflows/agent.yml@main
-    secrets: inherit
+    # Explicit secret map (plus inherit) so XAI_API_KEY / OLLAMA_* always reach the reusable
+    # workflow. secrets: inherit alone has left XAI_API_KEY empty on some Gitea cross-owner calls.
+    secrets:
+      GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+      OLLAMA_URL: ${{ secrets.OLLAMA_URL }}
+      OLLAMA_CLOUD_API_KEY: ${{ secrets.OLLAMA_CLOUD_API_KEY }}
+      XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
+      TOKEN_PM: ${{ secrets.TOKEN_PM }}
+      TOKEN_SENIOR: ${{ secrets.TOKEN_SENIOR }}
+      TOKEN_JUNIOR: ${{ secrets.TOKEN_JUNIOR }}
+      TOKEN_LEAD: ${{ secrets.TOKEN_LEAD }}
+      TOKEN_QA: ${{ secrets.TOKEN_QA }}
+      TOKEN_OPS: ${{ secrets.TOKEN_OPS }}
+      TOKEN_INTERN: ${{ secrets.TOKEN_INTERN }}

.gitea/workflows/scripts/agents.json

@@ -1,28 +1,31 @@
 {
   "pm": {
     "model": "ollama-cloud/minimax-m3:cloud",
+    "fallback": "xai-oc/grok-4.5",
     "vision": true,
     "mode": "comment",
     "skills": [
       "gitea-api"
     ],
-    "desc": "Product manager & orchestrator — plans and picks the dev, hands each finished PR to @qa for review, and reports back to the issue creator (in autopilot it merges approved PRs itself). Works from the issue thread only — comments only, never edits files, never reads the PR diff."
+    "desc": "Product manager & orchestrator — plans and picks the dev, hands each finished PR to @qa for review, and reports back to the issue creator (in autopilot it merges approved PRs itself). Works from the issue thread only — comments only, never edits files, never reads the PR diff. Falls back to xAI grok-4.5 if Ollama Cloud is unavailable/quota-exhausted."
   },
   "junior": {
     "model": "ollama-cloud/kimi-k2.7-code:cloud",
+    "fallback": "xai-oc/grok-4.3",
     "vision": false,
     "mode": "pr",
     "skills": [],
-    "desc": "Junior dev — small, low-risk changes (mostly YAML/compose/config). Text-only, cannot read images. Defers complex or image tasks to @senior or @lead."
+    "desc": "Junior dev — small, low-risk changes (mostly YAML/compose/config). Text-only, cannot read images. Defers complex or image tasks to @senior or @lead. Falls back to xAI grok-4.3 if Ollama Cloud fails."
   },
   "senior": {
     "model": "ollama-cloud/glm-5.2:cloud",
+    "fallback": "xai-oc/grok-4.5",
     "vision": false,
     "mode": "pr",
     "skills": [
       "gitea-api"
     ],
-    "desc": "Senior dev — complex, multi-file implementation (GLM-5.2 via Ollama Cloud, text-only)."
+    "desc": "Senior dev — complex, multi-file implementation (GLM-5.2 via Ollama Cloud, text-only). Falls back to xAI grok-4.5 if Ollama Cloud fails."
   },
   "lead": {
     "model": "xai-oc/grok-4.5",
@@ -35,12 +38,13 @@
   },
   "qa": {
     "model": "ollama-cloud/minimax-m3:cloud",
+    "fallback": "xai-oc/grok-4.5",
     "vision": true,
     "mode": "comment",
     "skills": [
       "gitea-api"
     ],
-    "desc": "QA / reviewer — reviews PRs: reads the diff, drives a headless browser (Playwright) to verify behavior, posts specific recommendations on the PR and the pass/fail verdict on the issue. Never edits code, never merges."
+    "desc": "QA / reviewer — reviews PRs: reads the diff, drives a headless browser (Playwright) to verify behavior, posts specific recommendations on the PR and the pass/fail verdict on the issue. Never edits code, never merges. Falls back to xAI grok-4.5 if Ollama Cloud fails."
   },
   "ops": {
     "model": "xai-oc/grok-4.5",
@@ -53,9 +57,10 @@
   },
   "intern": {
     "model": "ollama/ornith:35b",
+    "fallback": "xai-oc/grok-4.3",
     "vision": false,
     "mode": "pr",
     "skills": [],
-    "desc": "Intern — very basic tasks only, routed to the local Ollama model (ornith:35b). Text-only, cannot read images. Escalates anything non-trivial to @junior, @senior or @lead."
+    "desc": "Intern — very basic tasks only, routed to the local Ollama model (ornith:35b). Text-only, cannot read images. Escalates anything non-trivial to @junior, @senior or @lead. Falls back to xAI grok-4.3 if local Ollama is down."
   }
 }

.gitea/workflows/scripts/install-opencode.sh

@@ -5,6 +5,14 @@
 #   NAME SKILLS GITHUB_PATH HOME
 set -eu
 
+# Non-secret diagnostics — prove which provider keys reached the runner (length only).
+echo "provider key lengths: OLLAMA_URL=${#OLLAMA_URL} OLLAMA_CLOUD_API_KEY=${#OLLAMA_CLOUD_API_KEY} XAI_API_KEY=${#XAI_API_KEY}"
+if [ -z "${XAI_API_KEY:-}" ]; then
+  echo "WARNING: XAI_API_KEY is empty in this job. xai-oc fallback will fail."
+  echo "Fix: set Actions secret XAI_API_KEY on the CALLER repo (e.g. ffaerber/homelab),"
+  echo "not only on gitea/agents. Name must be exact: XAI_API_KEY"
+fi
+
 # PIN the opencode version: an unpinned `latest` means a breaking release (CLI flags, or the
 # --format json event schema that build-activity-log.sh parses) breaks every agent in every repo
 # at once. Bump deliberately by changing this default (or set OPENCODE_VERSION in the step env).
@@ -54,9 +62,11 @@ PERM=$(jq -nc --argjson s "$SKILLS" '
 # 1.17.13 — renaming the key to `xai-oc` avoids the collision entirely. See issue #118.
 # See issue #31.
 AGENTS_JSON="${SCRIPTS:-$(dirname -- "$0")}/agents.json"
-CLOUD_MODELS=$(jq -r '[.[] | .model | select(startswith("ollama-cloud/")) | sub("^ollama-cloud/";"")] | map({(.):{}}) | add // {}' "$AGENTS_JSON")
-LOCAL_MODELS=$(jq -r '[.[] | .model | select(startswith("ollama/")) | sub("^ollama/";"")] | map({(.):{}}) | add // {"ornith:35b":{}}' "$AGENTS_JSON")
-XAI_MODELS=$(jq -r '[.[] | .model | select(startswith("xai-oc/")) | sub("^xai-oc/";"")] | map({(.):{}}) | add // {}' "$AGENTS_JSON")
+# Include primary `.model` AND optional `.fallback` so failover models are always registered
+# in opencode provider maps (issue: Ollama Cloud quota → xAI).
+CLOUD_MODELS=$(jq -r '[.[] | (.model, .fallback) | select(type=="string" and startswith("ollama-cloud/")) | sub("^ollama-cloud/";"")] | unique | map({(.):{}}) | add // {}' "$AGENTS_JSON")
+LOCAL_MODELS=$(jq -r '[.[] | (.model, .fallback) | select(type=="string" and startswith("ollama/")) | sub("^ollama/";"")] | unique | map({(.):{}}) | add // {"ornith:35b":{}}' "$AGENTS_JSON")
+XAI_MODELS=$(jq -r '[.[] | (.model, .fallback) | select(type=="string" and startswith("xai-oc/")) | sub("^xai-oc/";"")] | unique | map({(.):{}}) | add // {}' "$AGENTS_JSON")
 jq -n --argjson mcp "$MCP" --argjson perm "$PERM" --argjson cloud "$CLOUD_MODELS" --argjson local "$LOCAL_MODELS" --argjson xai "$XAI_MODELS" --arg url "$OLLAMA_URL" --arg ckey "$OLLAMA_CLOUD_API_KEY" --arg xkey "$XAI_API_KEY" '{
   provider: {
     ollama: {npm:"@ai-sdk/openai-compatible", options:{baseURL:($url+"/v1")}, models:$local},

.gitea/workflows/scripts/route.sh

@@ -43,12 +43,14 @@ if [ -z "$name" ]; then
     # Not an agent task (e.g. the gate's contains() matched "@internal"). Skip GRACEFULLY: emit
     # mode=skip so every later step no-ops — a red run for a non-agent comment is just noise.
     echo "no known agent mentioned (word-boundary) — skipping run"
-    { echo "name=none"; echo "model=none"; echo "vision=false"; echo "mode=skip"; echo "skills=[]";
+    { echo "name=none"; echo "model=none"; echo "fallback="; echo "vision=false"; echo "mode=skip"; echo "skills=[]";
       echo "branch=main"; echo "new=false"; echo "autopilot=false"; echo "issnum=$NUM"; } >> "$GITHUB_OUTPUT"
     exit 0
   fi
 fi
 model=$(jq -r --arg a "$name" '.[$a].model'  /tmp/agents.json)
+# Optional provider failover when primary is quota/network-dead (e.g. Ollama Cloud → xAI).
+fallback=$(jq -r --arg a "$name" '.[$a].fallback // empty' /tmp/agents.json)
 vision=$(jq -r --arg a "$name" '.[$a].vision' /tmp/agents.json)
 mode=$(jq -r --arg a "$name" '.[$a].mode'   /tmp/agents.json)
 # Compact JSON array of the skills this agent may load (scopes permission.skill in install-opencode.sh).
@@ -70,8 +72,8 @@ if [ "$mode" = "pr" ] && [ -z "$IS_PR" ]; then
     workmode=discuss
   fi
 fi
-echo "Routing to @$name (model=$model vision=$vision mode=$mode workmode=$workmode skills=$skills)"
-{ echo "name=$name"; echo "model=$model"; echo "vision=$vision"; echo "mode=$mode"; echo "workmode=$workmode"; echo "skills=$skills"; } >> "$GITHUB_OUTPUT"
+echo "Routing to @$name (model=$model fallback=${fallback:-none} vision=$vision mode=$mode workmode=$workmode skills=$skills)"
+{ echo "name=$name"; echo "model=$model"; echo "fallback=$fallback"; echo "vision=$vision"; echo "mode=$mode"; echo "workmode=$workmode"; echo "skills=$skills"; } >> "$GITHUB_OUTPUT"
 
 # Act as the agent's own Gitea user when its token is set; else the built-in bot.
 case "$name" in

.gitea/workflows/scripts/run-agent.sh

@@ -186,6 +186,16 @@ rc=1
 # instance-wide. 20 min is far above any legitimate attempt. timeout SIGTERMs, then SIGKILLs 30s
 # later. rc=124 (timed out) is NOT retried — a hung backend stays hung; fail fast, free the runner.
 AGENT_TIMEOUT="${AGENT_TIMEOUT:-1200}"
+# Provider failover: when primary model dies on quota/auth/provider errors, switch once to
+# FALLBACK (from agents.json) and continue the retry loop. Transient rate-limits still back off
+# on the current model first.
+FALLBACK_MODEL="${FALLBACK:-}"
+fallback_used=0
+is_failover_error() {
+  # Ollama Cloud exhausted / provider hard-fail — switch to fallback rather than thrash.
+  grep -qiE 'overloaded|429|529|rate.?limit|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN|quota|credit|balance|usage.?limit|limit.?exceed|402|403|401|insufficient|out of credits|payment.?required|model_not_found|not found|Unavailable|capacity|ENOTFOUND|ECONNREFUSED' \
+    /tmp/events.jsonl /tmp/agent_err.log 2>/dev/null
+}
 for attempt in 1 2 3; do
   echo "opencode attempt $attempt/3 for @$NAME ($MODEL, timeout ${AGENT_TIMEOUT}s)"
   rc=0
@@ -196,6 +206,14 @@ for attempt in 1 2 3; do
   echo "--- stderr (trace) ---"; cat /tmp/agent_err.log
   [ $rc -eq 0 ] && break
   if [ $rc -eq 124 ]; then echo "attempt timed out after ${AGENT_TIMEOUT}s — backend hung, not retrying"; break; fi
+  if [ $fallback_used -eq 0 ] && [ -n "$FALLBACK_MODEL" ] && [ "$FALLBACK_MODEL" != "$MODEL" ] && is_failover_error; then
+    echo "primary model failed — failing over to fallback: $FALLBACK_MODEL"
+    MODEL="$FALLBACK_MODEL"
+    fallback_used=1
+    # short pause then use next attempt slot on the fallback provider
+    sleep 2
+    continue
+  fi
   if grep -qiE 'overloaded|429|529|rate.?limit|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN' /tmp/events.jsonl /tmp/agent_err.log; then
     echo "transient error — backing off $((attempt*20))s"; sleep $((attempt * 20)); continue
   fi