From 6334ebe8c6d4bd294719d92943ae8d8c9f5009dd Mon Sep 17 00:00:00 2001 From: Felix Faerber Date: Mon, 6 Jul 2026 09:22:00 +0300 Subject: [PATCH] agents: report tokens + $ cost on every agent comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each run now appends a report to the agent's reply: the tool calls it made plus input/output token totals and the dollar cost. opencode's --format json emits per-step `tokens` and `cost` (USD, priced from the model) on step_finish events; build-activity-log.sh sums them across the run. - build-activity-log.sh: compute for EVERY agent (not just devs — @pm/@qa also call tools and cost money); output a collapsed
report (summary line shows "N tool calls · in X · out Y · $Z"; body lists the tools + a token/cost breakdown). Zero-tool runs get a one-line "$Z · in X · out Y" footer. - publish.sh: build $activity once (near the top) and append it to every agent's reply — @pm plan/finalize, @qa verdict/recommendations, and dev PR comments. - agent.yml: rename the step accordingly. Models without pricing (self-hosted ollama) report cost $0.0000. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitea/workflows/agent.yml | 2 +- .../workflows/scripts/build-activity-log.sh | 55 ++++++++++++++----- .gitea/workflows/scripts/publish.sh | 25 ++++----- 3 files changed, 52 insertions(+), 30 deletions(-) diff --git a/.gitea/workflows/agent.yml b/.gitea/workflows/agent.yml index 9aa81f3..743d2b5 100644 --- a/.gitea/workflows/agent.yml +++ b/.gitea/workflows/agent.yml @@ -185,7 +185,7 @@ jobs: FILES: ${{ steps.imgs.outputs.files }} # opencode -f image flags (vision agents only) run: bash "$SCRIPTS/run-agent.sh" - - name: Build activity log (tool calls + reasoning) from the event stream + - name: Build run report (tool calls + input/output tokens + $ cost) from the event stream id: log env: SCRIPTS: ${{ runner.temp }}/agents-scripts diff --git a/.gitea/workflows/scripts/build-activity-log.sh b/.gitea/workflows/scripts/build-activity-log.sh index 5349690..15a6866 100755 --- a/.gitea/workflows/scripts/build-activity-log.sh +++ b/.gitea/workflows/scripts/build-activity-log.sh @@ -1,24 +1,51 @@ #!/usr/bin/env bash -# Build the activity log — the list of TOOL CALLS the agent made — into /tmp/activity_log.md. -# Only dev agents (mode=pr) get an activity-log comment — comment-only roles (pm/qa) do no tool calls. -# NOTE: we deliberately DO NOT include the agent's prose text parts. That final "here's what I did" -# text is just a restatement of the PR description (already published as the PR body), not a tool -# call — so it was noise in a section titled "tool calls". The log is the record of ACTIONS taken. +# Build the RUN REPORT appended to the agent's reply comment: the TOOL CALLS the agent made plus a +# usage line (input / output tokens + $ cost). Written to /tmp/activity_log.md; the Publish step +# appends it to the agent's reply. Applies to EVERY agent — @pm/@qa also call tools and cost money. # -# Required env (provided by the workflow step): MODE +# opencode --format json emits one JSON event per line. `step_finish` events carry, per LLM step, +# .part.tokens {input, output, reasoning, cache:{read, write}} and .part.cost (USD, already computed +# by opencode from the model's pricing). We sum them across all steps of the run. Models without +# pricing (e.g. self-hosted ollama) report cost 0 — shown as $0.0000. +# +# Required env (provided by the workflow step): (none needed; reads /tmp/events.jsonl) set -u +E=/tmp/events.jsonl +: > /tmp/activity_log.md +[ -s "$E" ] || { echo "no events — empty report"; exit 0; } -if [ "$MODE" != "pr" ]; then - echo "skipping activity log for comment-mode agent"; : > /tmp/activity_log.md; exit 0 -fi +# Tool calls = the ACTIONS taken (not the agent's prose text parts). jq -r ' def trunc(n): if length > n then (.[0:n] + "…") else . end; select(.type=="tool_use") | (.part.tool // "?") as $t | ((.part.state.title // (.part.state.input | tojson | trunc(160)) // "")) as $title | "🔧 **" + $t + "**: `" + ($title | trunc(240)) + "`" -' /tmp/events.jsonl > /tmp/activity_log.md 2>/dev/null || true -n=$(wc -l < /tmp/activity_log.md 2>/dev/null || echo 0) -echo "activity log: $n tool calls" -[ "$n" -eq 0 ] && : > /tmp/activity_log.md -head -3 /tmp/activity_log.md +' "$E" > /tmp/tools.md 2>/dev/null || true +n=$(wc -l < /tmp/tools.md 2>/dev/null || echo 0); n=${n:-0} + +# Usage: sum per-step tokens + cost across every step_finish event (tab-separated for `read`). +read -r COST INP OUT CR CW RE < <(jq -rs ' + [ .[] | select(.type=="step_finish") | .part ] as $s + | [ ([$s[].cost // 0]|add // 0), + ([$s[].tokens.input // 0]|add // 0), + ([$s[].tokens.output // 0]|add // 0), + ([$s[].tokens.cache.read // 0]|add // 0), + ([$s[].tokens.cache.write // 0]|add // 0), + ([$s[].tokens.reasoning // 0]|add // 0) ] + | @tsv' "$E" 2>/dev/null) +COST=${COST:-0}; INP=${INP:-0}; OUT=${OUT:-0}; CR=${CR:-0}; CW=${CW:-0}; RE=${RE:-0} +IN_TOTAL=$(( INP + CR + CW )) # total input context processed +COSTF=$(awk -v c="$COST" 'BEGIN{printf "$%.4f", c+0}') +echo "usage: in=$IN_TOTAL out=$OUT cost=$COSTF (fresh=$INP cache_r=$CR cache_w=$CW reasoning=$RE); tools=$n" + +{ + if [ "$n" -gt 0 ]; then + printf '\n\n
\n🔧 %s tool calls · in %s · out %s · %s\n\n' "$n" "$IN_TOTAL" "$OUT" "$COSTF" + cat /tmp/tools.md + printf '\n\ntokens — input %s (fresh %s · cache %sw / %sr) · output %s · reasoning %s · **cost %s**\n
' \ + "$IN_TOTAL" "$INP" "$CW" "$CR" "$OUT" "$RE" "$COSTF" + else + printf '\n\n💰 **%s** · in %s · out %s tokens (cache %sw / %sr)' "$COSTF" "$IN_TOTAL" "$OUT" "$CW" "$CR" + fi +} > /tmp/activity_log.md diff --git a/.gitea/workflows/scripts/publish.sh b/.gitea/workflows/scripts/publish.sh index 08320ce..05cfd31 100755 --- a/.gitea/workflows/scripts/publish.sh +++ b/.gitea/workflows/scripts/publish.sh @@ -91,6 +91,9 @@ reply=$(printf '%s' "$reply" | awk -v me="@$NAME" ' # Prefer the agent's clean delimited PR description; fall back to the whole reply. prdesc=$(awk '/BEGIN_PR_DESCRIPTION/{f=1;next} /END_PR_DESCRIPTION/{f=0} f' /tmp/agent_out.md) [ -z "$prdesc" ] && prdesc="$reply" +# Run report (tool calls + input/output tokens + $ cost) built by build-activity-log.sh. Appended to +# every agent's reply comment so each run reports what it did and what it cost. +activity="$(cat /tmp/activity_log.md 2>/dev/null || true)" # comment-only roles (pm/qa): never change files if [ "$MODE" != "pr" ]; then @@ -103,13 +106,13 @@ if [ "$MODE" != "pr" ]; then if [ "$NAME" = "qa" ]; then PRN=$(resolve_pr) if grep -qiE '^[[:space:]]*APPROVE[[:space:]]*$' /tmp/agent_out.md; then - post_to "$ISSN" "$(printf '✅ Reviewed PR #%s — looks good.\n\n%s' "${PRN:-?}" "$reply")" + post_to "$ISSN" "$(printf '✅ Reviewed PR #%s — looks good.\n\n%s%s' "${PRN:-?}" "$reply" "$activity")" trig "$ISSN" "@pm — @qa approved PR #${PRN:-?} (issue #$ISSN). Over to you." elif grep -qiE '^[[:space:]]*BOUNCE:[[:space:]]*@(junior|senior|lead)' /tmp/agent_out.md; then dev=$(grep -oiE 'BOUNCE:[[:space:]]*@(junior|senior|lead)' /tmp/agent_out.md | head -1 | grep -oiE '(junior|senior|lead)' | tr '[:upper:]' '[:lower:]') [ -z "$dev" ] && [ -n "$PRN" ] && dev=$(curl -sS "${hdr[@]}" "$API/pulls/$PRN" | jq -r '.user.login // "junior"') dest="${PRN:-$NUM}" - post_to "$dest" "$reply" # recommendations, on the PR + post_to "$dest" "$reply$activity" # recommendations, on the PR # Bounce budget: count prior "fix attempt" markers on the PR thread; stop after 3. prior=$(curl -sS "${hdr[@]}" "$API/issues/$dest/comments?limit=100" | jq -r 'if type=="array" then [.[]|select(.body|test("fix attempt"))]|length else 0 end' 2>/dev/null); prior=${prior:-0} if [ "$prior" -ge 3 ]; then @@ -121,9 +124,9 @@ if [ "$MODE" != "pr" ]; then fi elif grep -qiE '^[[:space:]]*HALT([_ ]AUTOPILOT)?[[:space:]]*$' /tmp/agent_out.md; then [ "$AUTOPILOT" = "true" ] && del_autopilot_label "$ISSN" - post_to "$ISSN" "$(printf '🛑 This needs a human decision (not a dev fix) — @ffaerber please take a look.\n\n%s' "$reply")" + post_to "$ISSN" "$(printf '🛑 This needs a human decision (not a dev fix) — @ffaerber please take a look.\n\n%s%s' "$reply" "$activity")" else - post_to "${PRN:-$NUM}" "$reply" # no verdict yet (a question) — post where qa works + post_to "${PRN:-$NUM}" "$reply$activity" # no verdict yet (a question) — post where qa works fi exit 0 fi @@ -173,7 +176,7 @@ if [ "$MODE" != "pr" ]; then done < /tmp/subtasks.txt subtext=$(printf '\n\n---\nCreated sub-issues%s (mention an agent on each when ready):%b' "${ms:+ under milestone **$ms**}" "$links") fi - post "$(printf '%s%s' "$msg" "$subtext")" + post "$(printf '%s%s%s' "$msg" "$subtext" "$activity")" # --- @pm autopilot merge: @pm is the ONLY agent that merges, and ONLY under the autopilot label --- # (@qa never merges — it approves and hands back here.) Merge with the PAT (TTOK), not the built-in @@ -246,16 +249,8 @@ git fetch -q origin 2>/dev/null || true prbody=$(printf '%s\n\n---\nResolves #%s' "$prdesc" "$NUM") owner=${GITHUB_REPOSITORY%%/*} -# Post the agent's activity trail (tool calls + reasoning) inline in the same comment so -# each run produces exactly ONE comment (issue #38). Computed once here so every dev-agent -# exit path (no-changes, PR-open-failed, normal) appends it to the single reply comment. -activity="" -if [ -s /tmp/activity_log.md ]; then - entries=$(wc -l < /tmp/activity_log.md 2>/dev/null || echo 0) - log=$(cat /tmp/activity_log.md) - activity=$(printf '\n\n
\n🔧 activity — %s tool calls\n\n%s\n\n
' "$entries" "$log") -fi - +# $activity (the run report: tool calls + tokens + $ cost) was built once near the top, so every +# dev-agent exit path (no-changes, PR-open-failed, normal) appends it to the single reply comment. # One PR per run: publish ONLY this run's own branch ($BRANCH), never sibling # ai/issue-N-* branches. This removes the multi-PR ambiguity that left the # activity log stranded on the triggering issue instead of the PR thread. -- 2.54.0