agents: report tokens + $ cost on every agent comment #75
@@ -185,7 +185,7 @@ jobs:
|
||||
FILES: ${{ steps.imgs.outputs.files }} # opencode -f image flags (vision agents only)
|
||||
run: bash "$SCRIPTS/run-agent.sh"
|
||||
|
||||
- name: Build activity log (tool calls + reasoning) from the event stream
|
||||
- name: Build run report (tool calls + input/output tokens + $ cost) from the event stream
|
||||
id: log
|
||||
env:
|
||||
SCRIPTS: ${{ runner.temp }}/agents-scripts
|
||||
|
||||
@@ -1,24 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the activity log — the list of TOOL CALLS the agent made — into /tmp/activity_log.md.
|
||||
# Only dev agents (mode=pr) get an activity-log comment — comment-only roles (pm/qa) do no tool calls.
|
||||
# NOTE: we deliberately DO NOT include the agent's prose text parts. That final "here's what I did"
|
||||
# text is just a restatement of the PR description (already published as the PR body), not a tool
|
||||
# call — so it was noise in a section titled "tool calls". The log is the record of ACTIONS taken.
|
||||
# Build the RUN REPORT appended to the agent's reply comment: the TOOL CALLS the agent made plus a
|
||||
# usage line (input / output tokens + $ cost). Written to /tmp/activity_log.md; the Publish step
|
||||
# appends it to the agent's reply. Applies to EVERY agent — @pm/@qa also call tools and cost money.
|
||||
#
|
||||
# Required env (provided by the workflow step): MODE
|
||||
# opencode --format json emits one JSON event per line. `step_finish` events carry, per LLM step,
|
||||
# .part.tokens {input, output, reasoning, cache:{read, write}} and .part.cost (USD, already computed
|
||||
# by opencode from the model's pricing). We sum them across all steps of the run. Models without
|
||||
# pricing (e.g. self-hosted ollama) report cost 0 — shown as $0.0000.
|
||||
#
|
||||
# Required env (provided by the workflow step): (none needed; reads /tmp/events.jsonl)
|
||||
set -u
|
||||
E=/tmp/events.jsonl
|
||||
: > /tmp/activity_log.md
|
||||
[ -s "$E" ] || { echo "no events — empty report"; exit 0; }
|
||||
|
||||
if [ "$MODE" != "pr" ]; then
|
||||
echo "skipping activity log for comment-mode agent"; : > /tmp/activity_log.md; exit 0
|
||||
fi
|
||||
# Tool calls = the ACTIONS taken (not the agent's prose text parts).
|
||||
jq -r '
|
||||
def trunc(n): if length > n then (.[0:n] + "…") else . end;
|
||||
select(.type=="tool_use") |
|
||||
(.part.tool // "?") as $t |
|
||||
((.part.state.title // (.part.state.input | tojson | trunc(160)) // "")) as $title |
|
||||
"🔧 **" + $t + "**: `" + ($title | trunc(240)) + "`"
|
||||
' /tmp/events.jsonl > /tmp/activity_log.md 2>/dev/null || true
|
||||
n=$(wc -l < /tmp/activity_log.md 2>/dev/null || echo 0)
|
||||
echo "activity log: $n tool calls"
|
||||
[ "$n" -eq 0 ] && : > /tmp/activity_log.md
|
||||
head -3 /tmp/activity_log.md
|
||||
' "$E" > /tmp/tools.md 2>/dev/null || true
|
||||
n=$(wc -l < /tmp/tools.md 2>/dev/null || echo 0); n=${n:-0}
|
||||
|
||||
# Usage: sum per-step tokens + cost across every step_finish event (tab-separated for `read`).
|
||||
read -r COST INP OUT CR CW RE < <(jq -rs '
|
||||
[ .[] | select(.type=="step_finish") | .part ] as $s
|
||||
| [ ([$s[].cost // 0]|add // 0),
|
||||
([$s[].tokens.input // 0]|add // 0),
|
||||
([$s[].tokens.output // 0]|add // 0),
|
||||
([$s[].tokens.cache.read // 0]|add // 0),
|
||||
([$s[].tokens.cache.write // 0]|add // 0),
|
||||
([$s[].tokens.reasoning // 0]|add // 0) ]
|
||||
| @tsv' "$E" 2>/dev/null)
|
||||
COST=${COST:-0}; INP=${INP:-0}; OUT=${OUT:-0}; CR=${CR:-0}; CW=${CW:-0}; RE=${RE:-0}
|
||||
IN_TOTAL=$(( INP + CR + CW )) # total input context processed
|
||||
COSTF=$(awk -v c="$COST" 'BEGIN{printf "$%.4f", c+0}')
|
||||
echo "usage: in=$IN_TOTAL out=$OUT cost=$COSTF (fresh=$INP cache_r=$CR cache_w=$CW reasoning=$RE); tools=$n"
|
||||
|
||||
{
|
||||
if [ "$n" -gt 0 ]; then
|
||||
printf '\n\n<details>\n<summary>🔧 %s tool calls · in %s · out %s · %s</summary>\n\n' "$n" "$IN_TOTAL" "$OUT" "$COSTF"
|
||||
cat /tmp/tools.md
|
||||
printf '\n\n<sub>tokens — input %s (fresh %s · cache %sw / %sr) · output %s · reasoning %s · **cost %s**</sub>\n</details>' \
|
||||
"$IN_TOTAL" "$INP" "$CW" "$CR" "$OUT" "$RE" "$COSTF"
|
||||
else
|
||||
printf '\n\n<sub>💰 **%s** · in %s · out %s tokens (cache %sw / %sr)</sub>' "$COSTF" "$IN_TOTAL" "$OUT" "$CW" "$CR"
|
||||
fi
|
||||
} > /tmp/activity_log.md
|
||||
|
||||
@@ -91,6 +91,9 @@ reply=$(printf '%s' "$reply" | awk -v me="@$NAME" '
|
||||
# Prefer the agent's clean delimited PR description; fall back to the whole reply.
|
||||
prdesc=$(awk '/BEGIN_PR_DESCRIPTION/{f=1;next} /END_PR_DESCRIPTION/{f=0} f' /tmp/agent_out.md)
|
||||
[ -z "$prdesc" ] && prdesc="$reply"
|
||||
# Run report (tool calls + input/output tokens + $ cost) built by build-activity-log.sh. Appended to
|
||||
# every agent's reply comment so each run reports what it did and what it cost.
|
||||
activity="$(cat /tmp/activity_log.md 2>/dev/null || true)"
|
||||
|
||||
# comment-only roles (pm/qa): never change files
|
||||
if [ "$MODE" != "pr" ]; then
|
||||
@@ -103,13 +106,13 @@ if [ "$MODE" != "pr" ]; then
|
||||
if [ "$NAME" = "qa" ]; then
|
||||
PRN=$(resolve_pr)
|
||||
if grep -qiE '^[[:space:]]*APPROVE[[:space:]]*$' /tmp/agent_out.md; then
|
||||
post_to "$ISSN" "$(printf '✅ Reviewed PR #%s — looks good.\n\n%s' "${PRN:-?}" "$reply")"
|
||||
post_to "$ISSN" "$(printf '✅ Reviewed PR #%s — looks good.\n\n%s%s' "${PRN:-?}" "$reply" "$activity")"
|
||||
trig "$ISSN" "@pm — @qa approved PR #${PRN:-?} (issue #$ISSN). Over to you."
|
||||
elif grep -qiE '^[[:space:]]*BOUNCE:[[:space:]]*@(junior|senior|lead)' /tmp/agent_out.md; then
|
||||
dev=$(grep -oiE 'BOUNCE:[[:space:]]*@(junior|senior|lead)' /tmp/agent_out.md | head -1 | grep -oiE '(junior|senior|lead)' | tr '[:upper:]' '[:lower:]')
|
||||
[ -z "$dev" ] && [ -n "$PRN" ] && dev=$(curl -sS "${hdr[@]}" "$API/pulls/$PRN" | jq -r '.user.login // "junior"')
|
||||
dest="${PRN:-$NUM}"
|
||||
post_to "$dest" "$reply" # recommendations, on the PR
|
||||
post_to "$dest" "$reply$activity" # recommendations, on the PR
|
||||
# Bounce budget: count prior "fix attempt" markers on the PR thread; stop after 3.
|
||||
prior=$(curl -sS "${hdr[@]}" "$API/issues/$dest/comments?limit=100" | jq -r 'if type=="array" then [.[]|select(.body|test("fix attempt"))]|length else 0 end' 2>/dev/null); prior=${prior:-0}
|
||||
if [ "$prior" -ge 3 ]; then
|
||||
@@ -121,9 +124,9 @@ if [ "$MODE" != "pr" ]; then
|
||||
fi
|
||||
elif grep -qiE '^[[:space:]]*HALT([_ ]AUTOPILOT)?[[:space:]]*$' /tmp/agent_out.md; then
|
||||
[ "$AUTOPILOT" = "true" ] && del_autopilot_label "$ISSN"
|
||||
post_to "$ISSN" "$(printf '🛑 This needs a human decision (not a dev fix) — @ffaerber please take a look.\n\n%s' "$reply")"
|
||||
post_to "$ISSN" "$(printf '🛑 This needs a human decision (not a dev fix) — @ffaerber please take a look.\n\n%s%s' "$reply" "$activity")"
|
||||
else
|
||||
post_to "${PRN:-$NUM}" "$reply" # no verdict yet (a question) — post where qa works
|
||||
post_to "${PRN:-$NUM}" "$reply$activity" # no verdict yet (a question) — post where qa works
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
@@ -173,7 +176,7 @@ if [ "$MODE" != "pr" ]; then
|
||||
done < /tmp/subtasks.txt
|
||||
subtext=$(printf '\n\n---\nCreated sub-issues%s (mention an agent on each when ready):%b' "${ms:+ under milestone **$ms**}" "$links")
|
||||
fi
|
||||
post "$(printf '%s%s' "$msg" "$subtext")"
|
||||
post "$(printf '%s%s%s' "$msg" "$subtext" "$activity")"
|
||||
|
||||
# --- @pm autopilot merge: @pm is the ONLY agent that merges, and ONLY under the autopilot label ---
|
||||
# (@qa never merges — it approves and hands back here.) Merge with the PAT (TTOK), not the built-in
|
||||
@@ -246,16 +249,8 @@ git fetch -q origin 2>/dev/null || true
|
||||
prbody=$(printf '%s\n\n---\nResolves #%s' "$prdesc" "$NUM")
|
||||
owner=${GITHUB_REPOSITORY%%/*}
|
||||
|
||||
# Post the agent's activity trail (tool calls + reasoning) inline in the same comment so
|
||||
# each run produces exactly ONE comment (issue #38). Computed once here so every dev-agent
|
||||
# exit path (no-changes, PR-open-failed, normal) appends it to the single reply comment.
|
||||
activity=""
|
||||
if [ -s /tmp/activity_log.md ]; then
|
||||
entries=$(wc -l < /tmp/activity_log.md 2>/dev/null || echo 0)
|
||||
log=$(cat /tmp/activity_log.md)
|
||||
activity=$(printf '\n\n<details>\n<summary>🔧 activity — %s tool calls</summary>\n\n%s\n\n</details>' "$entries" "$log")
|
||||
fi
|
||||
|
||||
# $activity (the run report: tool calls + tokens + $ cost) was built once near the top, so every
|
||||
# dev-agent exit path (no-changes, PR-open-failed, normal) appends it to the single reply comment.
|
||||
# One PR per run: publish ONLY this run's own branch ($BRANCH), never sibling
|
||||
# ai/issue-N-* branches. This removes the multi-PR ambiguity that left the
|
||||
# activity log stranded on the triggering issue instead of the PR thread.
|
||||
|
||||
Reference in New Issue
Block a user