diff --git a/hostinger/.env.example b/hostinger/.env.example index e793d0b..8fb07fe 100644 --- a/hostinger/.env.example +++ b/hostinger/.env.example @@ -18,9 +18,16 @@ ESCROW_ADDRESS=0x34Db8E014E71928f17E23eC1272B602582222c9c XBZZ_ADDRESS=0xdBF3Ea6F5beE45c02255B2c26a16F300502F68da # --- Bee node --- -# Postage batch is bought once on first deploy and stored in bee-data volume. +# Pin the batch explicitly. It used to be bought on first deploy and left to +# the gateway to manage, which is how the outage happened: the batch fell +# under minTtlDays, the gateway tried to renew it, the renewal cost more xBZZ +# than the wallet held, and the throw took down every route that boots the +# gateway context. Renewal now belongs to stamp-monitor, which has spend caps +# and alerts; the gateway just uses the batch it is given. +# +# Must be set. An empty value here is what re-enables the path above. BEE_POSTAGE_BATCH_ID= -BEE_STAMP_MANAGE=true +BEE_STAMP_MANAGE=false BEE_STAMP_LABEL=hosted-gateway BEE_STAMP_TTL_DAYS=30 @@ -38,3 +45,9 @@ LOG_LEVEL=info # --- Phase 2 (Stripe) — leave blank in v1 --- STRIPE_SECRET_KEY= STRIPE_WEBHOOK_SECRET= + +# --- stamp-monitor --- +# Bearer token for the monitor's dashboard and admin API. That UI can spend +# xBZZ (manual top-up), so treat it as a spending credential, not a viewer +# login. Generate with `openssl rand -hex 32`; never commit the real value. +STAMP_MONITOR_ADMIN_TOKEN=change-me diff --git a/hostinger/prod-stack.yaml b/hostinger/prod-stack.yaml index c5d2845..87f675f 100644 --- a/hostinger/prod-stack.yaml +++ b/hostinger/prod-stack.yaml @@ -8,6 +8,8 @@ networks: volumes: + stamp-monitor-data: + driver: local postgres-data: driver: local redis-data: @@ -20,6 +22,57 @@ volumes: services: + # Watches this node's postage batches and renews them before they lapse. + # + # Added after the gateway went down because its batch fell under + # minTtlDays and the top-up it then attempted (64.64 xBZZ on a depth-24 + # batch) exceeded the Bee wallet — ensureManagedStamp threw, and with it + # every route that boots the gateway context. Nothing was watching. + # + # Runs HERE rather than reading the chain from the homelab because bucket + # utilisation is not on-chain: stamps are off-chain signatures, so only a + # Bee node that saw them knows how full a batch is. Sitting next to prod_bee + # is what makes fullness, the bucket map and batch_full alerts possible. + stamp-monitor: + image: ghcr.io/ffaerber/swarm-stamp-monitor@sha256:62cf97f72090879f8d470fc9f2a09ab60d1d3a29dd658dfe2cd3a313c2b23540 + networks: [internal] + volumes: + - stamp-monitor-data:/data + env_file: + - .env + environment: + BEE_URL: http://bee:1633 + DB_PATH: /data/monitor.sqlite + ADMIN_TOKEN: '${STAMP_MONITOR_ADMIN_TOKEN}' + # Armed: the whole point is that nothing renewed the batch last time. + AUTO_TOPUP_ENABLED: "true" + DRY_RUN: "false" + TOPUP_WHEN_TTL_BELOW_DAYS: "14" + TOPUP_TARGET_TTL_DAYS: "60" + # 8 xBZZ is chosen, not arbitrary. A depth-20 top-up from 14d to 60d + # costs 6.19 and passes; the abandoned depth-25 batch would cost 129.28 + # and is refused by this cap alone. The cap is the guard that stops the + # monitor reviving a batch we deliberately let expire. + MAX_TOPUP_BZZ_PER_BATCH: "8" + MAX_TOPUP_BZZ_PER_DAY: "15" + MIN_WALLET_BZZ: "10" + MIN_WALLET_XDAI: "0.2" + # OFF. The gateway re-stamps an identical chunk, so one bucket always + # trends full while the batch holds ~2 MB. Diluting that halves the TTL + # and doubles the monthly cost for no capacity gained — it is what took + # this batch from depth 24 to 25 and 129 xBZZ/month. On a MUTABLE batch + # a full bucket simply recycles its own duplicates, which is harmless. + DILUTE_ENABLED: "false" + PRICE_ENABLED: "true" + deploy: + replicas: 1 + restart_policy: + condition: any + resources: + limits: + cpus: "0.30" + memory: 256M + frontend: image: ghcr.io/token-for-token/hosted-gateway-frontend:main networks: @@ -203,10 +256,20 @@ services: bee: - # Pinned to 2.8.0 — matches the homelab Bee. Both speak the v15 + # Pinned to 2.8.1 — matches the homelab Bee. Both speak the v15 # handshake. Homelab uses a fresh data dir + --target-neighborhood # to keep the same swarmOverlay as its on-chain provider registration. - image: ethersphere/bee:2.8.0 + # + # Moved off 2.8.0 because that build omits utilizationRatio from /stamps. + # The field is not optional in practice: without it a batch's fullness is + # unknown, and "unknown" read as 0% — a batch sitting at 215 of 256 + # reported as empty, which is the one direction that fails silently. + # 2.8.1 sends it. (stamp-monitor now derives it either way, but the node + # should not be the thing lying.) + # + # Digest-pinned so a re-deploy cannot silently pick up a re-tagged image; + # this is a node holding a chequebook and a stake. + image: ethersphere/bee:2.8.1@sha256:1a7eeab7fc2725132716abd1aa50ceafb83ece59f171e4ea11deefc773e73a57 # NOTE: BEE_BLOCKCHAIN_RPC_ENDPOINT env var is silently ignored — pass # the flag explicitly. Without it Bee logs "chain backend disabled". command: diff --git a/hostinger/readme.md b/hostinger/readme.md index 4254fef..616f2a2 100644 --- a/hostinger/readme.md +++ b/hostinger/readme.md @@ -11,7 +11,7 @@ Layout mirrors [cipherdolls/devops/hostinger/](https://github.com/cipherdolls/de | Stack | File | What it runs | | --- | --- | --- | | `system` | `system-stack.yaml` | Traefik (TLS + ingress), Grafana, Loki, Prometheus, node-exporter, cadvisor. Owns the `public` + `internal` overlay networks. | -| `prod` | `prod-stack.yaml` | hosted-gateway api + worker + one-shot prisma migrate, plus postgres / redis / bee / postgres-exporter. | +| `prod` | `prod-stack.yaml` | hosted-gateway api + worker + one-shot prisma migrate, plus postgres / redis / bee / postgres-exporter / stamp-monitor. | | `dev` | `dev-stack.yaml` | Staging on `dev.t4t-gateway.com` — the `:develop` image tag against its own postgres / redis / bee volumes. | Supporting configs: @@ -90,6 +90,38 @@ lives in `/opt/t4t-gateway/.env` (chmod 600). Future work: migrate to a proper Docker secret backed by the host's secret manager — never an env var in the stack file. +## Postage batches (stamp-monitor) + +The gateway's uploads are paid for by a postage batch that expires. When it +lapsed, the gateway returned 500 on every route — see the comment above the +`stamp-monitor` service for the full chain. `stamp-monitor` now owns renewal: +it polls `prod_bee` every 5 min, tops a batch up when it drops under 14 days, +and refuses any single action over 8 xBZZ. + +It runs beside Bee rather than reading the chain because **bucket fullness is +not on-chain** — postage stamps are off-chain signatures, so only a node that +saw them knows how full a batch is. A batch can be fully funded and still +refuse writes. + +No Traefik route: it is on `internal` only, and its UI can spend xBZZ. Reach it +over an SSH tunnel instead of publishing it: + +```bash +ssh -L 8088:127.0.0.1:8088 root@ +docker run --rm --network prod_internal -p 127.0.0.1:8088:8088 \ + alpine/socat tcp-listen:8088,fork,reuseaddr tcp-connect:stamp-monitor:3000 +# then http://127.0.0.1:8088 with STAMP_MONITOR_ADMIN_TOKEN +``` + +Quick check without the UI: + +```bash +docker service logs --tail 50 prod_stamp-monitor +``` + +`WEBHOOK_URL` is unset, so alerts (`wallet_low`, `batch_full`, `topup_blocked`) +only reach the logs. Set it to make them reach a person. + ## Backups Postgres volume: