From 032a22b6eedfe70f9344ee71769f00a95eae7d5e Mon Sep 17 00:00:00 2001 From: Felix Faerber Date: Wed, 27 May 2026 10:31:30 +0300 Subject: [PATCH] init devops repo Mirrors cipherdolls/devops layout. Hostinger Swarm stacks for the t4t hosted-gateway and local docker-compose for development + e2e tests. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/deploy.yml | 31 +++++ .gitignore | 3 + hostinger/.env.example | 41 ++++++ hostinger/dev-stack.yaml | 145 +++++++++++++++++++++ hostinger/loki.yaml | 37 ++++++ hostinger/prod-stack.yaml | 201 ++++++++++++++++++++++++++++ hostinger/prometheus.yml | 48 +++++++ hostinger/readme.md | 128 ++++++++++++++++++ hostinger/system-stack.yaml | 232 +++++++++++++++++++++++++++++++++ local/.env.example | 48 +++++++ local/Makefile | 159 ++++++++++++++++++++++ local/docker-compose-test.yaml | 166 +++++++++++++++++++++++ local/docker-compose.yaml | 108 +++++++++++++++ local/mock-ollama/Dockerfile | 8 ++ local/mock-ollama/server.ts | 47 +++++++ readme.md | 42 ++++++ 16 files changed, 1444 insertions(+) create mode 100644 .github/workflows/deploy.yml create mode 100644 .gitignore create mode 100644 hostinger/.env.example create mode 100644 hostinger/dev-stack.yaml create mode 100644 hostinger/loki.yaml create mode 100644 hostinger/prod-stack.yaml create mode 100644 hostinger/prometheus.yml create mode 100644 hostinger/readme.md create mode 100644 hostinger/system-stack.yaml create mode 100644 local/.env.example create mode 100644 local/Makefile create mode 100644 local/docker-compose-test.yaml create mode 100644 local/docker-compose.yaml create mode 100644 local/mock-ollama/Dockerfile create mode 100644 local/mock-ollama/server.ts create mode 100644 readme.md diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..35842d6 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,31 @@ +name: Deploy stacks + +on: + push: + branches: [main] + workflow_dispatch: + +jobs: + + deploy: + runs-on: ubuntu-latest + steps: + - name: docker stack deploy + uses: appleboy/ssh-action@v1.0.3 + with: + username: root + host: ${{ secrets.SSH_KNOWN_HOSTS }} + key: ${{ secrets.SSH_PRIV_KEY }} + script: | + cd ~/devops && git pull + docker stack deploy -c ~/devops/hostinger/system-stack.yaml system --with-registry-auth + docker stack deploy -c ~/devops/hostinger/prod-stack.yaml prod --with-registry-auth + + - name: prune old images + uses: appleboy/ssh-action@v1.0.3 + with: + username: root + host: ${{ secrets.SSH_KNOWN_HOSTS }} + key: ${{ secrets.SSH_PRIV_KEY }} + script: | + docker image prune -f --filter "until=24h" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2cef73e --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.env +.env.* +!.env.example diff --git a/hostinger/.env.example b/hostinger/.env.example new file mode 100644 index 0000000..3f3a9c2 --- /dev/null +++ b/hostinger/.env.example @@ -0,0 +1,41 @@ +# Production .env for the hosted-gateway swarm stack. +# Copy to .env on the Hostinger host, chmod 600, and `docker stack deploy`. +# DATABASE_URL, REDIS_*, BEE_API_URL are set in prod-stack.yaml — do NOT +# override here (the stack file uses internal DNS names). + +# --- Auth --- +JWT_SECRET_KEY=change-me-to-a-long-random-string +GOOGLE_OAUTH_CLIENT_ID= + +# --- Operator wallet (the ONLY on-chain identity) --- +# Holds xBZZ + xDAI for all tenants. Funded once at bootstrap. +OPERATOR_PRIVATE_KEY=0x... +GNOSIS_RPC_URL=https://rpc.gnosischain.com + +# --- T4T protocol contracts (Gnosis mainnet) --- +REGISTRY_ADDRESS=0xf81121AAbc2F7261224BaDd0Ed871711e6D1371E +ESCROW_ADDRESS=0x34Db8E014E71928f17E23eC1272B602582222c9c +XBZZ_ADDRESS=0xdBF3Ea6F5beE45c02255B2c26a16F300502F68da + +# --- Bee node --- +# Postage batch is bought once on first deploy and stored in bee-data volume. +BEE_POSTAGE_BATCH_ID= +BEE_STAMP_MANAGE=true +BEE_STAMP_LABEL=hosted-gateway +BEE_STAMP_TTL_DAYS=30 + +# Used by the Bee service itself (BEE_*_PASSWORD vars below): +BEE_PASSWORD=change-me +BEE_SWAP_ENDPOINT=https://rpc.gnosischain.com + +# --- Pricing --- +MARKUP_BPS=2000 +WELCOME_XBZZ_WEI=50000000000000000000 + +# --- Concurrency / server --- +MAX_CONCURRENT_JOBS=200 +LOG_LEVEL=info + +# --- Phase 2 (Stripe) — leave blank in v1 --- +STRIPE_SECRET_KEY= +STRIPE_WEBHOOK_SECRET= diff --git a/hostinger/dev-stack.yaml b/hostinger/dev-stack.yaml new file mode 100644 index 0000000..6a1f618 --- /dev/null +++ b/hostinger/dev-stack.yaml @@ -0,0 +1,145 @@ +version: '3.8' + +# Staging environment — points at the `develop` image tag, mounted on a +# separate dev.t4t-gateway.com subdomain. Shares the same overlay networks +# but uses its own postgres/redis/bee volumes so prod data is untouched. + +networks: + public: + external: true + internal: + external: true + + +volumes: + postgres-data-dev: + driver: local + redis-data-dev: + driver: local + bee-data-dev: + driver: local + + +services: + + api: + image: ghcr.io/token-for-token/hosted-gateway:develop + command: ["bun", "run", "src/api.ts"] + env_file: + - .env.dev + environment: + DATABASE_URL: postgresql://hosted:hosted@postgres-dev:5432/hosted_gateway?schema=public + REDIS_HOST: redis-dev + REDIS_PORT: 6379 + BEE_API_URL: http://bee-dev:1633 + HOSTED_GATEWAY_PORT: 8080 + networks: + - public + - internal + deploy: + replicas: 1 + update_config: + parallelism: 1 + order: start-first + resources: + limits: + memory: 256M + reservations: + memory: 128M + labels: + - "traefik.enable=true" + - "traefik.docker.network=public" + - "traefik.constraint-label=public" + - "traefik.http.routers.api-dev.rule=Host(`dev.t4t-gateway.com`)" + - "traefik.http.routers.api-dev.entrypoints=websecure" + - "traefik.http.routers.api-dev.tls.certresolver=le" + - "traefik.http.services.api-dev.loadbalancer.server.port=8080" + - "traefik.http.routers.api-dev.middlewares=redirect-to-https" + logging: + driver: loki + options: + loki-url: "https://loki.t4t-gateway.com/loki/api/v1/push" + + + worker: + image: ghcr.io/token-for-token/hosted-gateway:develop + command: ["bun", "run", "src/worker.ts"] + env_file: + - .env.dev + environment: + DATABASE_URL: postgresql://hosted:hosted@postgres-dev:5432/hosted_gateway?schema=public + REDIS_HOST: redis-dev + REDIS_PORT: 6379 + BEE_API_URL: http://bee-dev:1633 + networks: + - internal + deploy: + replicas: 1 + resources: + limits: + memory: 256M + reservations: + memory: 128M + logging: + driver: loki + options: + loki-url: "https://loki.t4t-gateway.com/loki/api/v1/push" + + + postgres-dev: + image: postgres:16 + networks: + - internal + environment: + POSTGRES_USER: hosted + POSTGRES_PASSWORD: hosted + POSTGRES_DB: hosted_gateway + volumes: + - postgres-data-dev:/var/lib/postgresql/data + deploy: + resources: + limits: + memory: 256M + reservations: + memory: 128M + logging: + driver: loki + options: + loki-url: "https://loki.t4t-gateway.com/loki/api/v1/push" + + + redis-dev: + image: redis:7 + networks: + - internal + volumes: + - redis-data-dev:/data + deploy: + resources: + limits: + memory: 64M + reservations: + memory: 32M + logging: + driver: loki + options: + loki-url: "https://loki.t4t-gateway.com/loki/api/v1/push" + + + bee-dev: + image: ethersphere/bee:latest + command: dev + networks: + - internal + volumes: + - bee-data-dev:/home/bee/.bee + deploy: + resources: + limits: + memory: 256M + reservations: + memory: 128M + logging: + driver: loki + options: + loki-url: "https://loki.t4t-gateway.com/loki/api/v1/push" diff --git a/hostinger/loki.yaml b/hostinger/loki.yaml new file mode 100644 index 0000000..b0acfbb --- /dev/null +++ b/hostinger/loki.yaml @@ -0,0 +1,37 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +ruler: + alertmanager_url: http://localhost:9093 diff --git a/hostinger/prod-stack.yaml b/hostinger/prod-stack.yaml new file mode 100644 index 0000000..0e6eaee --- /dev/null +++ b/hostinger/prod-stack.yaml @@ -0,0 +1,201 @@ +version: '3.8' + +networks: + public: + external: true + internal: + external: true + + +volumes: + postgres-data: + driver: local + redis-data: + driver: local + bee-data: + driver: local + + +services: + + api: + image: ghcr.io/token-for-token/hosted-gateway:main + command: ["bun", "run", "src/api.ts"] + env_file: + - .env + environment: + DATABASE_URL: postgresql://hosted:hosted@postgres:5432/hosted_gateway?schema=public + REDIS_HOST: redis + REDIS_PORT: 6379 + BEE_API_URL: http://bee:1633 + HOSTED_GATEWAY_PORT: 8080 + networks: + - public + - internal + deploy: + replicas: 2 + update_config: + parallelism: 1 + order: start-first + resources: + limits: + memory: 512M + reservations: + memory: 512M + labels: + - "traefik.enable=true" + - "traefik.docker.network=public" + - "traefik.constraint-label=public" + - "traefik.http.routers.api.rule=Host(`api.t4t-gateway.com`,`t4t-gateway.com`,`www.t4t-gateway.com`)" + - "traefik.http.routers.api.entrypoints=websecure" + - "traefik.http.routers.api.tls.certresolver=le" + - "traefik.http.services.api.loadbalancer.server.port=8080" + - "traefik.http.routers.api.middlewares=redirect-to-https" + logging: + driver: loki + options: + loki-url: "https://loki.t4t-gateway.com/loki/api/v1/push" + + + worker: + image: ghcr.io/token-for-token/hosted-gateway:main + command: ["bun", "run", "src/worker.ts"] + env_file: + - .env + environment: + DATABASE_URL: postgresql://hosted:hosted@postgres:5432/hosted_gateway?schema=public + REDIS_HOST: redis + REDIS_PORT: 6379 + BEE_API_URL: http://bee:1633 + networks: + - internal + deploy: + replicas: 1 + update_config: + parallelism: 1 + order: start-first + resources: + limits: + memory: 512M + reservations: + memory: 512M + logging: + driver: loki + options: + loki-url: "https://loki.t4t-gateway.com/loki/api/v1/push" + + + prisma: + image: ghcr.io/token-for-token/hosted-gateway:main + command: ["sh", "-c", "bunx prisma migrate deploy && tail -f /dev/null"] + env_file: + - .env + environment: + DATABASE_URL: postgresql://hosted:hosted@postgres:5432/hosted_gateway?schema=public + networks: + - internal + deploy: + replicas: 1 + restart_policy: + condition: on-failure + resources: + limits: + memory: 256M + reservations: + memory: 128M + logging: + driver: loki + options: + loki-url: "https://loki.t4t-gateway.com/loki/api/v1/push" + + + postgres: + image: postgres:16 + networks: + - internal + environment: + POSTGRES_USER: hosted + POSTGRES_PASSWORD: hosted + POSTGRES_DB: hosted_gateway + volumes: + - postgres-data:/var/lib/postgresql/data + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 512M + logging: + driver: loki + options: + loki-url: "https://loki.t4t-gateway.com/loki/api/v1/push" + + + redis: + image: redis:7 + networks: + - internal + volumes: + - redis-data:/data + deploy: + resources: + limits: + memory: 128M + reservations: + memory: 64M + logging: + driver: loki + options: + loki-url: "https://loki.t4t-gateway.com/loki/api/v1/push" + + + bee: + image: ethersphere/bee:latest + command: start + env_file: + - .env + environment: + BEE_API_ADDR: ":1633" + BEE_P2P_ADDR: ":1634" + BEE_VERBOSITY: "info" + BEE_FULL_NODE: "false" + BEE_SWAP_ENABLE: "true" + BEE_CHAIN_ENABLE: "true" + BEE_NETWORK_ID: "1" + BEE_WELCOME_MESSAGE: "t4t-gateway" + BEE_DATA_DIR: /home/bee/.bee + volumes: + - bee-data:/home/bee/.bee + networks: + - internal + deploy: + replicas: 1 + restart_policy: + condition: on-failure + resources: + limits: + memory: 1024M + reservations: + memory: 512M + logging: + driver: loki + options: + loki-url: "https://loki.t4t-gateway.com/loki/api/v1/push" + + + postgres-exporter: + image: quay.io/prometheuscommunity/postgres-exporter:latest + networks: + - internal + environment: + DATA_SOURCE_NAME: "postgresql://hosted:hosted@postgres:5432/hosted_gateway?sslmode=disable" + deploy: + resources: + limits: + memory: 128M + reservations: + memory: 64M + logging: + driver: loki + options: + loki-url: "https://loki.t4t-gateway.com/loki/api/v1/push" diff --git a/hostinger/prometheus.yml b/hostinger/prometheus.yml new file mode 100644 index 0000000..83685c7 --- /dev/null +++ b/hostinger/prometheus.yml @@ -0,0 +1,48 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: "traefik" + static_configs: + - targets: ["traefik:8080"] + + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] + + - job_name: "grafana" + static_configs: + - targets: ["grafana:3000"] + + - job_name: "loki" + static_configs: + - targets: ["loki:3100"] + + - job_name: "node-exporter" + dns_sd_configs: + - names: + - "tasks.system_node-exporter" + type: A + port: 9100 + + - job_name: "postgres-exporter" + dns_sd_configs: + - names: + - "tasks.prod_postgres-exporter" + type: A + port: 9187 + + - job_name: "cadvisor" + dns_sd_configs: + - names: + - "tasks.system_cadvisor" + type: A + port: 8080 + + - job_name: "hosted-gateway" + dns_sd_configs: + - names: + - "tasks.prod_api" + type: A + port: 8080 diff --git a/hostinger/readme.md b/hostinger/readme.md new file mode 100644 index 0000000..4254fef --- /dev/null +++ b/hostinger/readme.md @@ -0,0 +1,128 @@ +# Hostinger (production) stacks + +Docker Swarm stack files for the t4t hosted-gateway, deployed to a single +Hostinger VPS at `ssh root@195.35.25.26` and reachable at +[t4t-gateway.com](https://t4t-gateway.com). + +Layout mirrors [cipherdolls/devops/hostinger/](https://github.com/cipherdolls/devops). + +## Stacks + +| Stack | File | What it runs | +| --- | --- | --- | +| `system` | `system-stack.yaml` | Traefik (TLS + ingress), Grafana, Loki, Prometheus, node-exporter, cadvisor. Owns the `public` + `internal` overlay networks. | +| `prod` | `prod-stack.yaml` | hosted-gateway api + worker + one-shot prisma migrate, plus postgres / redis / bee / postgres-exporter. | +| `dev` | `dev-stack.yaml` | Staging on `dev.t4t-gateway.com` — the `:develop` image tag against its own postgres / redis / bee volumes. | + +Supporting configs: + +- `loki.yaml` — Loki single-binary filesystem config +- `prometheus.yml` — scrape targets (traefik, grafana, loki, node-exporter, + postgres-exporter, cadvisor, hosted-gateway api) +- `.env.example` — copy to `.env` on the host, fill in + `OPERATOR_PRIVATE_KEY` / `JWT_SECRET_KEY` / etc. + +## DNS records (Hostinger DNS panel) + +Point all of these at `195.35.25.26`: + +``` +A t4t-gateway.com 195.35.25.26 +A www.t4t-gateway.com 195.35.25.26 +A api.t4t-gateway.com 195.35.25.26 +A dev.t4t-gateway.com 195.35.25.26 +A traefik.t4t-gateway.com 195.35.25.26 +A grafana.t4t-gateway.com 195.35.25.26 +A loki.t4t-gateway.com 195.35.25.26 +A prometheus.t4t-gateway.com 195.35.25.26 +``` + +## One-time host setup + +```bash +ssh root@195.35.25.26 + +# Docker + swarm +curl -fsSL https://get.docker.com | sh +docker swarm init --advertise-addr 195.35.25.26 + +# Loki driver (so container logs ship to Loki via the `loki` logging driver) +docker plugin install grafana/loki-docker-driver:2.9.7 \ + --alias loki --grant-all-permissions + +# Drop the stack files + configs on the host +mkdir -p /opt/t4t-gateway && cd /opt/t4t-gateway +# rsync this directory up here (or `git clone` the devops repo) +cp .env.example .env && chmod 600 .env +$EDITOR .env # fill in OPERATOR_PRIVATE_KEY, JWT_SECRET_KEY, ... +``` + +## Deploy + +The system stack creates the shared overlay networks, so deploy it first. + +```bash +cd /opt/t4t-gateway + +# 1. Networks + ingress + monitoring +docker stack deploy -c system-stack.yaml system + +# 2. App + datastores +docker stack deploy -c prod-stack.yaml prod + +# (optional) staging on dev.t4t-gateway.com +docker stack deploy -c dev-stack.yaml dev +``` + +Re-deploying after a new image tag: + +```bash +docker service update --image ghcr.io/token-for-token/hosted-gateway:main \ + --force prod_api +docker service update --image ghcr.io/token-for-token/hosted-gateway:main \ + --force prod_worker +``` + +## Secrets + +`OPERATOR_PRIVATE_KEY` is the on-chain identity for every tenant. For now it +lives in `/opt/t4t-gateway/.env` (chmod 600). Future work: migrate to a +proper Docker secret backed by the host's secret manager — never an env var +in the stack file. + +## Backups + +Postgres volume: + +```bash +docker run --rm \ + -v prod_postgres-data:/from_volume \ + -v $(pwd):/workdir \ + alpine \ + sh -c "cd /from_volume && tar cvf /workdir/prod_postgres-data.tar ." + +scp root@195.35.25.26:/opt/t4t-gateway/prod_postgres-data.tar ~/Downloads +``` + +Bee data volume (postage batch / swarm keys — DO NOT lose this): + +```bash +docker run --rm \ + -v prod_bee-data:/from_volume \ + -v $(pwd):/workdir \ + alpine \ + sh -c "cd /from_volume && tar cvf /workdir/prod_bee-data.tar ." +``` + +## Troubleshooting + +```bash +docker stack services system # health of ingress + monitoring +docker stack services prod # health of gateway + data +docker service logs -f prod_api +docker service logs -f prod_worker + +# Replay a failed Prisma migration: +docker exec -it $(docker ps -q -f name=prod_prisma) sh +bunx prisma migrate resolve --rolled-back +``` diff --git a/hostinger/system-stack.yaml b/hostinger/system-stack.yaml new file mode 100644 index 0000000..f6841eb --- /dev/null +++ b/hostinger/system-stack.yaml @@ -0,0 +1,232 @@ +version: '3.8' + + +networks: + public: + name: public + driver: overlay + + internal: + name: internal + driver: overlay + + +volumes: + traefik-letsencrypt: + driver: local + traefik-logs: + driver: local + grafana-data: + driver: local + loki-data: + driver: local + prometheus-data: + driver: local + + +configs: + loki_config: + file: ./loki.yaml + prometheus_config: + file: ./prometheus.yml + + +services: + + traefik: + image: traefik:v2.11.22 + ports: + - "80:80" + - "443:443" + command: + - --providers.docker + - --providers.docker.network=public + - --providers.docker.constraints=Label(`traefik.constraint-label`, `public`) + - --providers.docker.exposedbydefault=false + - --providers.docker.swarmmode + - --entrypoints.web.address=:80 + - --entrypoints.websecure.address=:443 + - --certificatesresolvers.le.acme.email=t4t-gateway@proton.me + - --certificatesresolvers.le.acme.storage=/certificates/acme.json + - --certificatesresolvers.le.acme.tlschallenge=true + - --api=true + - --api.dashboard=true + - --metrics.prometheus=true + - --accesslog=true + - --log=true + networks: + - public + volumes: + - "/var/run/docker.sock:/var/run/docker.sock:ro" + - traefik-letsencrypt:/certificates + - traefik-logs:/var/log/traefik + deploy: + placement: + constraints: + - node.role == manager + resources: + limits: + memory: 256M + reservations: + memory: 128M + labels: + - "traefik.enable=true" + - "traefik.docker.network=public" + - "traefik.constraint-label=public" + - "traefik.http.routers.dashboard.service=api@internal" + - "traefik.http.routers.traefikk.rule=Host(`traefik.t4t-gateway.com`)" + - "traefik.http.routers.traefikk.entrypoints=websecure" + - "traefik.http.routers.traefikk.tls.certresolver=le" + - "traefik.http.routers.traefikk.middlewares=admin-auth" + - "traefik.http.services.traefikk.loadbalancer.server.port=8080" + - "traefik.http.middlewares.admin-auth.basicauth.users=admin:$$apr1$$EyLN9D7y$$4v2q7sYTN7aIlE2yeujRJ/" + - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https" + - "traefik.http.middlewares.redirect-to-https.redirectscheme.permanent=true" + logging: + driver: loki + options: + loki-url: "https://loki.t4t-gateway.com/loki/api/v1/push" + + + grafana: + image: grafana/grafana:12.4.2 + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:3000/api/health || exit 1"] + interval: 10s + timeout: 3s + retries: 3 + start_period: 10s + networks: + - public + - internal + volumes: + - grafana-data:/var/lib/grafana + environment: + - GF_AUTH_DISABLE_LOGIN_FORM=true + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_SECURITY_ALLOW_EMBEDDING=true + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 256M + labels: + - "traefik.enable=true" + - "traefik.http.routers.grafana.rule=Host(`grafana.t4t-gateway.com`)" + - "traefik.http.routers.grafana.entrypoints=websecure" + - "traefik.http.routers.grafana.tls.certresolver=le" + - "traefik.docker.network=public" + - "traefik.constraint-label=public" + - "traefik.http.services.grafana.loadbalancer.server.port=3000" + - "traefik.http.middlewares.admin-auth.basicauth.users=admin:$$apr1$$EyLN9D7y$$4v2q7sYTN7aIlE2yeujRJ/" + - "traefik.http.routers.grafana.middlewares=admin-auth,redirect-to-https" + logging: + driver: loki + options: + loki-url: "https://loki.t4t-gateway.com/loki/api/v1/push" + + + loki: + image: grafana/loki:2.9.7 + networks: + - public + - internal + configs: + - source: loki_config + target: /etc/loki/loki.yaml + volumes: + - loki-data:/loki + command: -config.file=/etc/loki/loki.yaml + deploy: + resources: + limits: + memory: 256M + reservations: + memory: 128M + labels: + - "traefik.enable=true" + - "traefik.http.routers.loki.rule=Host(`loki.t4t-gateway.com`)" + - "traefik.http.routers.loki.entrypoints=websecure" + - "traefik.http.routers.loki.tls.certresolver=le" + - "traefik.docker.network=public" + - "traefik.constraint-label=public" + - "traefik.http.services.loki.loadbalancer.server.port=3100" + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + + + prometheus: + image: prom/prometheus:v3.7.2 + networks: + - public + - internal + configs: + - source: prometheus_config + target: /etc/prometheus/prometheus.yml + volumes: + - prometheus-data:/prometheus + command: --config.file=/etc/prometheus/prometheus.yml + deploy: + resources: + limits: + memory: 1024M + reservations: + memory: 512M + labels: + - "traefik.enable=true" + - "traefik.http.routers.prometheus.rule=Host(`prometheus.t4t-gateway.com`)" + - "traefik.http.routers.prometheus.entrypoints=websecure" + - "traefik.http.routers.prometheus.tls.certresolver=le" + - "traefik.docker.network=public" + - "traefik.constraint-label=public" + - "traefik.http.services.prometheus.loadbalancer.server.port=9090" + - "traefik.http.middlewares.admin-auth.basicauth.users=admin:$$apr1$$EyLN9D7y$$4v2q7sYTN7aIlE2yeujRJ/" + - "traefik.http.routers.prometheus.middlewares=admin-auth" + logging: + driver: loki + options: + loki-url: "https://loki.t4t-gateway.com/loki/api/v1/push" + + + node-exporter: + image: prom/node-exporter:master + command: --path.rootfs=/host + networks: + - internal + deploy: + mode: global + resources: + limits: + memory: 32M + reservations: + memory: 24M + logging: + driver: loki + options: + loki-url: "https://loki.t4t-gateway.com/loki/api/v1/push" + + + cadvisor: + image: gcr.io/cadvisor/cadvisor:latest + networks: + - internal + volumes: + - /:/rootfs:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /sys/fs/cgroup:/sys/fs/cgroup:ro + deploy: + mode: global + resources: + limits: + memory: 128M + reservations: + memory: 65M + logging: + driver: loki + options: + loki-url: "https://loki.t4t-gateway.com/loki/api/v1/push" diff --git a/local/.env.example b/local/.env.example new file mode 100644 index 0000000..61374d6 --- /dev/null +++ b/local/.env.example @@ -0,0 +1,48 @@ +# Local dev + test env. Copy to .env and adjust. +# Compose files load this with `env_file: [.env]`. + +# Postgres (used by hosted-gateway via DATABASE_URL) +POSTGRES_USER=hosted +POSTGRES_PASSWORD=hosted +POSTGRES_DB=hosted_gateway +DATABASE_URL=postgresql://hosted:hosted@postgres:5432/hosted_gateway?schema=public + +# Redis +REDIS_HOST=redis +REDIS_PORT=6379 + +# Auth +JWT_SECRET_KEY=dev-only-jwt-secret-change-me +GOOGLE_OAUTH_CLIENT_ID= + +# Operator wallet — anvil deterministic key (account 0) for local dev/test. +# The bootstrap step in test/setup.ts funds this address with xDAI + xBZZ +# via anvil_setBalance + xBZZ-whale impersonation. +OPERATOR_PRIVATE_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 +GNOSIS_RPC_URL=http://anvil:8545 + +# T4T protocol contracts (Gnosis mainnet — managed by the t4t repo). +# These are valid on the anvil fork because we --fork-url'd Gnosis mainnet. +REGISTRY_ADDRESS=0xf81121AAbc2F7261224BaDd0Ed871711e6D1371E +ESCROW_ADDRESS=0x34Db8E014E71928f17E23eC1272B602582222c9c +XBZZ_ADDRESS=0xdBF3Ea6F5beE45c02255B2c26a16F300502F68da + +# Bee — we point at a real running Bee node (bee.ffaerber.duckdns.org) rather +# than spinning up dev-mode Bee in compose. The remote Bee already has real +# postage stamps for PSS routing to mainnet providers. +BEE_API_URL=https://bee.ffaerber.duckdns.org +BEE_POSTAGE_BATCH_ID= +BEE_STAMP_MANAGE=false +BEE_STAMP_LABEL=hosted-gateway-local +BEE_STAMP_TTL_DAYS=30 + +# Pricing +MARKUP_BPS=2000 +WELCOME_XBZZ_WEI=50000000000000000000 + +# Concurrency +MAX_CONCURRENT_JOBS=200 + +# Server +HOSTED_GATEWAY_PORT=8080 +LOG_LEVEL=debug diff --git a/local/Makefile b/local/Makefile new file mode 100644 index 0000000..07857d9 --- /dev/null +++ b/local/Makefile @@ -0,0 +1,159 @@ +# Hosted gateway — local dev + test orchestration. +# Mirrors cipherdolls/devops/local/Makefile. + +PROJECT := hosted-gateway +COMPOSE_DEV := docker-compose.yaml +COMPOSE_TEST := docker-compose-test.yaml +DC := docker compose -p $(PROJECT) -f $(COMPOSE_TEST) +DC_DEV := docker compose -p $(PROJECT)-dev -f $(COMPOSE_DEV) +TEST_CONTAINER := $(PROJECT)-bun +LOG_DIR := /tmp/hosted-gateway-test-logs + +SPEC ?= accounts +SPECS := health auth models gateway ledger cancellation concurrency +TIMEOUT ?= 600 + +SERVICES := postgres redis prisma api worker + +# Specs that need the gnosis-fork stack (anvil + bee) and/or a registered +# provider → activate the `gateway` compose profile. `models` only needs the +# fork (read-only chain calls); `gateway`/`concurrency`/`cancellation` also +# need the t4t-provider + mock-ollama containers. +GATEWAY_SPECS := models gateway concurrency cancellation +gateway_profile = $(if $(filter $(1),$(GATEWAY_SPECS)),--profile gateway,) + +.PHONY: dev-up dev-down dev-watch up up-detached down logs build clean test test-all $(addprefix test-,$(SPECS)) help lock + +# ---------- dev ---------- + +dev-up: + $(DC_DEV) up + +dev-down: + $(DC_DEV) down -v --remove-orphans -t 0 + +dev-watch: + $(DC_DEV) watch + +# ---------- test ---------- + +# Generate / refresh bun.lock inside the hosted-gateway repo via a one-shot +# container. Mirrors cipherdolls' approach so no host-side Bun install is +# required. +lock: + @echo "========== Generating bun.lock ==========" + docker run --rm -v ../../hosted-gateway:/app -w /app oven/bun:1.2-alpine bun install + @echo "========== bun.lock updated ==========" + +# Run a single spec by name. Example: `make test SPEC=accounts` +define run_test + bash -c '\ + set +e; \ + mkdir -p $(LOG_DIR); \ + cleanup() { \ + echo ""; \ + echo "========== Cleaning up =========="; \ + docker rm -f $(TEST_CONTAINER) 2>/dev/null; \ + $(DC) kill 2>/dev/null; \ + $(DC) down -v --remove-orphans -t 0 2>/dev/null; \ + }; \ + trap cleanup EXIT; \ + echo "========== Updating bun.lock =========="; \ + docker run --rm -v ../../hosted-gateway:/app -w /app oven/bun:1.2-alpine bun install --frozen-lockfile 2>/dev/null || \ + docker run --rm -v ../../hosted-gateway:/app -w /app oven/bun:1.2-alpine bun install; \ + echo "========== Starting services =========="; \ + $(DC) $(call gateway_profile,$(1)) up -d --build --wait $(SERVICES); \ + if [ "$(call gateway_profile,$(1))" != "" ]; then \ + echo "========== Starting gateway profile (anvil) =========="; \ + $(DC) $(call gateway_profile,$(1)) up -d --build --wait anvil || { echo "gateway profile failed to start"; exit 1; }; \ + if [ "$(1)" != "models" ]; then \ + echo "========== Starting t4t-provider + mock-ollama =========="; \ + $(DC) $(call gateway_profile,$(1)) up -d --build --wait t4t-provider mock-ollama || { echo "provider profile failed to start"; exit 1; }; \ + fi; \ + fi; \ + echo "========== Running $(1) =========="; \ + $(DC) build --no-cache bun; \ + $(DC) $(call gateway_profile,$(1)) run -d --no-deps --name $(TEST_CONTAINER) \ + -e SPEC_GLOB="./test/$(1).e2e-spec.ts" \ + bun; \ + \ + docker wait $(TEST_CONTAINER) &>/dev/null & \ + WAIT_PID=$$!; \ + stdbuf -oL docker logs -f $(TEST_CONTAINER) 2>&1 \ + | stdbuf -oL sed "s/\x1b\[[0-9;]*m//g" \ + > $(LOG_DIR)/$(1).log & \ + LOGS_PID=$$!; \ + \ + if ! timeout $(TIMEOUT) tail --pid=$$WAIT_PID -f /dev/null 2>/dev/null; then \ + echo "========== TIMEOUT after $(TIMEOUT)s =========="; \ + docker kill $(TEST_CONTAINER) 2>/dev/null; \ + fi; \ + wait $$WAIT_PID 2>/dev/null; \ + rc=$$(docker inspect -f "{{.State.ExitCode}}" $(TEST_CONTAINER) 2>/dev/null || echo 1); \ + kill $$LOGS_PID 2>/dev/null; \ + wait $$LOGS_PID 2>/dev/null; \ + echo "========== Tests finished (rc=$$rc) — log: $(LOG_DIR)/$(1).log =========="; \ + exit $$rc' +endef + +test: + $(call run_test,$(SPEC)) + +test-all: + @failed=""; \ + for spec in $(SPECS); do \ + echo ""; \ + echo "========== Suite: $$spec =========="; \ + $(MAKE) --no-print-directory test SPEC=$$spec; \ + if [ $$? -ne 0 ]; then failed="$$failed $$spec"; fi; \ + done; \ + if [ -n "$$failed" ]; then \ + echo ""; \ + echo "Failed specs:$$failed"; exit 1; \ + else \ + echo ""; \ + echo "All specs passed"; \ + fi + +$(addprefix test-,$(SPECS)): + $(call run_test,$(subst test-,,$@)) + +up: + $(DC) up + +up-detached: + $(DC) up -d + +down: + $(DC) kill 2>/dev/null || true + $(DC) down -v --remove-orphans -t 0 + +logs: + $(DC) logs -f + +build: + $(DC) build + +clean: + $(DC) kill 2>/dev/null || true + $(DC) down -v --remove-orphans --rmi local -t 0 + @docker ps -aq --filter "label=com.docker.compose.project=$(PROJECT)" | xargs -r docker rm -f 2>/dev/null || true + @docker volume ls -q --filter "label=com.docker.compose.project=$(PROJECT)" | xargs -r docker volume rm -f 2>/dev/null || true + @docker network ls -q --filter "label=com.docker.compose.project=$(PROJECT)" | xargs -r docker network rm 2>/dev/null || true + +help: + @echo "Usage:" + @echo " make dev-up Start dev stack (postgres, redis, anvil, bee, api, worker)" + @echo " make dev-down Stop and clean dev stack" + @echo " make dev-watch Dev with file sync" + @echo " make test SPEC=accounts Run one e2e spec inside docker" + @echo " make test-accounts Same as `make test SPEC=accounts`" + @echo " make test-all Run every spec sequentially" + @echo " make up Start test stack (no test runner)" + @echo " make down Tear down test stack" + @echo " make logs Tail test stack logs" + @echo " make lock Regenerate bun.lock via docker" + @echo " make clean Nuclear cleanup" + @echo "" + @echo "Available specs:" + @echo " $(SPECS)" | fold -s -w 70 diff --git a/local/docker-compose-test.yaml b/local/docker-compose-test.yaml new file mode 100644 index 0000000..0862024 --- /dev/null +++ b/local/docker-compose-test.yaml @@ -0,0 +1,166 @@ +# Local test stack for the hosted gateway. The smart contracts are NOT +# deployed here — they're managed by the t4t repo and already live on Gnosis +# mainnet. The anvil service forks Gnosis mainnet so those addresses are +# usable as-is. +# +# Funding (giving the operator + test-provider xBZZ and xDAI on the fork) is +# handled inside `test/setup.ts` via viem's anvil test actions +# (anvil_setBalance + impersonateAccount on an xBZZ whale). +# +# Run from `devops/local/`: `make test SPEC=accounts` / `make test-all`. + +services: + + postgres: + image: postgres:16 + env_file: [.env] + expose: + - "5432" + tmpfs: + - /var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 2s + timeout: 2s + retries: 60 + networks: [test] + + redis: + image: redis:7 + expose: + - "6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + retries: 30 + networks: [test] + + # Anvil + Bee are only needed by specs that hit chain / Swarm. Profile-gated + # so `make test SPEC=auth` runs without them; `make test SPEC=gateway` adds + # the gateway profile. + anvil: + image: ghcr.io/foundry-rs/foundry:latest + pull_policy: always + profiles: [gateway] + # The foundry image's ENTRYPOINT is `["/bin/sh", "-c"]`, which interprets + # only its FIRST argument as the command — anvil's flags then ended up as + # shell positional params and were silently dropped. Override the + # entrypoint to invoke anvil directly. Don't pass `--chain-id` here — + # `--fork-url` propagates the chain id (100) automatically. + entrypoint: ["anvil"] + command: + - --host=0.0.0.0 + - --port=8545 + - --fork-url=https://rpc.gnosischain.com + - --block-time=1 + environment: + ANVIL_IP_ADDR: "0.0.0.0" + expose: + - "8545" + healthcheck: + test: ["CMD-SHELL", "cast block-number --rpc-url http://localhost:8545 >/dev/null 2>&1"] + interval: 3s + retries: 60 + start_period: 60s + networks: [test] + + # Local Bee was removed — see devops/local/.env BEE_API_URL. We hit + # bee.ffaerber.duckdns.org (real running Bee node) for PSS routing + Swarm + # uploads. Faster than the local dev container and avoids the dev-mode + # postage / wget healthcheck quirks. + + prisma: + build: + context: ../../hosted-gateway + command: ["sh", "-c", "bunx prisma migrate deploy"] + env_file: [.env] + depends_on: + postgres: { condition: service_healthy } + networks: [test] + + api: + build: + context: ../../hosted-gateway + command: ["bun", "run", "src/api.ts"] + env_file: [.env] + environment: + NODE_ENV: test + expose: + - "8080" + depends_on: + postgres: { condition: service_healthy } + redis: { condition: service_healthy } + prisma: { condition: service_completed_successfully } + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:8080/health || exit 1"] + interval: 3s + timeout: 3s + retries: 30 + start_period: 30s + networks: [test] + + worker: + build: + context: ../../hosted-gateway + command: ["bun", "run", "src/worker.ts"] + env_file: [.env] + environment: + NODE_ENV: test + depends_on: + postgres: { condition: service_healthy } + redis: { condition: service_healthy } + prisma: { condition: service_completed_successfully } + healthcheck: + test: ["CMD-SHELL", "ps aux | grep 'src/worker.ts' | grep -v grep || exit 1"] + interval: 3s + retries: 30 + networks: [test] + + # The existing t4t-container in provider mode. Reads the canonical Gnosis + # mainnet contract addresses from its baked defaults (see + # t4t/container/src/lib/config.ts) — those addresses are valid on the anvil + # fork too. Funding (xBZZ + xDAI on the fork) happens in `test/setup.ts`. + # + # Profile-gated: only specs that actually need to settle a chat completion + # on-chain require this container. `make test SPEC=auth` / `make test + # SPEC=accounts` skip it; `make test SPEC=gateway` activates the profile. + t4t-provider: + image: ghcr.io/token-for-token/t4t:latest + pull_policy: missing + profiles: [gateway] + environment: + T4T_MODE: provider + T4T_OLLAMA_URL: http://mock-ollama:11434 + GNOSIS_RPC_URL: http://anvil:8545 + BEE_API_URL: http://bee:1633 + depends_on: + anvil: { condition: service_healthy } + mock-ollama: { condition: service_started } + networks: [test] + + mock-ollama: + build: + context: ./mock-ollama + profiles: [gateway] + expose: + - "11434" + networks: [test] + + bun: + build: + context: ../../hosted-gateway + # The leading `$$` escapes compose's variable substitution so the inner + # shell expands $SPEC_GLOB at runtime (where `make test` set it via `-e`). + command: ["sh", "-c", "bun test --timeout 60000 $${SPEC_GLOB:-./test/**/*.e2e-spec.ts}"] + env_file: [.env] + environment: + NODE_ENV: test + HOSTED_GATEWAY_URL: http://api:8080 + depends_on: + api: { condition: service_healthy } + worker: { condition: service_healthy } + networks: [test] + +networks: + test: + driver: bridge diff --git a/local/docker-compose.yaml b/local/docker-compose.yaml new file mode 100644 index 0000000..ce2ed48 --- /dev/null +++ b/local/docker-compose.yaml @@ -0,0 +1,108 @@ +# Local dev stack for the hosted gateway. Brings up everything the gateway +# needs to serve `/v1/chat/completions` end-to-end against a forked Gnosis +# chain + a real t4t-provider talking to a mock Ollama. +# +# Run from `devops/local/`: `make dev-up` / `docker compose up`. + +services: + + postgres: + image: postgres:16 + env_file: [.env] + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 3s + timeout: 3s + retries: 30 + networks: [dev] + + redis: + image: redis:7 + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + retries: 30 + networks: [dev] + + anvil: + image: ghcr.io/foundry-rs/foundry:latest + pull_policy: always + command: + - anvil + - --host=0.0.0.0 + - --port=8545 + - --chain-id=100 + - --fork-url=${GNOSIS_RPC_URL_UPSTREAM:-https://rpc.gnosischain.com} + - --block-time=1 + environment: + ANVIL_IP_ADDR: "0.0.0.0" + expose: + - "8545" + ports: + - "8545:8545" + healthcheck: + test: ["CMD-SHELL", "cast block-number --rpc-url http://localhost:8545 >/dev/null 2>&1"] + interval: 3s + timeout: 3s + retries: 60 + start_period: 30s + networks: [dev] + + bee: + image: ethersphere/bee:latest + command: dev + ports: + - "1633:1633" + networks: [dev] + + # Prisma migration step — runs once on `up`, then exits. + prisma: + build: + context: ../../hosted-gateway + command: ["sh", "-c", "bunx prisma migrate deploy"] + env_file: [.env] + depends_on: + postgres: { condition: service_healthy } + networks: [dev] + + api: + build: + context: ../../hosted-gateway + command: ["bun", "run", "src/api.ts"] + env_file: [.env] + ports: + - "8080:8080" + depends_on: + postgres: { condition: service_healthy } + redis: { condition: service_healthy } + anvil: { condition: service_healthy } + bee: { condition: service_started } + prisma: { condition: service_completed_successfully } + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:8080/health || exit 1"] + interval: 5s + timeout: 3s + retries: 30 + start_period: 15s + networks: [dev] + + worker: + build: + context: ../../hosted-gateway + command: ["bun", "run", "src/worker.ts"] + env_file: [.env] + depends_on: + postgres: { condition: service_healthy } + redis: { condition: service_healthy } + anvil: { condition: service_healthy } + bee: { condition: service_started } + prisma: { condition: service_completed_successfully } + networks: [dev] + +networks: + dev: + driver: bridge diff --git a/local/mock-ollama/Dockerfile b/local/mock-ollama/Dockerfile new file mode 100644 index 0000000..7c5f7be --- /dev/null +++ b/local/mock-ollama/Dockerfile @@ -0,0 +1,8 @@ +# Minimal mock Ollama HTTP server for E2E tests — returns a canned +# `/api/chat`-shaped completion regardless of input. The t4t-provider treats +# this as its inference backend; no GPU needed. +FROM oven/bun:1.2-alpine +WORKDIR /app +COPY server.ts ./server.ts +EXPOSE 11434 +CMD ["bun", "run", "server.ts"] diff --git a/local/mock-ollama/server.ts b/local/mock-ollama/server.ts new file mode 100644 index 0000000..3cbf29e --- /dev/null +++ b/local/mock-ollama/server.ts @@ -0,0 +1,47 @@ +// Stub Ollama HTTP for E2E tests. Returns a deterministic completion so the +// gateway -> provider -> claim flow can be asserted end-to-end without a GPU. +// +// Implements just enough of the Ollama HTTP surface to satisfy the +// t4t-provider's inference call: GET /api/tags and POST /api/chat. + +const PORT = 11434; + +Bun.serve({ + port: PORT, + hostname: '0.0.0.0', + async fetch(req: Request): Promise { + const url = new URL(req.url); + if (url.pathname === '/api/tags') { + return Response.json({ + models: [ + { + name: 'qwen2.5:0.5b', + modified_at: new Date().toISOString(), + size: 0, + digest: 'mock', + details: { + format: 'gguf', + family: 'qwen', + parameter_size: '0.5B', + quantization_level: 'Q4_0', + }, + }, + ], + }); + } + if (url.pathname === '/api/chat' && req.method === 'POST') { + return Response.json({ + model: 'qwen2.5:0.5b', + created_at: new Date().toISOString(), + message: { role: 'assistant', content: 'pong (mock-ollama)' }, + done: true, + total_duration: 100_000_000, + prompt_eval_count: 10, + eval_count: 5, + }); + } + return new Response('not implemented', { status: 404 }); + }, +}); + +console.log(`mock-ollama listening on :${PORT}`); diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..1ddb9ce --- /dev/null +++ b/readme.md @@ -0,0 +1,42 @@ +# DevOps + +This repo manages the two environments for the Token4Token hosted gateway: + +- **`local/`** — docker-compose files for running development and testing + environments locally. Tests run inside Docker, no host-side Bun needed. +- **`hostinger/`** — Docker Swarm stack files for the remote production + cluster on Hostinger. + +The layout mirrors [cipherdolls/devops](https://github.com/cipherdolls/devops). + +## Local development + +```bash +cd local +make up # bring the dev stack up (postgres, redis, bee, anvil, api, worker) +make logs # tail +make down # clean shutdown +``` + +The dev compose mounts `../../hosted-gateway/src` so file changes hot-reload +inside the container (`bun run --watch`). + +## Local testing + +Every E2E spec runs inside Docker against a fresh stack (anvil fork of Gnosis, +deployed contracts, a t4t-provider container against a stub Ollama). + +```bash +cd local +make test SPEC=accounts # run one spec +make test-all # run every spec sequentially +``` + +`make test` brings up postgres + redis + anvil + bee + t4t-provider + hosted +api + worker, then runs the `bun` test container against them. After the run +it tears everything down (`-v --remove-orphans`). + +## Hostinger / production + +Stack files in `hostinger/` are applied via `docker stack deploy` from the CI +pipeline. Not relevant for local development.