grafana: provision Loki/Prometheus datasources + log dashboards

Bind two ready-to-use dashboards at grafana.t4t-gateway.com:

- Logs — All services: per-service log volume timeseries + live tail,
  with a service selector and a free-text search box.
- Hosted-gateway API — errors & PSS inbox: error rate, ack/deliver
  counters, and a live tail filtered to PSS inbox traffic so the
  webhook fallback is observable.

Datasources + dashboard providers are mounted via Docker configs from
hostinger/grafana/ so they survive grafana container restarts and live
in git rather than the Grafana UI.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

hostinger/grafana/dashboards.yaml

@@ -0,0 +1,13 @@
+apiVersion: 1
+
+providers:
+  - name: t4t
+    orgId: 1
+    folder: ''
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 30
+    allowUiUpdates: true
+    options:
+      path: /etc/grafana/dashboards
+      foldersFromFilesStructure: false

hostinger/grafana/dashboards/api-errors.json

@@ -0,0 +1,83 @@
+{
+  "title": "Hosted-gateway API — errors & PSS inbox",
+  "uid": "t4t-api-errors",
+  "schemaVersion": 39,
+  "version": 1,
+  "editable": true,
+  "refresh": "10s",
+  "tags": ["t4t", "api"],
+  "time": {"from": "now-1h", "to": "now"},
+  "templating": {"list": []},
+  "panels": [
+    {
+      "type": "timeseries",
+      "title": "Errors per minute (api + worker)",
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
+      "datasource": {"type": "loki", "uid": "loki"},
+      "targets": [
+        {
+          "expr": "sum by (swarm_service) (count_over_time({swarm_service=~\"prod_(api|worker)\"} |~ \"(?i)\\\"level\\\":(50|40)\" [1m]))",
+          "legendFormat": "{{swarm_service}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {"defaults": {"unit": "logs", "custom": {"drawStyle": "bars", "fillOpacity": 60}}}
+    },
+    {
+      "type": "stat",
+      "title": "Acks last 15m",
+      "gridPos": {"h": 8, "w": 6, "x": 12, "y": 0},
+      "datasource": {"type": "loki", "uid": "loki"},
+      "targets": [
+        {
+          "expr": "sum(count_over_time({swarm_service=\"prod_api\"} |~ \"ack received\" [15m]))",
+          "refId": "A"
+        }
+      ],
+      "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "graphMode": "area"}
+    },
+    {
+      "type": "stat",
+      "title": "Deliveries last 15m",
+      "gridPos": {"h": 8, "w": 6, "x": 18, "y": 0},
+      "datasource": {"type": "loki", "uid": "loki"},
+      "targets": [
+        {
+          "expr": "sum(count_over_time({swarm_service=\"prod_api\"} |~ \"envelope dispatched\" |~ \"job_deliver\" [15m]))",
+          "refId": "A"
+        }
+      ],
+      "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "graphMode": "area"}
+    },
+    {
+      "type": "logs",
+      "title": "API — errors",
+      "gridPos": {"h": 14, "w": 24, "x": 0, "y": 8},
+      "datasource": {"type": "loki", "uid": "loki"},
+      "targets": [
+        {
+          "expr": "{swarm_service=~\"prod_(api|worker)\"} |~ \"(?i)\\\"level\\\":(50|40)\"",
+          "refId": "A"
+        }
+      ],
+      "options": {
+        "showTime": true,
+        "wrapLogMessage": true,
+        "sortOrder": "Descending"
+      }
+    },
+    {
+      "type": "logs",
+      "title": "PSS inbox traffic",
+      "gridPos": {"h": 14, "w": 24, "x": 0, "y": 22},
+      "datasource": {"type": "loki", "uid": "loki"},
+      "targets": [
+        {
+          "expr": "{swarm_service=\"prod_api\"} |~ \"pss/inbox|ack received|delivered\"",
+          "refId": "A"
+        }
+      ],
+      "options": {"showTime": true, "wrapLogMessage": true, "sortOrder": "Descending"}
+    }
+  ]
+}

hostinger/grafana/dashboards/logs-overview.json

@@ -0,0 +1,78 @@
+{
+  "title": "Logs — All services",
+  "uid": "t4t-logs-overview",
+  "schemaVersion": 39,
+  "version": 1,
+  "editable": true,
+  "graphTooltip": 0,
+  "refresh": "10s",
+  "tags": ["t4t", "loki"],
+  "time": {"from": "now-30m", "to": "now"},
+  "timepicker": {},
+  "templating": {
+    "list": [
+      {
+        "name": "service",
+        "type": "query",
+        "datasource": {"type": "loki", "uid": "loki"},
+        "label": "Service",
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true,
+        "current": {"text": "All", "value": "$__all"},
+        "query": "label_values(swarm_service)",
+        "sort": 1
+      },
+      {
+        "name": "search",
+        "type": "textbox",
+        "label": "Search",
+        "current": {"text": "", "value": ""}
+      }
+    ]
+  },
+  "panels": [
+    {
+      "type": "timeseries",
+      "title": "Log volume per service",
+      "gridPos": {"h": 8, "w": 24, "x": 0, "y": 0},
+      "datasource": {"type": "loki", "uid": "loki"},
+      "targets": [
+        {
+          "expr": "sum by (swarm_service) (count_over_time({swarm_service=~\"$service\"} |~ \"(?i)$search\" [1m]))",
+          "legendFormat": "{{swarm_service}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {"drawStyle": "bars", "lineWidth": 1, "fillOpacity": 60, "stacking": {"mode": "normal"}},
+          "unit": "logs"
+        }
+      },
+      "options": {"legend": {"displayMode": "table", "placement": "right", "calcs": ["sum"]}}
+    },
+    {
+      "type": "logs",
+      "title": "Live logs",
+      "gridPos": {"h": 22, "w": 24, "x": 0, "y": 8},
+      "datasource": {"type": "loki", "uid": "loki"},
+      "targets": [
+        {
+          "expr": "{swarm_service=~\"$service\"} |~ \"(?i)$search\"",
+          "refId": "A"
+        }
+      ],
+      "options": {
+        "showTime": true,
+        "showLabels": false,
+        "showCommonLabels": false,
+        "wrapLogMessage": true,
+        "prettifyLogMessage": false,
+        "enableLogDetails": true,
+        "dedupStrategy": "none",
+        "sortOrder": "Descending"
+      }
+    }
+  ]
+}

hostinger/grafana/datasources.yaml

@@ -0,0 +1,18 @@
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    uid: loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    isDefault: true
+    editable: false
+    jsonData:
+      maxLines: 5000
+  - name: Prometheus
+    uid: prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    editable: false

hostinger/system-stack.yaml

@@ -29,6 +29,14 @@ configs:
     file: ./loki.yaml
   prometheus_config:
     file: ./prometheus.yml
+  grafana_datasources:
+    file: ./grafana/datasources.yaml
+  grafana_dashboard_providers:
+    file: ./grafana/dashboards.yaml
+  grafana_dashboard_logs:
+    file: ./grafana/dashboards/logs-overview.json
+  grafana_dashboard_api:
+    file: ./grafana/dashboards/api-errors.json
 
 
 services:
@@ -103,6 +111,15 @@ services:
       - internal
     volumes:
       - grafana-data:/var/lib/grafana
+    configs:
+      - source: grafana_datasources
+        target: /etc/grafana/provisioning/datasources/datasources.yaml
+      - source: grafana_dashboard_providers
+        target: /etc/grafana/provisioning/dashboards/dashboards.yaml
+      - source: grafana_dashboard_logs
+        target: /etc/grafana/dashboards/logs-overview.json
+      - source: grafana_dashboard_api
+        target: /etc/grafana/dashboards/api-errors.json
     environment:
       - GF_AUTH_DISABLE_LOGIN_FORM=true
       - GF_AUTH_ANONYMOUS_ENABLED=true