From 7fcd9df9d3c76161789c3957829848451a9a7117 Mon Sep 17 00:00:00 2001 From: Felix Faerber Date: Fri, 29 May 2026 10:53:26 +0300 Subject: [PATCH] grafana: provision Loki/Prometheus datasources + log dashboards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bind two ready-to-use dashboards at grafana.t4t-gateway.com: - Logs — All services: per-service log volume timeseries + live tail, with a service selector and a free-text search box. - Hosted-gateway API — errors & PSS inbox: error rate, ack/deliver counters, and a live tail filtered to PSS inbox traffic so the webhook fallback is observable. Datasources + dashboard providers are mounted via Docker configs from hostinger/grafana/ so they survive grafana container restarts and live in git rather than the Grafana UI. Co-Authored-By: Claude Opus 4.7 --- hostinger/grafana/dashboards.yaml | 13 +++ hostinger/grafana/dashboards/api-errors.json | 83 +++++++++++++++++++ .../grafana/dashboards/logs-overview.json | 78 +++++++++++++++++ hostinger/grafana/datasources.yaml | 18 ++++ hostinger/system-stack.yaml | 17 ++++ 5 files changed, 209 insertions(+) create mode 100644 hostinger/grafana/dashboards.yaml create mode 100644 hostinger/grafana/dashboards/api-errors.json create mode 100644 hostinger/grafana/dashboards/logs-overview.json create mode 100644 hostinger/grafana/datasources.yaml diff --git a/hostinger/grafana/dashboards.yaml b/hostinger/grafana/dashboards.yaml new file mode 100644 index 0000000..230ec02 --- /dev/null +++ b/hostinger/grafana/dashboards.yaml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: t4t + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /etc/grafana/dashboards + foldersFromFilesStructure: false diff --git a/hostinger/grafana/dashboards/api-errors.json b/hostinger/grafana/dashboards/api-errors.json new file mode 100644 index 0000000..b9eb78f --- /dev/null +++ b/hostinger/grafana/dashboards/api-errors.json @@ -0,0 +1,83 @@ +{ + "title": "Hosted-gateway API — errors & PSS inbox", + "uid": "t4t-api-errors", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "10s", + "tags": ["t4t", "api"], + "time": {"from": "now-1h", "to": "now"}, + "templating": {"list": []}, + "panels": [ + { + "type": "timeseries", + "title": "Errors per minute (api + worker)", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}, + "datasource": {"type": "loki", "uid": "loki"}, + "targets": [ + { + "expr": "sum by (swarm_service) (count_over_time({swarm_service=~\"prod_(api|worker)\"} |~ \"(?i)\\\"level\\\":(50|40)\" [1m]))", + "legendFormat": "{{swarm_service}}", + "refId": "A" + } + ], + "fieldConfig": {"defaults": {"unit": "logs", "custom": {"drawStyle": "bars", "fillOpacity": 60}}} + }, + { + "type": "stat", + "title": "Acks last 15m", + "gridPos": {"h": 8, "w": 6, "x": 12, "y": 0}, + "datasource": {"type": "loki", "uid": "loki"}, + "targets": [ + { + "expr": "sum(count_over_time({swarm_service=\"prod_api\"} |~ \"ack received\" [15m]))", + "refId": "A" + } + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "graphMode": "area"} + }, + { + "type": "stat", + "title": "Deliveries last 15m", + "gridPos": {"h": 8, "w": 6, "x": 18, "y": 0}, + "datasource": {"type": "loki", "uid": "loki"}, + "targets": [ + { + "expr": "sum(count_over_time({swarm_service=\"prod_api\"} |~ \"envelope dispatched\" |~ \"job_deliver\" [15m]))", + "refId": "A" + } + ], + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "graphMode": "area"} + }, + { + "type": "logs", + "title": "API — errors", + "gridPos": {"h": 14, "w": 24, "x": 0, "y": 8}, + "datasource": {"type": "loki", "uid": "loki"}, + "targets": [ + { + "expr": "{swarm_service=~\"prod_(api|worker)\"} |~ \"(?i)\\\"level\\\":(50|40)\"", + "refId": "A" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "sortOrder": "Descending" + } + }, + { + "type": "logs", + "title": "PSS inbox traffic", + "gridPos": {"h": 14, "w": 24, "x": 0, "y": 22}, + "datasource": {"type": "loki", "uid": "loki"}, + "targets": [ + { + "expr": "{swarm_service=\"prod_api\"} |~ \"pss/inbox|ack received|delivered\"", + "refId": "A" + } + ], + "options": {"showTime": true, "wrapLogMessage": true, "sortOrder": "Descending"} + } + ] +} diff --git a/hostinger/grafana/dashboards/logs-overview.json b/hostinger/grafana/dashboards/logs-overview.json new file mode 100644 index 0000000..615c242 --- /dev/null +++ b/hostinger/grafana/dashboards/logs-overview.json @@ -0,0 +1,78 @@ +{ + "title": "Logs — All services", + "uid": "t4t-logs-overview", + "schemaVersion": 39, + "version": 1, + "editable": true, + "graphTooltip": 0, + "refresh": "10s", + "tags": ["t4t", "loki"], + "time": {"from": "now-30m", "to": "now"}, + "timepicker": {}, + "templating": { + "list": [ + { + "name": "service", + "type": "query", + "datasource": {"type": "loki", "uid": "loki"}, + "label": "Service", + "refresh": 2, + "includeAll": true, + "multi": true, + "current": {"text": "All", "value": "$__all"}, + "query": "label_values(swarm_service)", + "sort": 1 + }, + { + "name": "search", + "type": "textbox", + "label": "Search", + "current": {"text": "", "value": ""} + } + ] + }, + "panels": [ + { + "type": "timeseries", + "title": "Log volume per service", + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 0}, + "datasource": {"type": "loki", "uid": "loki"}, + "targets": [ + { + "expr": "sum by (swarm_service) (count_over_time({swarm_service=~\"$service\"} |~ \"(?i)$search\" [1m]))", + "legendFormat": "{{swarm_service}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": {"drawStyle": "bars", "lineWidth": 1, "fillOpacity": 60, "stacking": {"mode": "normal"}}, + "unit": "logs" + } + }, + "options": {"legend": {"displayMode": "table", "placement": "right", "calcs": ["sum"]}} + }, + { + "type": "logs", + "title": "Live logs", + "gridPos": {"h": 22, "w": 24, "x": 0, "y": 8}, + "datasource": {"type": "loki", "uid": "loki"}, + "targets": [ + { + "expr": "{swarm_service=~\"$service\"} |~ \"(?i)$search\"", + "refId": "A" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Descending" + } + } + ] +} diff --git a/hostinger/grafana/datasources.yaml b/hostinger/grafana/datasources.yaml new file mode 100644 index 0000000..24842e2 --- /dev/null +++ b/hostinger/grafana/datasources.yaml @@ -0,0 +1,18 @@ +apiVersion: 1 + +datasources: + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + isDefault: true + editable: false + jsonData: + maxLines: 5000 + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + editable: false diff --git a/hostinger/system-stack.yaml b/hostinger/system-stack.yaml index 8280b30..0cd9edf 100644 --- a/hostinger/system-stack.yaml +++ b/hostinger/system-stack.yaml @@ -29,6 +29,14 @@ configs: file: ./loki.yaml prometheus_config: file: ./prometheus.yml + grafana_datasources: + file: ./grafana/datasources.yaml + grafana_dashboard_providers: + file: ./grafana/dashboards.yaml + grafana_dashboard_logs: + file: ./grafana/dashboards/logs-overview.json + grafana_dashboard_api: + file: ./grafana/dashboards/api-errors.json services: @@ -103,6 +111,15 @@ services: - internal volumes: - grafana-data:/var/lib/grafana + configs: + - source: grafana_datasources + target: /etc/grafana/provisioning/datasources/datasources.yaml + - source: grafana_dashboard_providers + target: /etc/grafana/provisioning/dashboards/dashboards.yaml + - source: grafana_dashboard_logs + target: /etc/grafana/dashboards/logs-overview.json + - source: grafana_dashboard_api + target: /etc/grafana/dashboards/api-errors.json environment: - GF_AUTH_DISABLE_LOGIN_FORM=true - GF_AUTH_ANONYMOUS_ENABLED=true