{ "annotations": { "list": [] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], "liveNow": true, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100, "panels": [], "title": "Scheduler Status Overview", "type": "row" }, { "datasource": { "type": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "0": { "color": "red", "index": 0, "text": "Not Running" }, "1": { "color": "green", "index": 1, "text": "Running" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, "id": 1, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "textMode": "value" }, "targets": [ { "datasource": { "type": "prometheus" }, "expr": "count(scheduler_runs_total{schedule_type=\"morning_scrape\"}) > 0", "refId": "A" } ], "title": "Morning Scrape", "type": "stat" }, { "datasource": { "type": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "0": { "color": "red", "index": 0, "text": "Not Running" }, "1": { "color": "green", "index": 1, "text": "Running" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, "id": 2, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "textMode": "value" }, "targets": [ { "datasource": { "type": "prometheus" }, "expr": "count(scheduler_runs_total{schedule_type=~\"pre_race.*\"}) > 0", "refId": "A" } ], "title": "Pre-Race", "type": "stat" }, { "datasource": { "type": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "0": { "color": "red", "index": 0, "text": "Not Running" }, "1": { "color": "green", "index": 1, "text": "Running" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, "id": 3, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "textMode": "value" }, "targets": [ { "datasource": { "type": "prometheus" }, "expr": "count(scheduler_runs_total{schedule_type=~\"post_race.*\"}) > 0", "refId": "A" } ], "title": "Post-Race", "type": "stat" }, { "datasource": { "type": "grafana-postgresql-datasource", "uid": "PCC52D03280B7034C" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "dateTimeAsIso" }, "overrides": [] }, "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, "id": 4, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "textMode": "value" }, "targets": [ { "datasource": { "type": "grafana-postgresql-datasource", "uid": "PCC52D03280B7034C" }, "format": "table", "rawSql": "SELECT EXTRACT(EPOCH FROM MAX(\"startedAt\")) * 1000 as time FROM job_runs WHERE status = 'success' LIMIT 1", "refId": "A" } ], "title": "Last Successful Run", "type": "stat" }, { "datasource": { "type": "grafana-postgresql-datasource", "uid": "PCC52D03280B7034C" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 5 } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, "id": 5, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "textMode": "value" }, "targets": [ { "datasource": { "type": "grafana-postgresql-datasource", "uid": "PCC52D03280B7034C" }, "format": "table", "rawSql": "SELECT COUNT(*) as value FROM job_runs WHERE status = 'failed' AND \"startedAt\" > NOW() - INTERVAL '1 hour'", "refId": "A" } ], "title": "Failed Jobs (1h)", "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 101, "panels": [], "title": "Scheduler Execution Metrics", "type": "row" }, { "datasource": { "type": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "id": 6, "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus" }, "expr": "rate(scheduler_runs_total{status=\"success\"}[5m]) * 60", "legendFormat": "{{schedule_type}} - Success", "refId": "A" }, { "datasource": { "type": "prometheus" }, "expr": "rate(scheduler_runs_total{status=\"failed\"}[5m]) * 60", "legendFormat": "{{schedule_type}} - Failed", "refId": "B" } ], "title": "Scheduler Execution Rate (per minute)", "type": "timeseries" }, { "datasource": { "type": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "id": 7, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus" }, "expr": "histogram_quantile(0.95, rate(scheduler_duration_seconds_bucket[5m]))", "legendFormat": "{{schedule_type}} - p95", "refId": "A" }, { "datasource": { "type": "prometheus" }, "expr": "histogram_quantile(0.50, rate(scheduler_duration_seconds_bucket[5m]))", "legendFormat": "{{schedule_type}} - p50", "refId": "B" } ], "title": "Scheduler Duration", "type": "timeseries" }, { "datasource": { "type": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, "id": 8, "options": { "legend": { "calcs": [ "sum" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus" }, "expr": "increase(scheduler_itemsProcessed_total[5m])", "legendFormat": "{{schedule_type}}", "refId": "A" } ], "title": "Items Processed (5m windows)", "type": "timeseries" }, { "datasource": { "type": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "tooltip": false, "viz": false, "legend": false } }, "mappings": [], "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 }, "id": 9, "options": { "displayLabels": [ "percent" ], "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": [ "value" ] }, "pieType": "donut", "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus" }, "expr": "sum by (status) (increase(scheduler_runs_total[1h]))", "legendFormat": "{{status}}", "refId": "A" } ], "title": "Job Status Distribution (1h)", "type": "piechart" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, "id": 102, "panels": [], "title": "Race Data Status", "type": "row" }, { "datasource": { "type": "grafana-postgresql-datasource", "uid": "PCC52D03280B7034C" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 6, "x": 0, "y": 23 }, "id": 10, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "textMode": "value" }, "targets": [ { "datasource": { "type": "grafana-postgresql-datasource", "uid": "PCC52D03280B7034C" }, "format": "table", "rawSql": "SELECT COUNT(*) as value FROM races WHERE \"startTime\"::date = CURRENT_DATE", "refId": "A" } ], "title": "Races Today", "type": "stat" }, { "datasource": { "type": "grafana-postgresql-datasource", "uid": "PCC52D03280B7034C" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 6, "x": 6, "y": 23 }, "id": 11, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "textMode": "value" }, "targets": [ { "datasource": { "type": "grafana-postgresql-datasource", "uid": "PCC52D03280B7034C" }, "format": "table", "rawSql": "SELECT COUNT(*) as value FROM meetings WHERE date = CURRENT_DATE", "refId": "A" } ], "title": "Meetings Today", "type": "stat" }, { "datasource": { "type": "grafana-postgresql-datasource", "uid": "PCC52D03280B7034C" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 60 }, { "color": "red", "value": 120 } ] }, "unit": "m" }, "overrides": [] }, "gridPos": { "h": 4, "w": 6, "x": 12, "y": 23 }, "id": 12, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "textMode": "value" }, "targets": [ { "datasource": { "type": "grafana-postgresql-datasource", "uid": "PCC52D03280B7034C" }, "format": "table", "rawSql": "SELECT EXTRACT(EPOCH FROM (NOW() - MAX(\"updatedAt\"))) / 60 as value FROM races WHERE \"startTime\"::date = CURRENT_DATE", "refId": "A" } ], "title": "Minutes Since Last Update", "type": "stat" }, { "datasource": { "type": "grafana-postgresql-datasource", "uid": "PCC52D03280B7034C" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "dateTimeAsIso" }, "overrides": [] }, "gridPos": { "h": 4, "w": 6, "x": 18, "y": 23 }, "id": 13, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "textMode": "value" }, "targets": [ { "datasource": { "type": "grafana-postgresql-datasource", "uid": "PCC52D03280B7034C" }, "format": "table", "rawSql": "SELECT EXTRACT(EPOCH FROM MIN(\"startTime\")) * 1000 as time FROM races WHERE \"startTime\"::date = CURRENT_DATE AND \"startTime\" > NOW()", "refId": "A" } ], "title": "Next Race Start", "type": "stat" }, { "datasource": { "type": "grafana-postgresql-datasource", "uid": "PCC52D03280B7034C" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "start_time" }, "properties": [ { "id": "unit", "value": "dateTimeAsIso" } ] }, { "matcher": { "id": "byName", "options": "minutes_until" }, "properties": [ { "id": "unit", "value": "m" }, { "id": "custom.cellOptions", "value": { "type": "color-text" } }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 60 }, { "color": "orange", "value": 15 }, { "color": "red", "value": 5 } ] } } ] } ] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 27 }, "id": 14, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": false, "displayName": "start_time" } ] }, "targets": [ { "datasource": { "type": "grafana-postgresql-datasource", "uid": "PCC52D03280B7034C" }, "format": "table", "rawSql": "SELECT \n m.name as meeting,\n r.\"raceNumber\",\n r.name as race_name,\n r.\"startTime\",\n ROUND(EXTRACT(EPOCH FROM (r.\"startTime\" - NOW())) / 60) as minutes_until,\n r.status,\n r.\"updatedAt\"\nFROM races r\nJOIN meetings m ON r.\"meetingId\" = m.id\nWHERE r.\"startTime\"::date = CURRENT_DATE\nAND r.\"startTime\" > NOW() - INTERVAL '5 minutes'\nORDER BY r.\"startTime\"\nLIMIT 20", "refId": "A" } ], "title": "Upcoming Races", "type": "table" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 35 }, "id": 103, "panels": [], "title": "Job Run History", "type": "row" }, { "datasource": { "type": "grafana-postgresql-datasource", "uid": "PCC52D03280B7034C" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "status" }, "properties": [ { "id": "custom.cellOptions", "value": { "type": "color-background" } }, { "id": "mappings", "value": [ { "options": { "success": { "color": "green", "index": 0 }, "failed": { "color": "red", "index": 1 }, "partial": { "color": "yellow", "index": 2 }, "running": { "color": "blue", "index": 3 } }, "type": "value" } ] } ] }, { "matcher": { "id": "byName", "options": "startedAt" }, "properties": [ { "id": "unit", "value": "dateTimeAsIso" } ] }, { "matcher": { "id": "byName", "options": "duration_sec" }, "properties": [ { "id": "unit", "value": "s" } ] } ] }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 36 }, "id": 15, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "startedAt" } ] }, "targets": [ { "datasource": { "type": "grafana-postgresql-datasource", "uid": "PCC52D03280B7034C" }, "format": "table", "rawSql": "SELECT \n \"jobType\",\n status,\n \"startedAt\",\n ROUND(\"durationMs\" / 1000.0, 2) as duration_sec,\n \"itemsProcessed\",\n \"itemsFailed\",\n \"retryCount\",\n SUBSTRING(\"errorMessage\", 1, 100) as error\nFROM job_runs\nORDER BY \"startedAt\" DESC\nLIMIT 50", "refId": "A" } ], "title": "Recent Job Runs", "type": "table" } ], "refresh": "10s", "schemaVersion": 38, "style": "dark", "tags": [ "scheduler", "monitoring", "racing" ], "templating": { "list": [] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "Pacific/Auckland", "title": "TAB API - Scheduler Monitoring", "uid": "scheduler-monitoring", "version": 1, "weekStart": "" }