From 32b22f238db0c9517f117c0d4f642d3d1bb82e77 Mon Sep 17 00:00:00 2001 From: Deezzir Date: Fri, 26 Sep 2025 20:10:47 -0400 Subject: [PATCH 01/10] feat: enable `pgbackrest_exporter` by default --- src/charm.py | 18 +++++++++++++++++- src/constants.py | 1 + 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index e89ae10aa8..e6d8d914a3 100755 --- a/src/charm.py +++ b/src/charm.py @@ -104,6 +104,7 @@ MONITORING_USER, PATRONI_PASSWORD_KEY, PEER, + PGBACKREST_METRICS_PORT, PLUGIN_OVERRIDES, POSTGRES_LOG_FILES, REPLICATION_PASSWORD_KEY, @@ -213,6 +214,7 @@ def __init__(self, *args): self.pgbackrest_server_service = "pgbackrest server" self.ldap_sync_service = "ldap-sync" self.metrics_service = "metrics_server" + self.pgbackrest_metrics_service = "pgbackrest_metrics_exporter" self._unit = self.model.unit.name self._name = self.model.app.name self._namespace = self.model.name @@ -299,6 +301,7 @@ def _generate_metrics_jobs(self, enable_tls: bool) -> dict: """Generate spec for Prometheus scraping.""" return [ {"static_configs": [{"targets": [f"*:{METRICS_PORT}"]}]}, + {"static_configs": [{"targets": [f"*:{PGBACKREST_METRICS_PORT}"]}]}, { "static_configs": [{"targets": ["*:8008"]}], "scheme": "https" if enable_tls else "http", @@ -1805,7 +1808,7 @@ def _generate_ldap_service(self) -> dict: } def _generate_metrics_service(self) -> dict: - """Generate the metrics service definition.""" + """Generate the postgresql metrics service definition.""" return { "override": "replace", "summary": "postgresql metrics exporter", @@ -1827,6 +1830,18 @@ def _generate_metrics_service(self) -> dict: }, } + def _generate_pgbackrest_metrics_service(self) -> dict: + """Generate the pgbackrest metrics service definition.""" + return { + "override": "replace", + "summary": "pgbackrest metrics exporter", + "command": "/usr/bin/pgbackrest_exporter", + "startup": "enabled", + "after": [self.postgresql_service], + "user": WORKLOAD_OS_USER, + "group": WORKLOAD_OS_GROUP, + } + def _postgresql_layer(self) -> Layer: """Returns a Pebble configuration layer for PostgreSQL.""" pod_name = self._unit_name_to_pod_name(self._unit) @@ -1871,6 +1886,7 @@ def _postgresql_layer(self) -> Layer: "startup": "disabled", }, self.metrics_service: self._generate_metrics_service(), + self.pgbackrest_metrics_service: self._generate_pgbackrest_metrics_service(), self.rotate_logs_service: { "override": "replace", "summary": "rotate logs", diff --git a/src/constants.py b/src/constants.py index ac5abdd39a..8a293eeea8 100644 --- a/src/constants.py +++ b/src/constants.py @@ -17,6 +17,7 @@ WORKLOAD_OS_GROUP = "postgres" WORKLOAD_OS_USER = "postgres" METRICS_PORT = "9187" +PGBACKREST_METRICS_PORT = "9854" POSTGRESQL_DATA_PATH = "/var/lib/postgresql/data/pgdata" POSTGRESQL_LOGS_PATH = "/var/log/postgresql" POSTGRESQL_LOGS_PATTERN = "postgresql*.log" From 9a8391489eb292e8b3be859a47f79f078b1db1c0 Mon Sep 17 00:00:00 2001 From: Deezzir Date: Fri, 26 Sep 2025 20:11:41 -0400 Subject: [PATCH 02/10] feat: `pgbackrest` Grafana dashboard --- .../postgresql-pgbackrest-metrics.json | 7121 +++++++++++++++++ 1 file changed, 7121 insertions(+) create mode 100644 src/grafana_dashboards/postgresql-pgbackrest-metrics.json diff --git a/src/grafana_dashboards/postgresql-pgbackrest-metrics.json b/src/grafana_dashboards/postgresql-pgbackrest-metrics.json new file mode 100644 index 0000000000..3b71fb4f80 --- /dev/null +++ b/src/grafana_dashboards/postgresql-pgbackrest-metrics.json @@ -0,0 +1,7121 @@ +{ + "__inputs": [], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.5.1" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "description": "Dashboard for pgBackRest metrics collected by the pgbackrest_exporter.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 46, + "panels": [], + "title": "General info", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Count of selected stanzas.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 57, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "sum(count by (stanza)(pgbackrest_stanza_status{stanza=~\"$stanza\"}))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Stanzas CNT", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Count of all backups for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 52, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "sum(count by (stanza) (pgbackrest_backup_info{stanza=~\"$stanza\", repo_key=~\"$repo_key\"}))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Backups CNT", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Number of full backups for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 55, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "sum(count by (stanza) (pgbackrest_backup_info{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=\"full\"}))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Full backups CNT", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Number of diff backups for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 58, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "sum(count by (stanza) (pgbackrest_backup_info{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=\"diff\"}))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Diff backups CNT", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Number of incr backups for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 59, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "sum(count by (stanza) (pgbackrest_backup_info{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=\"incr\"}))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Incr backups CNT", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Number of backups with errors for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.5 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 60, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "sum(pgbackrest_backup_error_status{stanza=~\"$stanza\", repo_key=~\"$repo_key\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Backups with errors CNT", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Current stanza and active operations statuses for selected stanzas.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "text": "ok" + }, + "1": { + "text": "missing stanza path" + }, + "2": { + "text": "no valid backups" + }, + "3": { + "text": "missing stanza data" + }, + "4": { + "text": "different across repos" + }, + "5": { + "text": "database mismatch across repos" + }, + "6": { + "text": "requested backup not found" + }, + "99": { + "text": "other" + } + }, + "type": "value" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.5 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "custom.width", + "value": 250 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Since last full" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "unit", + "value": "s" + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "custom.width", + "value": 135 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Since last diff" + }, + "properties": [ + { + "id": "displayName", + "value": "Since last diff" + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.width", + "value": 135 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Since last incr" + }, + "properties": [ + { + "id": "decimals", + "value": 1 + }, + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.width", + "value": 135 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Stanza" + }, + "properties": [ + { + "id": "custom.width", + "value": 250 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Lock status" + }, + "properties": [ + { + "id": "custom.width", + "value": 155 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "no active operation" + }, + "1": { + "index": 1, + "text": "backup/expire" + } + }, + "type": "value" + } + ] + }, + { + "id": "noValue", + "value": "-" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "thresholds" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + }, + { + "color": "purple", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "NaN": { + "index": 0, + "text": "no active operation" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "noValue", + "value": "-" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + }, + { + "color": "green", + "value": 0 + } + ] + } + }, + { + "id": "unit", + "value": "percent" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 30, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Stanza" + } + ] + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_stanza_status{stanza=~\"$stanza\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_since_last_completion_seconds{stanza=~\"$stanza\", backup_type=\"full\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_since_last_completion_seconds{stanza=~\"$stanza\", backup_type=\"diff\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_since_last_completion_seconds{stanza=~\"$stanza\", backup_type=\"incr\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_stanza_lock_status{stanza=~\"$stanza\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(pgbackrest_stanza_backup_compete_bytes{stanza=~\"$stanza\"}/pgbackrest_stanza_backup_total_bytes{stanza=~\"$stanza\"})*100", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "F" + } + ], + "title": "Stanza status and last backups", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "stanza", + "Value #A", + "Value #B", + "Value #C", + "Value #D", + "Value #E", + "Value #F" + ], + "pattern": "stanza" + } + } + }, + { + "id": "seriesToColumns", + "options": { + "byField": "stanza" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value #A": 4, + "Value #B": 1, + "Value #C": 2, + "Value #D": 3, + "stanza": 0 + }, + "renameByName": { + "Value #A": "Status", + "Value #B": "Since last full", + "Value #C": "Since last diff", + "Value #D": "Since last incr", + "Value #E": "Lock status", + "Value #F": "Completed", + "stanza": "Stanza" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Current repository status for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "color-text" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.5 + } + ] + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "ok" + }, + "1": { + "index": 1, + "text": "missing stanza path" + }, + "2": { + "index": 2, + "text": "no valid backups" + }, + "3": { + "index": 3, + "text": "missing stanza data" + }, + "4": { + "index": 4, + "text": "different across repos" + }, + "5": { + "index": 5, + "text": "database mismatch across repos" + }, + "6": { + "index": 6, + "text": "requested backup not found" + }, + "99": { + "index": 7, + "text": "other" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.width", + "value": 250 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Repo key" + }, + "properties": [ + { + "id": "custom.width", + "value": 95 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cipher" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Stanza" + }, + "properties": [ + { + "id": "custom.width", + "value": 250 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 15, + "x": 0, + "y": 13 + }, + "id": 32, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Stanza" + } + ] + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_repo_status{stanza=~\"$stanza\", repo_key=~\"$repo_key\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Repository status", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "cipher", + "repo_key", + "stanza", + "Value" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "stanza 2": false + }, + "indexByName": { + "Value": 3, + "cipher": 2, + "repo_key": 1, + "stanza": 0 + }, + "renameByName": { + "Value": "Status", + "Value #A": "status", + "cipher": "Cipher", + "repo_key": "Repo key", + "stanza": "Stanza" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Number of backups for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 13 + }, + "id": 72, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "count by (stanza) (pgbackrest_backup_info{stanza=~\"$stanza\", repo_key=~\"$repo_key\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Backups CNT", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Current WAL archive status for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + }, + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 1, + "text": "error" + }, + "1": { + "index": 0, + "text": "ok" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PG Version" + }, + "properties": [ + { + "id": "decimals", + "value": 1 + }, + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Stanza" + }, + "properties": [ + { + "id": "custom.width", + "value": 250 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Repo Key" + }, + "properties": [ + { + "id": "custom.width", + "value": 95 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Database ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 15, + "x": 0, + "y": 21 + }, + "id": 33, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Stanza" + } + ] + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_wal_archive_status{stanza=~\"$stanza\", repo_key=~\"$repo_key\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "WAL archive status", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "database_id", + "pg_version", + "repo_key", + "stanza", + "Value" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "stanza 2": false + }, + "indexByName": { + "Value": 4, + "database_id": 3, + "pg_version": 2, + "repo_key": 1, + "stanza": 0 + }, + "renameByName": { + "Value": "Status", + "Value #A": "status", + "database_id": "Database ID", + "pg_version": "PG Version", + "repo_key": "Repo Key", + "stanza": "Stanza" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average full uncompressed size of the database for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 21 + }, + "id": 67, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_size_bytes{stanza=~\"$stanza\", repo_key=~\"$repo_key\"})", + "legendFormat": "{{ stanza }}", + "range": true, + "refId": "A" + } + ], + "title": "Average full uncompressed size of the database ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Last backup error status and duration for selected stanzas.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Error status" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + }, + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 1, + "text": "ok" + }, + "1": { + "index": 0, + "text": "error" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Backup type" + }, + "properties": [ + { + "id": "custom.width", + "value": 130 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Stanza" + }, + "properties": [ + { + "id": "custom.width", + "value": 250 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Block incremental" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Duration" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + }, + { + "id": "unit", + "value": "s" + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 15, + "x": 0, + "y": 29 + }, + "id": 77, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Stanza" + } + ] + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_last_error_status{stanza=~\"$stanza\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_last_duration_seconds{stanza=~\"$stanza\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "title": "Last backup error status and duration", + "transformations": [ + { + "id": "concatenate", + "options": {} + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "backup_type 1", + "block_incr 1", + "Value #A", + "Value #B", + "stanza 1" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "stanza 2": false + }, + "indexByName": { + "Value #A": 3, + "Value #B": 4, + "backup_type 1": 1, + "block_incr 1": 2, + "stanza 1": 0 + }, + "renameByName": { + "Value": "Status", + "Value #A": "Error status", + "Value #B": "Duration", + "backup_type 1": "Backup type", + "block_incr 1": "Block incremental", + "database_id": "Database ID", + "pg_version": "PG Version", + "repo_key": "Repo Key", + "stanza": "Stanza", + "stanza 1": "Stanza" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Total size occupied by backups in the repository for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 29 + }, + "id": 69, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "sum by (stanza) (pgbackrest_backup_repo_delta_bytes{stanza=~\"$stanza\", repo_key=~\"$repo_key\"})", + "legendFormat": "{{ stanza }}", + "range": true, + "refId": "A" + } + ], + "title": "Total size occupied by backups in the repository", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 76, + "panels": [], + "title": "Last backups info", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average last full backup duration for selected stanzas.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 38 + }, + "id": 78, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_last_duration_seconds{stanza=~\"$stanza\", backup_type=\"full\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average last full backup duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average last differential backup duration for selected stanzas.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 38 + }, + "id": 79, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_last_duration_seconds{stanza=~\"$stanza\", backup_type=\"diff\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average last diff backup duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average last incremental backup duration for selected stanzas.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 38 + }, + "id": 73, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_last_duration_seconds{stanza=~\"$stanza\", backup_type=\"incr\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average last incr backup duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average size occupied by last full backup in the repository for selected stanzas.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 46 + }, + "id": 81, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_last_repo_delta_bytes{stanza=~\"$stanza\", backup_type=~\"full\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average size of last full backup in the repository", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average size occupied by last differential backup in the repository for selected stanzas.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 46 + }, + "id": 82, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_last_repo_delta_bytes{stanza=~\"$stanza\", backup_type=\"diff\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average size of last diff backup in the repository", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average size occupied by last incremental backup in the repository for selected stanzas.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 46 + }, + "id": 74, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_last_repo_delta_bytes{stanza=~\"$stanza\", backup_type=\"incr\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average size of last incr backup in the repository", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "For selected stanzas.\n\nBackup sizes:\n* **databaze size** - full uncompressed size of the database;\n* **database backup size** - the amount of data in the database to actually back up.\n\nThese will be the same for full backups.\n\nRepo sizes:\n* **repo backup set size** - includes all the files from this backup and any referenced backups in the repository that are required to restore the database from this backup;\n* **repo backup size** - includes only the files in this backup.\n\nThese will also be the same for full backups. Repository sizes reflect compressed file sizes if compression is enabled.\n\nSee https://pgbackrest.org/command.html#command-info\n\nFor block incremental backups **repo backup set size** is zero.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "fixed" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Database ID" + }, + "properties": [ + { + "id": "unit", + "value": "string" + }, + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Repo Key" + }, + "properties": [ + { + "id": "unit", + "value": "string" + }, + { + "id": "custom.width", + "value": 95 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Stanza" + }, + "properties": [ + { + "id": "custom.width", + "value": 250 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Backup type" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Backup name" + }, + "properties": [ + { + "id": "custom.width", + "value": 290 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Database backup size" + }, + "properties": [ + { + "id": "custom.width", + "value": 176 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Repo backup set size" + }, + "properties": [ + { + "id": "custom.width", + "value": 170 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Database size" + }, + "properties": [ + { + "id": "custom.width", + "value": 135 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Repo backup size" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Block incremental" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Repo block incr map size" + }, + "properties": [ + { + "id": "custom.width", + "value": 200 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Repo block incr delta map size" + }, + "properties": [ + { + "id": "custom.width", + "value": 230 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 85, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Stanza" + } + ] + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_last_size_bytes{stanza=~\"$stanza\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_last_delta_bytes{stanza=~\"$stanza\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_last_repo_size_bytes{stanza=~\"$stanza\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_last_repo_delta_bytes{stanza=~\"$stanza\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_last_repo_size_map_bytes{stanza=~\"$stanza\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_last_repo_delta_map_bytes{stanza=~\"$stanza\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "F" + } + ], + "title": "Last backup size", + "transformations": [ + { + "id": "concatenate", + "options": { + "frameNameLabel": "frame", + "frameNameMode": "field" + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "backup_type 1", + "block_incr 1", + "stanza 1", + "Value #A", + "Value #B", + "Value #C", + "Value #D", + "Value #F", + "Value #E" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value #A": 3, + "Value #B": 4, + "Value #C": 5, + "Value #D": 6, + "Value #E": 7, + "Value #F": 8, + "backup_type 1": 1, + "block_incr 1": 2, + "stanza 1": 0 + }, + "renameByName": { + "Value #A": "Database size", + "Value #B": "Database backup size", + "Value #C": "Repo backup set size", + "Value #D": "Repo backup size", + "Value #E": "Repo block incr map size", + "Value #F": "Repo block incr delta map size", + "backup_name 1": "Backup name", + "backup_type 1": "Backup type", + "block_incr 1": "Block incremental", + "database_id 1": "Database ID", + "repo_key 1": "Repo Key", + "stanza 1": "Stanza" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Number of annotations in the last backups for selected stanzas.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Annotations in last full" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + }, + { + "id": "noValue", + "value": "-" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Annotations in last diff" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + }, + { + "id": "noValue", + "value": "-" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Annotations in last incr" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + }, + { + "id": "noValue", + "value": "-" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Stanza" + }, + "properties": [ + { + "id": "custom.width", + "value": 250 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 15, + "x": 0, + "y": 63 + }, + "id": 86, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Stanza" + } + ] + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_last_annotations{stanza=~\"$stanza\", backup_type=\"full\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_last_annotations{stanza=~\"$stanza\", backup_type=\"diff\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_last_annotations{stanza=~\"$stanza\", backup_type=\"incr\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + } + ], + "title": "Number of annotations in the last backups", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "stanza", + "Value #A", + "Value #B", + "Value #C", + "Value #D" + ], + "pattern": "stanza" + } + } + }, + { + "id": "seriesToColumns", + "options": { + "byField": "stanza" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value #A": 4, + "Value #B": 1, + "Value #C": 2, + "Value #D": 3, + "stanza": 0 + }, + "renameByName": { + "Value #A": "Annotations in last incr", + "Value #B": "Annotations in last full", + "Value #C": "Annotations in last diff", + "Value #D": "Since last incr", + "stanza": "Stanza" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average number of annotations in the last backup for selected stanzas.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 63 + }, + "id": 87, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_last_annotations{stanza=~\"$stanza\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average number of annotations in the last backup", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Number of databases in the last backups for selected stanzas.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "DBs in last full" + }, + "properties": [ + { + "id": "custom.width", + "value": 135 + }, + { + "id": "noValue", + "value": "-" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "DBs in last diff" + }, + "properties": [ + { + "id": "custom.width", + "value": 135 + }, + { + "id": "noValue", + "value": "-" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "DBs in last incr" + }, + "properties": [ + { + "id": "custom.width", + "value": 135 + }, + { + "id": "noValue", + "value": "-" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Stanza" + }, + "properties": [ + { + "id": "custom.width", + "value": 250 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 15, + "x": 0, + "y": 71 + }, + "id": 50, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Stanza" + } + ] + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_last_databases{stanza=~\"$stanza\", backup_type=\"full\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_last_databases{stanza=~\"$stanza\", backup_type=\"diff\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_last_databases{stanza=~\"$stanza\", backup_type=\"incr\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + } + ], + "title": "Number of databases in the last backups", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "stanza", + "Value #A", + "Value #B", + "Value #C", + "Value #D" + ], + "pattern": "stanza" + } + } + }, + { + "id": "seriesToColumns", + "options": { + "byField": "stanza" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value #A": 4, + "Value #B": 1, + "Value #C": 2, + "Value #D": 3, + "stanza": 0 + }, + "renameByName": { + "Value #A": "DBs in last incr", + "Value #B": "DBs in last full", + "Value #C": "DBs in last diff", + "Value #D": "Since last incr", + "stanza": "Stanza" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average number of databases in the last backup for selected stanzas.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 71 + }, + "id": 61, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_last_databases{stanza=~\"$stanza\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average number of databases in the last backup", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 79 + }, + "id": 65, + "panels": [], + "title": "General backups info", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average full backup duration for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 80 + }, + "id": 54, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_duration_seconds{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=\"full\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average full backup duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average differential backup duration for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 80 + }, + "id": 62, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_duration_seconds{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=\"diff\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average diff backup duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average incremental backup duration for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 80 + }, + "id": 80, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_duration_seconds{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=\"incr\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average incr backup duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average size occupied by full backup in the repository for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 88 + }, + "id": 68, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_repo_delta_bytes{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=~\"full\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average size of full backup in the repository", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average size occupied by differential backup in the repository for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 88 + }, + "id": 70, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_repo_delta_bytes{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=\"diff\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average size of diff backup in the repository", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average size occupied by incremental backup in the repository for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 88 + }, + "id": 83, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_repo_delta_bytes{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=\"incr\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average size of incr backup in the repository", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average number of annotations in the full backup for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 96 + }, + "id": 88, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_annotations{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=\"full\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average number of annotations in the full backup", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average number of annotations in the diff backup for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 96 + }, + "id": 90, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_annotations{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=\"diff\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average number of annotations in the diff backup", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average number of annotations in the incr backup for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 96 + }, + "id": 91, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_annotations{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=\"incr\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average number of annotations in the incr backup", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average number of databases in the full backup for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 104 + }, + "id": 89, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_databases{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=\"full\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average number of databases in the full backup", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average number of databases in the diff backup for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 104 + }, + "id": 92, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_databases{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=\"diff\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average number of databases in the diff backup", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Average number of databases in the incr backup for selected stanzas and repos.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 104 + }, + "id": 93, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "avg by (stanza) (pgbackrest_backup_databases{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=\"incr\"})", + "legendFormat": "{{stanza}}", + "range": true, + "refId": "A" + } + ], + "title": "Average number of databases in the incr backup", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 112 + }, + "id": 48, + "panels": [], + "title": "Detailed backups info", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Backup error status for selected stanzas, repos, backup types, block incremental statuses and backup names.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Error status" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "text": "ok" + }, + "1": { + "text": "error" + } + }, + "type": "value" + } + ] + }, + { + "id": "noValue", + "value": "null" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + }, + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + { + "id": "custom.width", + "value": 135 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Backup type" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Repo key" + }, + "properties": [ + { + "id": "custom.width", + "value": 95 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Database ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Backup name" + }, + "properties": [ + { + "id": "custom.width", + "value": 290 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Stanza" + }, + "properties": [ + { + "id": "custom.width", + "value": 250 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Block Incremental" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 113 + }, + "id": 37, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Stanza" + } + ] + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "pgbackrest_backup_error_status{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=~\"$backup_type\", block_incr=~\"$block_incr\", backup_name=~\"$backup_name\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Backup error status", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "backup_name", + "backup_type", + "database_id", + "repo_key", + "stanza", + "Value", + "block_incr" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Value #A": false, + "__name__": true, + "__name__ 1": true, + "__name__ 2": true, + "backup_type 2": true, + "database_id 2": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "job": true, + "job 1": true, + "job 2": true, + "prj 1": true, + "prj 2": true, + "prometheus 1": true, + "prometheus 2": true, + "repo_key 2": true, + "service 1": true, + "service 2": true, + "stanza 2": true + }, + "indexByName": { + "Value": 6, + "backup_name": 4, + "backup_type": 2, + "block_incr": 3, + "database_id": 5, + "repo_key": 1, + "stanza": 0 + }, + "renameByName": { + "Value": "Error status", + "Value #A": "Error status", + "Value #B": "Databases CNT", + "backrest_ver": "pgBackRest version", + "backup_name": "Backup name", + "backup_type": "Backup type", + "backup_type 1": "Backup type", + "block_incr": "Block Incremental", + "database_id": "Database ID", + "database_id 1": "Database ID", + "lsn_start": "LSN start", + "lsn_stop": "LSN stop", + "pg_version": "PG version", + "prior": "Prior", + "repo_key": "Repo key", + "repo_key 1": "Repo key", + "stanza": "Stanza", + "stanza 1": "Stanza", + "wal_start": "WAL start", + "wal_stop": "WAL stop" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Backup duration for selected stanzas, repos, backup types, block incremental statuses and backup names.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [ + { + "options": { + "": { + "text": "" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Duration" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.width", + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Stanza" + }, + "properties": [ + { + "id": "custom.width", + "value": 250 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Repo key" + }, + "properties": [ + { + "id": "custom.width", + "value": 95 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Backup name" + }, + "properties": [ + { + "id": "custom.width", + "value": 290 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Backup type" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Database ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Start time" + }, + "properties": [ + { + "id": "custom.width", + "value": 165 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Stop time" + }, + "properties": [ + { + "id": "custom.width", + "value": 165 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Block incremental" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 121 + }, + "id": 35, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Stanza" + } + ] + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "expr": "pgbackrest_backup_duration_seconds{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=~\"$backup_type\", backup_name=~\"$backup_name\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Backup duration", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "backup_name", + "backup_type", + "database_id", + "repo_key", + "stanza", + "start_time", + "stop_time", + "Value", + "block_incr" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value": 6, + "backup_name": 4, + "backup_type": 2, + "block_incr": 3, + "database_id": 5, + "repo_key": 1, + "stanza": 0, + "start_time": 7, + "stop_time": 8 + }, + "renameByName": { + "Value": "Duration", + "backup_name": "Backup name", + "backup_type": "Backup type", + "block_incr": "Block incremental", + "database_id": "Database ID", + "repo_key": "Repo key", + "stanza": "Stanza", + "start_time": "Start time", + "stop_time": "Stop time" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "For selected stanzas, repos, backup types, block incremental statuses and backup names.\n\nBackup sizes:\n* **databaze size** - full uncompressed size of the database;\n* **database backup size** - the amount of data in the database to actually back up.\n\nThese will be the same for full backups.\n\nRepo sizes:\n* **repo backup set size** - includes all the files from this backup and any referenced backups in the repository that are required to restore the database from this backup;\n* **repo backup size** - includes only the files in this backup.\n\nThese will also be the same for full backups. Repository sizes reflect compressed file sizes if compression is enabled.\n\nSee https://pgbackrest.org/command.html#command-info\n\nFor block incremental backups **repo backup set size** is zero.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "fixed" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Database ID" + }, + "properties": [ + { + "id": "unit", + "value": "string" + }, + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Repo Key" + }, + "properties": [ + { + "id": "unit", + "value": "string" + }, + { + "id": "custom.width", + "value": 95 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Stanza" + }, + "properties": [ + { + "id": "custom.width", + "value": 250 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Backup type" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Backup name" + }, + "properties": [ + { + "id": "custom.width", + "value": 290 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Database backup size" + }, + "properties": [ + { + "id": "custom.width", + "value": 176 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Repo backup set size" + }, + "properties": [ + { + "id": "custom.width", + "value": 170 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Database size" + }, + "properties": [ + { + "id": "custom.width", + "value": 135 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Repo backup size" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Block incremental" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Repo block incr map size" + }, + "properties": [ + { + "id": "custom.width", + "value": 200 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Repo block incr delta map size" + }, + "properties": [ + { + "id": "custom.width", + "value": 230 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 130 + }, + "id": 39, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Stanza" + } + ] + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_size_bytes{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=~\"$backup_type\", block_incr=~\"$block_incr\", backup_name=~\"$backup_name\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_delta_bytes{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=~\"$backup_type\", block_incr=~\"$block_incr\", backup_name=~\"$backup_name\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_repo_size_bytes{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=~\"$backup_type\", block_incr=~\"$block_incr\", backup_name=~\"$backup_name\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_repo_delta_bytes{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=~\"$backup_type\", block_incr=~\"$block_incr\", backup_name=~\"$backup_name\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_repo_size_map_bytes{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=~\"$backup_type\", block_incr=~\"$block_incr\", backup_name=~\"$backup_name\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_repo_delta_map_bytes{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=~\"$backup_type\", block_incr=~\"$block_incr\", backup_name=~\"$backup_name\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "F" + } + ], + "title": "Backup size", + "transformations": [ + { + "id": "concatenate", + "options": { + "frameNameLabel": "frame", + "frameNameMode": "field" + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "backup_name 1", + "backup_type 1", + "database_id 1", + "repo_key 1", + "stanza 1", + "Value #A", + "Value #B", + "Value #C", + "Value #D", + "Value #E", + "Value #F", + "block_incr 1" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value #A": 6, + "Value #B": 7, + "Value #C": 8, + "Value #D": 9, + "Value #E": 10, + "Value #F": 11, + "backup_name 1": 4, + "backup_type 1": 2, + "block_incr 1": 3, + "database_id 1": 5, + "repo_key 1": 1, + "stanza 1": 0 + }, + "renameByName": { + "Value #A": "Database size", + "Value #B": "Database backup size", + "Value #C": "Repo backup set size", + "Value #D": "Repo backup size", + "Value #E": "Repo block incr map size", + "Value #F": "Repo block incr delta map size", + "backup_name 1": "Backup name", + "backup_type 1": "Backup type", + "block_incr 1": "Block incremental", + "database_id 1": "Database ID", + "repo_key 1": "Repo Key", + "stanza 1": "Stanza" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Backup detail info for selected stanzas, repos, backup types, block incremental statuses and backup names.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "LSN start" + }, + "properties": [ + { + "id": "noValue", + "value": "-" + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "LSN stop" + }, + "properties": [ + { + "id": "noValue", + "value": "-" + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Stanza" + }, + "properties": [ + { + "id": "custom.width", + "value": 250 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Repo key" + }, + "properties": [ + { + "id": "custom.width", + "value": 95 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Backup type" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Backup name" + }, + "properties": [ + { + "id": "custom.width", + "value": 290 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Database ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "WAL start" + }, + "properties": [ + { + "id": "custom.width", + "value": 225 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "WAL stop" + }, + "properties": [ + { + "id": "custom.width", + "value": 225 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Prior" + }, + "properties": [ + { + "id": "custom.width", + "value": 290 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PG ver." + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + }, + { + "id": "decimals", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "pgBackRest ver." + }, + "properties": [ + { + "id": "custom.width", + "value": 140 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Block Incremental" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 139 + }, + "id": 49, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Stanza" + } + ] + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "expr": "pgbackrest_backup_info{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=~\"$backup_type\", block_incr=~\"$block_incr\", backup_name=~\"$backup_name\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Backup detail info", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "backrest_ver", + "backup_name", + "backup_type", + "database_id", + "lsn_start", + "lsn_stop", + "pg_version", + "repo_key", + "stanza", + "wal_start", + "wal_stop", + "prior", + "Value", + "block_incr" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Value": true, + "Value #A": true, + "__name__": true, + "__name__ 1": true, + "__name__ 2": true, + "backup_type 2": true, + "database_id 2": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "job": true, + "job 1": true, + "job 2": true, + "prj 1": true, + "prj 2": true, + "prometheus 1": true, + "prometheus 2": true, + "repo_key 2": true, + "service 1": true, + "service 2": true, + "stanza 2": true + }, + "indexByName": { + "Value": 13, + "backrest_ver": 12, + "backup_name": 4, + "backup_type": 2, + "block_incr": 3, + "database_id": 5, + "lsn_start": 6, + "lsn_stop": 7, + "pg_version": 11, + "prior": 10, + "repo_key": 1, + "stanza": 0, + "wal_start": 8, + "wal_stop": 9 + }, + "renameByName": { + "Value #B": "Error", + "backrest_ver": "pgBackRest ver.", + "backup_name": "Backup name", + "backup_type": "Backup type", + "backup_type 1": "Backup type", + "block_incr": "Block Incremental", + "database_id": "Database ID", + "database_id 1": "Database ID", + "lsn_start": "LSN start", + "lsn_stop": "LSN stop", + "pg_version": "PG ver.", + "prior": "Prior", + "repo_key": "Repo key", + "repo_key 1": "Repo key", + "stanza": "Stanza", + "stanza 1": "Stanza", + "wal_start": "WAL start", + "wal_stop": "WAL stop" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Number of annotations in the backup for selected stanzas, repos, backup types, block incremental statuses and backup names.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Annotations in last full" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Annotations in last diff" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Annotations in last incr" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Stanza" + }, + "properties": [ + { + "id": "custom.width", + "value": 250 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Repo key" + }, + "properties": [ + { + "id": "custom.width", + "value": 95 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Backup type" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Block incremental" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Backup name" + }, + "properties": [ + { + "id": "custom.width", + "value": 290 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Annotations" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + }, + { + "id": "noValue", + "value": "-" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Database ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 147 + }, + "id": 94, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Stanza" + } + ] + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_annotations{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=~\"$backup_type\", block_incr=~\"$block_incr\", backup_name=~\"$backup_name\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Number of annotations in the backup", + "transformations": [ + { + "id": "concatenate", + "options": {} + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "stanza", + "repo_key", + "Value", + "backup_name", + "backup_type", + "block_incr", + "database_id" + ], + "pattern": "stanza" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value": 6, + "backup_name": 4, + "backup_type": 2, + "block_incr": 3, + "database_id": 5, + "repo_key": 1, + "stanza": 0 + }, + "renameByName": { + "Value": "Annotations", + "Value #A": "Annotations", + "Value #B": "DBs ", + "Value #C": "Annotations in last diff", + "Value #D": "Since last incr", + "backup_name": "Backup name", + "backup_name 1": "Backup name", + "backup_type": "Backup type", + "backup_type 1": "Backup type", + "block_incr": "Block incremental", + "block_incr 1": "Block incremental", + "database_id": "Database ID", + "database_id 1": "Database ID", + "repo_key": "Repo key", + "repo_key 1": "Repo key", + "stanza": "Stanza", + "stanza 1": "Stanza" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Number of databases in the backup for selected stanzas, repos, backup types, block incremental statuses and backup names.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Annotations in last full" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Annotations in last diff" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Annotations in last incr" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Stanza" + }, + "properties": [ + { + "id": "custom.width", + "value": 250 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Repo key" + }, + "properties": [ + { + "id": "custom.width", + "value": 95 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Backup type" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Block incremental" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Backup name" + }, + "properties": [ + { + "id": "custom.width", + "value": 290 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "DBs" + }, + "properties": [ + { + "id": "custom.width", + "value": 95 + }, + { + "id": "noValue", + "value": "-" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Database ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 156 + }, + "id": 95, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Stanza" + } + ] + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_backup_databases{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=~\"$backup_type\", block_incr=~\"$block_incr\", backup_name=~\"$backup_name\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Number of databases in the backup", + "transformations": [ + { + "id": "concatenate", + "options": {} + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "stanza", + "Value", + "repo_key", + "backup_name", + "backup_type", + "block_incr", + "database_id" + ], + "pattern": "stanza" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value": 6, + "backup_name": 4, + "backup_type": 2, + "block_incr": 3, + "database_id": 5, + "repo_key": 1, + "stanza": 0 + }, + "renameByName": { + "Value": "DBs", + "Value #A": "Annotations", + "Value #B": "DBs ", + "Value #C": "Annotations in last diff", + "Value #D": "Since last incr", + "backup_name": "Backup name", + "backup_name 1": "Backup name", + "backup_type": "Backup type", + "backup_type 1": "Backup type", + "block_incr": "Block incremental", + "block_incr 1": "Block incremental", + "database_id": "Database ID", + "database_id 1": "Database ID", + "repo_key": "Repo key", + "repo_key 1": "Repo key", + "stanza": "Stanza", + "stanza 1": "Stanza" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 165 + }, + "id": 97, + "panels": [], + "title": "pgBackRest Exporter info", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Information about exporter build parameters and status of getting data from pgBackRest.\n\nWhen the information is collected for all available stanzas, the **stanza for metrics** is **all-stanzas**.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Instance" + }, + "properties": [ + { + "id": "custom.width", + "value": 250 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Job" + }, + "properties": [ + { + "id": "custom.width", + "value": 200 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Exporter version" + }, + "properties": [ + { + "id": "custom.width", + "value": 135 + }, + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Stanza for metrics" + }, + "properties": [ + { + "id": "custom.width", + "value": 145 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Get data status" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + }, + { + "id": "noValue", + "value": "-" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "error" + }, + "1": { + "index": 1, + "text": "ok" + } + }, + "type": "value" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + }, + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Build goarch" + }, + "properties": [ + { + "id": "custom.width", + "value": 105 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Build goos" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Build gover" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Build branch" + }, + "properties": [ + { + "id": "custom.width", + "value": 105 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Build rev" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Build tags" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 166 + }, + "id": 99, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_exporter_build_info", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pgbackrest_exporter_status", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "B" + } + ], + "title": "Exporter status and build info", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "instance", + "mode": "outer" + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "instance", + "goarch", + "goos", + "goversion", + "job 1", + "revision", + "tags", + "version", + "stanza", + "Value #B", + "branch" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "Value #B": 3, + "branch": 8, + "goarch": 5, + "goos": 6, + "goversion": 7, + "instance": 0, + "job 1": 1, + "revision": 9, + "stanza": 2, + "tags": 10, + "version": 4 + }, + "renameByName": { + "Value #B": "Get data status", + "branch": "Build branch", + "goarch": "Build goarch", + "goos": "Build goos", + "goversion": "Build gover", + "instance": "Instance", + "job 1": "Job", + "revision": "Build rev", + "stanza": "Stanza for metrics", + "tags": "Build tags", + "version": "Exporter version" + } + } + } + ], + "type": "table" + } + ], + "refresh": "5m", + "schemaVersion": 40, + "templating": { + "list": [ + { + "current": {}, + "includeAll": false, + "label": "Datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "definition": "label_values(pgbackrest_repo_status, stanza)", + "includeAll": true, + "label": "Stanza", + "multi": true, + "name": "stanza", + "options": [], + "query": { + "query": "label_values(pgbackrest_repo_status, stanza)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "definition": "label_values(pgbackrest_repo_status{stanza=~\"$stanza\"}, repo_key)", + "includeAll": true, + "label": "Repo key", + "multi": true, + "name": "repo_key", + "options": [], + "query": { + "query": "label_values(pgbackrest_repo_status{stanza=~\"$stanza\"}, repo_key)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "definition": "label_values(pgbackrest_backup_info{stanza=~\"$stanza\", repo_key=~\"$repo_key\"}, backup_type)", + "includeAll": true, + "label": "Backup type", + "multi": true, + "name": "backup_type", + "options": [], + "query": { + "query": "label_values(pgbackrest_backup_info{stanza=~\"$stanza\", repo_key=~\"$repo_key\"}, backup_type)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "definition": "label_values(pgbackrest_backup_info{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=~\"$backup_type\"}, block_incr)", + "includeAll": true, + "label": "Block incr", + "multi": true, + "name": "block_incr", + "options": [], + "query": { + "query": "label_values(pgbackrest_backup_info{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=~\"$backup_type\"}, block_incr)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "definition": "label_values(pgbackrest_backup_info{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=~\"$backup_type\",block_incr=~\"$block_incr\"}, backup_name)", + "includeAll": true, + "label": "Backup name", + "multi": true, + "name": "backup_name", + "options": [], + "query": { + "query": "label_values(pgbackrest_backup_info{stanza=~\"$stanza\", repo_key=~\"$repo_key\", backup_type=~\"$backup_type\",block_incr=~\"$block_incr\"}, backup_name)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "PgBackrest K8s", + "tags": [ + "postgres", + "pgbackrest", + "backups", + "k8s" + ], + "uid": "Wyf8STx7z", + "version": 2, + "weekStart": "", + "gnetId": 17709 +} From e2ecc3e9d69c129da52ffdf247e7baae586bb849 Mon Sep 17 00:00:00 2001 From: Deezzir Date: Fri, 26 Sep 2025 20:11:58 -0400 Subject: [PATCH 03/10] feat: `pgbackrest` alert rules --- .../pgbackrest_rules.yaml | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 src/prometheus_alert_rules/pgbackrest_rules.yaml diff --git a/src/prometheus_alert_rules/pgbackrest_rules.yaml b/src/prometheus_alert_rules/pgbackrest_rules.yaml new file mode 100644 index 0000000000..496688162c --- /dev/null +++ b/src/prometheus_alert_rules/pgbackrest_rules.yaml @@ -0,0 +1,69 @@ +# Reference: https://github.com/woblerr/pgbackrest_exporter/blob/master/README.md + +groups: + +- name: PgbackrestExporterK8s + + rules: + + - alert: PgBackRestBackupError + expr: pgbackrest_backup_last_error_status > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Backup failed for stanza {{ $labels.stanza }}" + description: | + The last pgBackRest backup failed with error status > 0. + Check pgBackRest logs for stanza {{ $labels.stanza }}. + LABELS = {{ $labels }} + + - alert: PgBackRestBackupTooOld + expr: pgbackrest_backup_since_last_completion_seconds > (60 * 60 * 24 * 7) + for: 10m + labels: + severity: warning + annotations: + summary: "No recent backup for stanza {{ $labels.stanza }}" + description: | + The last pgBackRest backup is older than 7 days. + Consider checking your backup schedule, capacity, and logs. + LABELS = {{ $labels }} + + - alert: PgBackRestStanzaError + expr: pgbackrest_stanza_status > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "pgBackRest stanza {{ $labels.stanza }} has errors" + description: | + pgBackRest reports stanza status {{ $value }} > 0. + This indicates that stanza {{ $labels.stanza }} has issues (e.g., missing or invalid backups). + Check pgBackRest logs for details. + LABELS = {{ $labels }} + + - alert: PgBackRestRepoError + expr: pgbackrest_repo_status > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "pgBackRest repo {{ $labels.repo }} has errors" + description: | + pgBackRest reports repo status {{ $value }} > 0. + This indicates repository {{ $labels.repo }} is in an error state (e.g., inaccessible, out of space). + Check pgBackRest logs and storage. + LABELS = {{ $labels }} + + - alert: PgBackRestExporterError + expr: pgbackrest_exporter_status == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "pgBackRest exporter failed" + description: | + pgBackRest exporter failed to fetch data for stanza {{ $labels.stanza }}. + This may indicate configuration or runtime errors. + LABELS = {{ $labels }} From 9e313511f42a68a834f2ed225b28f112f380032a Mon Sep 17 00:00:00 2001 From: Deezzir Date: Fri, 26 Sep 2025 20:12:32 -0400 Subject: [PATCH 04/10] chore: enable `instance` label for some `postgresql` alerts --- src/prometheus_alert_rules/postgresql_rules.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/prometheus_alert_rules/postgresql_rules.yaml b/src/prometheus_alert_rules/postgresql_rules.yaml index 2e0f760ccc..ebea69c958 100644 --- a/src/prometheus_alert_rules/postgresql_rules.yaml +++ b/src/prometheus_alert_rules/postgresql_rules.yaml @@ -86,7 +86,7 @@ groups: # 2.2.7 # warning -> info - alert: PostgresqlNotEnoughConnections - expr: 'sum by (datname) (pg_stat_activity_count{datname!~"template.*|postgres"}) < 5' + expr: 'sum by (datname,instance) (pg_stat_activity_count{datname!~"template.*|postgres"}) < 5' for: 2m labels: severity: info @@ -111,7 +111,7 @@ groups: # 2.2.9 - alert: PostgresqlHighRollbackRate - expr: 'sum by (namespace,datname) ((rate(pg_stat_database_xact_rollback{datname!~"template.*|postgres",datid!="0"}[3m])) / ((rate(pg_stat_database_xact_rollback{datname!~"template.*|postgres",datid!="0"}[3m])) + (rate(pg_stat_database_xact_commit{datname!~"template.*|postgres",datid!="0"}[3m])))) > 0.02' + expr: 'sum by (namespace,datname,instance,datid) ((rate(pg_stat_database_xact_rollback{datname!~"template.*|postgres",datid!="0"}[3m])) / ((rate(pg_stat_database_xact_rollback{datname!~"template.*|postgres",datid!="0"}[3m])) + (rate(pg_stat_database_xact_commit{datname!~"template.*|postgres",datid!="0"}[3m])))) > 0.02' for: 0m labels: severity: warning @@ -216,7 +216,7 @@ groups: # 2.2.17 # critical -> warning - alert: PostgresqlSslCompressionActive - expr: 'sum(pg_stat_ssl_compression) > 0' + expr: 'sum by (instance) (pg_stat_ssl_compression) > 0' for: 0m labels: severity: warning @@ -230,7 +230,7 @@ groups: # 2.2.18 # critical -> warning - alert: PostgresqlTooManyLocksAcquired - expr: '((sum (pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.20' + expr: '((sum by (instance) (pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.20' for: 2m labels: severity: warning From 8aedc2dcfd1a9da078e4c594e3a00c34c12d4949 Mon Sep 17 00:00:00 2001 From: Deezzir Date: Fri, 26 Sep 2025 20:12:50 -0400 Subject: [PATCH 05/10] chore: add alert rules unit tests --- .github/workflows/ci.yaml | 14 + tests/alerts/test_patroni_rules.yaml | 80 +++ tests/alerts/test_pgbackrest_rules.yaml | 157 ++++++ tests/alerts/test_pgbouncer_rules.yaml | 96 ++++ tests/alerts/test_postgresql_rules.yaml | 693 ++++++++++++++++++++++++ 5 files changed, 1040 insertions(+) create mode 100644 tests/alerts/test_patroni_rules.yaml create mode 100644 tests/alerts/test_pgbackrest_rules.yaml create mode 100644 tests/alerts/test_pgbouncer_rules.yaml create mode 100644 tests/alerts/test_postgresql_rules.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 08cbb37f67..05db40e295 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -47,6 +47,20 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + alert-test: + name: Test Prometheus Alert Rules + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout repo + uses: actions/checkout@v5 + - name: Install prometheus snap + run: sudo snap install prometheus + - name: Check validity of prometheus alert rules + run: promtool check rules src/alert_rules/prometheus/* + - name: Run unit tests for prometheus alert rules + run: promtool test rules tests/alerts/*.yaml + build: name: Build charm uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v35.0.2 diff --git a/tests/alerts/test_patroni_rules.yaml b/tests/alerts/test_patroni_rules.yaml new file mode 100644 index 0000000000..32d1fde02a --- /dev/null +++ b/tests/alerts/test_patroni_rules.yaml @@ -0,0 +1,80 @@ +rule_files: + - ../../src/prometheus_alert_rules/patroni_rules.yaml + +evaluation_interval: 1m + +tests: + - name: PatroniPostgresqlDown fires immediately when running=0 + interval: 1m + input_series: + - series: 'patroni_postgres_running{instance="pg1"}' + values: '0 0 0 0 0' + alert_rule_test: + - alertname: PatroniPostgresqlDown + eval_time: 0m + exp_alerts: + - exp_labels: + alertname: PatroniPostgresqlDown + severity: critical + instance: pg1 + exp_annotations: + summary: Patroni PostgreSQL instance pg1 is down. + description: | + Check for errors in the Loki logs. + LABELS = map[__name__:patroni_postgres_running instance:pg1] + + - name: PatroniPostgresqlDown does not fire when running=1 + interval: 1m + input_series: + - series: 'patroni_postgres_running{instance="pg2"}' + values: '1' + alert_rule_test: + - alertname: PatroniPostgresqlDown + eval_time: 1m + exp_alerts: [] + + - name: PatroniHasNoLeader fires when both master and standby_leader=0 + interval: 1m + input_series: + - series: 'patroni_master{scope="cluster1"}' + values: '0' + - series: 'patroni_standby_leader{scope="cluster1"}' + values: '0' + alert_rule_test: + - alertname: PatroniHasNoLeader + eval_time: 0m + exp_alerts: + - exp_labels: + alertname: PatroniHasNoLeader + severity: critical + scope: cluster1 + exp_annotations: + summary: Patroni instance has no leader node. + description: | + A leader node (neither primary nor standby) cannot be found inside the cluster cluster1. + Check for errors in the Loki logs. + LABELS = map[scope:cluster1] + + - name: PatroniHasNoLeader does not fire if master=1 + interval: 1m + input_series: + - series: 'patroni_master{scope="cluster1"}' + values: '1' + - series: 'patroni_standby_leader{scope="cluster1"}' + values: '0' + alert_rule_test: + - alertname: PatroniHasNoLeader + eval_time: 1m + exp_alerts: [] + + - name: PatroniHasNoLeader does not fire if standby_leader=1 + interval: 1m + input_series: + - series: 'patroni_master{scope="cluster1"}' + values: '0' + - series: 'patroni_standby_leader{scope="cluster1"}' + values: '1' + alert_rule_test: + - alertname: PatroniHasNoLeader + eval_time: 1m + exp_alerts: [] diff --git a/tests/alerts/test_pgbackrest_rules.yaml b/tests/alerts/test_pgbackrest_rules.yaml new file mode 100644 index 0000000000..c882a3e1ac --- /dev/null +++ b/tests/alerts/test_pgbackrest_rules.yaml @@ -0,0 +1,157 @@ +rule_files: + - ../../src/prometheus_alert_rules/pgbackrest_rules.yaml + +evaluation_interval: 1m + +tests: + - name: PgBackRestBackupError fires when last_error_status >0 + interval: 1m + input_series: + - series: 'pgbackrest_backup_last_error_status{stanza="demo"}' + values: '0 1 1 1 1 1 1' + alert_rule_test: + - alertname: PgBackRestBackupError + eval_time: 6m + exp_alerts: + - exp_labels: + alertname: PgBackRestBackupError + severity: critical + stanza: demo + exp_annotations: + summary: Backup failed for stanza demo + description: | + The last pgBackRest backup failed with error status > 0. + Check pgBackRest logs for stanza demo. + LABELS = map[__name__:pgbackrest_backup_last_error_status stanza:demo] + + - name: PgBackRestBackupError does not fire when last_error_status==0 + interval: 1m + input_series: + - series: 'pgbackrest_backup_last_error_status{stanza="demo"}' + values: '0 0 0 0 0 0' + alert_rule_test: + - alertname: PgBackRestBackupError + eval_time: 5m + exp_alerts: [] + + - name: PgBackRestBackupTooOld fires when last backup >7d + interval: 1m + input_series: + - series: 'pgbackrest_backup_since_last_completion_seconds{stanza="demo"}' + values: '0 604801 604801 604801 604801 604801 604801 604801 604801 604801 604801 604801' + alert_rule_test: + - alertname: PgBackRestBackupTooOld + eval_time: 11m + exp_alerts: + - exp_labels: + alertname: PgBackRestBackupTooOld + severity: warning + stanza: demo + exp_annotations: + summary: No recent backup for stanza demo + description: | + The last pgBackRest backup is older than 7 days. + Consider checking your backup schedule, capacity, and logs. + LABELS = map[__name__:pgbackrest_backup_since_last_completion_seconds stanza:demo] + + - name: PgBackRestBackupTooOld does not fire when last backup <7d + interval: 1h + input_series: + - series: 'pgbackrest_backup_since_last_completion_seconds{stanza="demo"}' + values: '0 1000 1000 1000 1000' + alert_rule_test: + - alertname: PgBackRestBackupTooOld + eval_time: 10h + exp_alerts: [] + + - name: PgBackRestStanzaError fires when stanza_status >0 + interval: 1m + input_series: + - series: 'pgbackrest_stanza_status{stanza="demo"}' + values: '0 1 1 1 1 1 1' + alert_rule_test: + - alertname: PgBackRestStanzaError + eval_time: 6m + exp_alerts: + - exp_labels: + alertname: PgBackRestStanzaError + severity: warning + stanza: demo + exp_annotations: + summary: pgBackRest stanza demo has errors + description: | + pgBackRest reports stanza status 1 > 0. + This indicates that stanza demo has issues (e.g., missing or invalid backups). + Check pgBackRest logs for details. + LABELS = map[__name__:pgbackrest_stanza_status stanza:demo] + + - name: PgBackRestStanzaError does not fire when stanza_status==0 + interval: 1m + input_series: + - series: 'pgbackrest_stanza_status{stanza="demo"}' + values: '0 0 0 0 0 0' + alert_rule_test: + - alertname: PgBackRestStanzaError + eval_time: 5m + exp_alerts: [] + + - name: PgBackRestRepoError fires when repo_status >0 + interval: 1m + input_series: + - series: 'pgbackrest_repo_status{repo="repo1"}' + values: '0 1 1 1 1 1 1' + alert_rule_test: + - alertname: PgBackRestRepoError + eval_time: 6m + exp_alerts: + - exp_labels: + alertname: PgBackRestRepoError + severity: warning + repo: repo1 + exp_annotations: + summary: pgBackRest repo repo1 has errors + description: | + pgBackRest reports repo status 1 > 0. + This indicates repository repo1 is in an error state (e.g., inaccessible, out of space). + Check pgBackRest logs and storage. + LABELS = map[__name__:pgbackrest_repo_status repo:repo1] + + - name: PgBackRestRepoError does not fire when repo_status==0 + interval: 1m + input_series: + - series: 'pgbackrest_repo_status{repo="repo1"}' + values: '0 0 0 0 0 0' + alert_rule_test: + - alertname: PgBackRestRepoError + eval_time: 5m + exp_alerts: [] + + - name: PgBackRestExporterError fires when exporter_status==0 + interval: 1m + input_series: + - series: 'pgbackrest_exporter_status{stanza="demo"}' + values: '1 0 0 0 0 0 0' + alert_rule_test: + - alertname: PgBackRestExporterError + eval_time: 6m + exp_alerts: + - exp_labels: + alertname: PgBackRestExporterError + severity: critical + stanza: demo + exp_annotations: + summary: pgBackRest exporter failed + description: | + pgBackRest exporter failed to fetch data for stanza demo. + This may indicate configuration or runtime errors. + LABELS = map[__name__:pgbackrest_exporter_status stanza:demo] + + - name: PgBackRestExporterError does not fire when exporter_status==1 + interval: 1m + input_series: + - series: 'pgbackrest_exporter_status{stanza="demo"}' + values: '1 1 1 1 1 1' + alert_rule_test: + - alertname: PgBackRestExporterError + eval_time: 5m + exp_alerts: [] diff --git a/tests/alerts/test_pgbouncer_rules.yaml b/tests/alerts/test_pgbouncer_rules.yaml new file mode 100644 index 0000000000..24e4f1c709 --- /dev/null +++ b/tests/alerts/test_pgbouncer_rules.yaml @@ -0,0 +1,96 @@ +rule_files: + - ../../src/prometheus_alert_rules/pgbouncer_rules.yaml + +evaluation_interval: 1m + +tests: + - name: PgbouncerActiveConnections fires after 2m when >200 + interval: 1m + input_series: + - series: 'pgbouncer_pools_server_active_connections{instance="pgb1"}' + values: '210 210 210' + alert_rule_test: + - alertname: PgbouncerActiveConnections + eval_time: 2m + exp_alerts: + - exp_labels: + alertname: PgbouncerActiveConnections + severity: warning + instance: pgb1 + exp_annotations: + summary: PgBouncer instance pgb1 has > 200 active connections + description: | + Consider checking the client application responsible for generating those additional connections. + LABELS = map[__name__:pgbouncer_pools_server_active_connections instance:pgb1] + + - name: PgbouncerActiveConnections does not fire when <=200 + interval: 1m + input_series: + - series: 'pgbouncer_pools_server_active_connections{instance="pgb2"}' + values: '150 180 200' + alert_rule_test: + - alertname: PgbouncerActiveConnections + eval_time: 2m + exp_alerts: [] + + - name: PgbouncerErrors fires immediately when increase>3 + interval: 1m + input_series: + - series: 'pgbouncer_errors_count{instance="pgb1",errmsg="some error"}' + values: '0 4 9 16 20' + alert_rule_test: + - alertname: PgbouncerErrors + eval_time: 3m + exp_alerts: + - exp_labels: + alertname: PgbouncerErrors + severity: warning + instance: pgb1 + errmsg: some error + exp_annotations: + summary: PgBouncer instance pgb1 is logging errors. + description: | + This may be due to a a server restart or an admin typing commands at the PgBouncer console. + VALUE = 7 + LABELS = map[errmsg:some error instance:pgb1] + + - name: PgbouncerErrors does not fire when errmsg="server conn crashed?" + interval: 1m + input_series: + - series: 'pgbouncer_errors_count{instance="pgb2",errmsg="server conn crashed?"}' + values: '0 5 10' + alert_rule_test: + - alertname: PgbouncerErrors + eval_time: 2m + exp_alerts: [] + + - name: PgbouncerMaxConnections fires when errmsg="no more connections allowed (max_client_conn)" + interval: 30s + input_series: + - series: 'pgbouncer_errors_count{instance="pgb1",errmsg="no more connections allowed (max_client_conn)"}' + values: '0 1 2 3 4 5 6' + alert_rule_test: + - alertname: PgbouncerMaxConnections + eval_time: 1m + exp_alerts: + - exp_labels: + alertname: PgbouncerMaxConnections + severity: critical + instance: pgb1 + errmsg: 'no more connections allowed (max_client_conn)' + exp_annotations: + summary: PgBouncer instance pgb1 has reached `max_client_conn`. + description: | + Consider checking how many connections the client application is opening. + VALUE = 1 + LABELS = map[errmsg:no more connections allowed (max_client_conn) instance:pgb1] + + - name: PgbouncerMaxConnections does not fire when no matching errmsg + interval: 30s + input_series: + - series: 'pgbouncer_errors_count{instance="pgb2",errmsg="other error"}' + values: '0 1' + alert_rule_test: + - alertname: PgbouncerMaxConnections + eval_time: 30s + exp_alerts: [] diff --git a/tests/alerts/test_postgresql_rules.yaml b/tests/alerts/test_postgresql_rules.yaml new file mode 100644 index 0000000000..cbeb1dca9d --- /dev/null +++ b/tests/alerts/test_postgresql_rules.yaml @@ -0,0 +1,693 @@ +rule_files: + - ../../src/prometheus_alert_rules/postgresql_rules.yaml + +evaluation_interval: 1m + +tests: + # 2.2.1 + - name: PostgresqlDown fires immediately when pg_up=0 + interval: 1m + input_series: + - series: 'pg_up{instance="pg1"}' + values: '0' + alert_rule_test: + - alertname: PostgresqlDown + eval_time: 0m + exp_alerts: + - exp_labels: + alertname: PostgresqlDown + severity: critical + instance: pg1 + exp_annotations: + summary: PostgreSQL instance pg1 is down. + description: | + If you are not upgrading or configuring cross-region async replication clusters, check for errors in the Loki logs. + LABELS = map[__name__:pg_up instance:pg1] + + - name: PostgresqlDown does not fire when pg_up=1 + interval: 1m + input_series: + - series: 'pg_up{instance="pg2"}' + values: '1' + alert_rule_test: + - alertname: PostgresqlDown + eval_time: 0m + exp_alerts: [] + + # 2.2.2 + - name: PostgresqlRestarted fires if uptime <60s + interval: 1m + input_series: + - series: 'pg_postmaster_start_time_seconds{instance="pg1"}' + values: '0' + alert_rule_test: + - alertname: PostgresqlRestarted + eval_time: 30s + exp_alerts: + - exp_labels: + alertname: PostgresqlRestarted + severity: info + instance: pg1 + exp_annotations: + summary: PostgreSQL instance pg1 has restarted. + description: | + If you are not enabling/disabling TLS or upgrading or configuring cross-region async replication clusters, check for errors in the Loki logs. + LABELS = map[instance:pg1] + + - name: PostgresqlRestarted does not fire if uptime >60s + interval: 1m + input_series: + - series: 'pg_postmaster_start_time_seconds{instance="pg2"}' + values: '0' + alert_rule_test: + - alertname: PostgresqlRestarted + eval_time: 2m + exp_alerts: [] + + # 2.2.3 + - name: PostgresqlExporterError fires if scrape error >0 + interval: 1m + input_series: + - series: 'pg_exporter_last_scrape_error{instance="pg1"}' + values: '1' + alert_rule_test: + - alertname: PostgresqlExporterError + eval_time: 0m + exp_alerts: + - exp_labels: + alertname: PostgresqlExporterError + severity: critical + instance: pg1 + exp_annotations: + summary: PostgreSQL instance pg1 is showing an exporter error. + description: | + There may be a buggy query in query.yaml + LABELS = map[__name__:pg_exporter_last_scrape_error instance:pg1] + + - name: PostgresqlExporterError does not fire when =0 + interval: 1m + input_series: + - series: 'pg_exporter_last_scrape_error{instance="pg2"}' + values: '0' + alert_rule_test: + - alertname: PostgresqlExporterError + eval_time: 0m + exp_alerts: [] + + # 2.2.4 + - name: PostgresqlTableNotAutoVacuumed fires if >7d + interval: 1d + input_series: + - series: 'pg_stat_user_tables_last_autovacuum{instance="pg1",relname="orders"}' + values: '1 2 3 4 5 6 7 8 9' + alert_rule_test: + - alertname: PostgresqlTableNotAutoVacuumed + eval_time: 8d + exp_alerts: + - exp_labels: + alertname: PostgresqlTableNotAutoVacuumed + severity: warning + instance: pg1 + relname: orders + exp_annotations: + summary: A PostgreSQL table in instance pg1 is not auto vacuumed. + description: | + Table orders has not been auto vacuumed for 7 days. + Double-check your VACUUM settings. + LABELS = map[__name__:pg_stat_user_tables_last_autovacuum instance:pg1 relname:orders] + + - name: PostgresqlTableNotAutoVacuumed does not fire if <7d + interval: 1d + input_series: + - series: 'pg_stat_user_tables_last_autovacuum{instance="pg2",relname="customers"}' + values: '0 0 0 0 0' + alert_rule_test: + - alertname: PostgresqlTableNotAutoVacuumed + eval_time: 5d + exp_alerts: [] + + # 2.2.5 + - name: PostgresqlTableNotAutoAnalyzed fires if >7d + interval: 1d + input_series: + - series: 'pg_stat_user_tables_last_autoanalyze{instance="pg1",relname="orders"}' + values: '1 2 3 4 5 6 7 8 9' + alert_rule_test: + - alertname: PostgresqlTableNotAutoAnalyzed + eval_time: 8d + exp_alerts: + - exp_labels: + alertname: PostgresqlTableNotAutoAnalyzed + severity: warning + instance: pg1 + relname: orders + exp_annotations: + summary: A PostgreSQL table in instance pg1 is not auto analyzed. + description: | + Table orders has not been auto analyzed for 7 days. + Double-check your AUTOVACUUM ANALYZE settings. + LABELS = map[__name__:pg_stat_user_tables_last_autoanalyze instance:pg1 relname:orders] + + - name: PostgresqlTableNotAutoAnalyzed does not fire if <7d + interval: 1d + input_series: + - series: 'pg_stat_user_tables_last_autoanalyze{instance="pg2",relname="customers"}' + values: '0 0 0 0 0' + alert_rule_test: + - alertname: PostgresqlTableNotAutoAnalyzed + eval_time: 5d + exp_alerts: [] + + # 2.2.6 + - name: PostgresqlTooManyConnections fires after 2m + interval: 1m + input_series: + - series: 'pg_stat_activity_count{instance="pg1",job="db",server="s1"}' + values: '90 90 90' + - series: 'pg_settings_max_connections{instance="pg1",job="db",server="s1"}' + values: '100 100 100' + alert_rule_test: + - alertname: PostgresqlTooManyConnections + eval_time: 2m + exp_alerts: + - exp_labels: + alertname: PostgresqlTooManyConnections + severity: warning + instance: pg1 + job: db + server: s1 + exp_annotations: + summary: PostgreSQL instance pg1 is using > 80% of the maximum connections. + description: | + Consider checking how many connections the client application is opening, or using PgBouncer in front of the database. + LABELS = map[instance:pg1 job:db server:s1] + + - name: PostgresqlTooManyConnections does not fire if <80% + interval: 1m + input_series: + - series: 'pg_stat_activity_count{instance="pg2",job="db",server="s2"}' + values: '40 40 40' + - series: 'pg_settings_max_connections{instance="pg2",job="db",server="s2"}' + values: '100 100 100' + alert_rule_test: + - alertname: PostgresqlTooManyConnections + eval_time: 2m + exp_alerts: [] + + # 2.2.7 + - name: PostgresqlNotEnoughConnections fires if <5 (non-template, non-postgres) + interval: 1m + input_series: + - series: 'pg_stat_activity_count{instance="pg1",datname="appdb"}' + values: '2 2 2' + alert_rule_test: + - alertname: PostgresqlNotEnoughConnections + eval_time: 2m + exp_alerts: + - exp_labels: + alertname: PostgresqlNotEnoughConnections + severity: info + instance: pg1 + datname: appdb + exp_annotations: + summary: PostgreSQL instance pg1 does not have enough connections. + description: | + PostgreSQL instance pg1 should have more connections (> 5). + Consider double-checking how many connections the client application is opening and/or using PgBouncer in front of the database. + LABELS = map[datname:appdb instance:pg1] + + - name: PostgresqlNotEnoughConnections does not fire if >=5 + interval: 1m + input_series: + - series: 'pg_stat_activity_count{instance="pg2",datname="appdb"}' + values: '6 6 6' + alert_rule_test: + - alertname: PostgresqlNotEnoughConnections + eval_time: 2m + exp_alerts: [] + + # 2.2.8 + - name: PostgresqlDeadLocks fires if increase>5 in 1m + interval: 1m + input_series: + - series: 'pg_stat_database_deadlocks{instance="pg1",datname="appdb"}' + values: '0 6' + alert_rule_test: + - alertname: PostgresqlDeadLocks + eval_time: 1m + exp_alerts: + - exp_labels: + alertname: PostgresqlDeadLocks + severity: warning + instance: pg1 + datname: appdb + exp_annotations: + summary: PostgreSQL instance pg1 has dead locks. + description: | + See more details with the pg_locks view. + LABELS = map[datname:appdb instance:pg1] + + - name: PostgresqlDeadLocks does not fire when increase <=5 + interval: 1m + input_series: + - series: 'pg_stat_database_deadlocks{instance="pg2",datname="appdb"}' + values: '0 2' + alert_rule_test: + - alertname: PostgresqlDeadLocks + eval_time: 1m + exp_alerts: [] + + # 2.2.9 + - name: PostgresqlHighRollbackRate fires if >2% + interval: 1m + input_series: + - series: 'pg_stat_database_xact_rollback{instance="pg1",datname="appdb",datid="1"}' + values: '10 20 30' + - series: 'pg_stat_database_xact_commit{instance="pg1",datname="appdb",datid="1"}' + values: '10 10 10' + alert_rule_test: + - alertname: PostgresqlHighRollbackRate + eval_time: 3m + exp_alerts: + - exp_labels: + alertname: PostgresqlHighRollbackRate + severity: warning + instance: pg1 + datname: appdb + datid: "1" + exp_annotations: + summary: PostgreSQL instance pg1 has a high rollback rate instance. + description: | + The ratio of transactions being aborted compared to committed is > 2 %. + This is probably happening due to unoptimized configurations related to commit delay, connections, memory, and WAL files. + LABELS = map[datid:1 datname:appdb instance:pg1] + + - name: PostgresqlHighRollbackRate does not fire if <=2% + interval: 1m + input_series: + - series: 'pg_stat_database_xact_rollback{instance="pg2",datname="appdb",datid="2"}' + values: '1 1 1' + - series: 'pg_stat_database_xact_commit{instance="pg2",datname="appdb",datid="2"}' + values: '100 100 100' + alert_rule_test: + - alertname: PostgresqlHighRollbackRate + eval_time: 3m + exp_alerts: [] + + # 2.2.10 + - name: PostgresqlCommitRateLow fires when commit rate <10 + interval: 1m + input_series: + - series: 'pg_stat_database_xact_commit{instance="pg1",datname="appdb"}' + values: '1 2 2 3 3 4 4 5 5 6 6 7 7 8 8' + alert_rule_test: + - alertname: PostgresqlCommitRateLow + eval_time: 6m + exp_alerts: + - exp_labels: + alertname: PostgresqlCommitRateLow + severity: info + instance: pg1 + datname: appdb + exp_annotations: + summary: PostgreSQL instance pg1 has a low commit rate. + description: | + PostgreSQL seems to be processing very few transactions. + Please check for long-running queries and configuration issues, like insufficient cache size. + LABELS = map[datname:appdb instance:pg1] + + - name: PostgresqlCommitRateLow does not fire when commit rate >=10 + interval: 1m + input_series: + - series: 'pg_stat_database_xact_commit{instance="pg2",datname="appdb"}' + values: '20 20 20' + alert_rule_test: + - alertname: PostgresqlCommitRateLow + eval_time: 2m + exp_alerts: [] + + # 2.2.11 + - name: PostgresqlLowXidConsumption fires if <5 + interval: 1m + input_series: + - series: 'pg_txid_current{instance="pg1"}' + values: '0 1 2 2 2 3 3 4 4' + alert_rule_test: + - alertname: PostgresqlLowXidConsumption + eval_time: 6m + exp_alerts: + - exp_labels: + alertname: PostgresqlLowXidConsumption + severity: info + instance: pg1 + exp_annotations: + summary: PostgreSQL instance pg1 shows low XID consumption. + description: | + PostgreSQL seems to be consuming transaction IDs very slowly. + Run ANALYZE to update the optimizer statistics, ensure that query plans are correct, and double-check your VACUUM settings. + LABELS = map[instance:pg1] + + - name: PostgresqlLowXidConsumption does not fire if >=5 + interval: 1m + input_series: + - series: 'pg_txid_current{instance="pg2"}' + values: '0 10 20 30' + alert_rule_test: + - alertname: PostgresqlLowXidConsumption + eval_time: 2m + exp_alerts: [] + + # 2.2.12 + - name: PostgresqlHighRateStatementTimeout fires when rate >3 + interval: 1m + input_series: + - series: 'postgresql_errors_total{instance="pg1",type="statement_timeout"}' + values: '0 500 1000 1500 2000 2600' + alert_rule_test: + - alertname: PostgresqlHighRateStatementTimeout + eval_time: 5m + exp_alerts: + - exp_labels: + alertname: PostgresqlHighRateStatementTimeout + severity: critical + instance: pg1 + type: statement_timeout + exp_annotations: + summary: PostgreSQL instance pg1 shows a high rate of statement timeout. + description: | + Either tune `statement_timeout` when sending queries or use EXPLAIN ANALYZE to understand how the queries can be improved. + VALUE = 10 + LABELS = map[instance:pg1 type:statement_timeout] + + - name: PostgresqlHighRateStatementTimeout does not fire when rate <=3 + interval: 1m + input_series: + - series: 'postgresql_errors_total{instance="pg2",type="statement_timeout"}' + values: '0 1 2 2' + alert_rule_test: + - alertname: PostgresqlHighRateStatementTimeout + eval_time: 3m + exp_alerts: [] + + # 2.2.13 + - name: PostgresqlHighRateDeadlock fires when increase >1 + interval: 1m + input_series: + - series: 'postgresql_errors_total{instance="pg1",type="deadlock_detected"}' + values: '0 2' + alert_rule_test: + - alertname: PostgresqlHighRateDeadlock + eval_time: 1m + exp_alerts: + - exp_labels: + alertname: PostgresqlHighRateDeadlock + severity: warning + instance: pg1 + type: deadlock_detected + exp_annotations: + summary: PostgreSQL instance pg1 shows a high deadlock rate. + description: | + More details can be obtained through the pg_locks view. + LABELS = map[instance:pg1 type:deadlock_detected] + + - name: PostgresqlHighRateDeadlock does not fire if increase <=1 + interval: 1m + input_series: + - series: 'postgresql_errors_total{instance="pg2",type="deadlock_detected"}' + values: '0 1' + alert_rule_test: + - alertname: PostgresqlHighRateDeadlock + eval_time: 1m + exp_alerts: [] + + # 2.2.14 + - name: PostgresqlUnusedReplicationSlot fires if active==0 for 1m + interval: 1m + input_series: + - series: 'pg_replication_slots_active{instance="pg1"}' + values: '0 0' + alert_rule_test: + - alertname: PostgresqlUnusedReplicationSlot + eval_time: 1m + exp_alerts: + - exp_labels: + alertname: PostgresqlUnusedReplicationSlot + severity: info + instance: pg1 + exp_annotations: + summary: PostgreSQL instance pg1 has unused replication slots. + description: | + Check if a replica is not using any of them before deleting it. + LABELS = map[__name__:pg_replication_slots_active instance:pg1] + + - name: PostgresqlUnusedReplicationSlot does not fire if active==1 + interval: 1m + input_series: + - series: 'pg_replication_slots_active{instance="pg2"}' + values: '1 1' + alert_rule_test: + - alertname: PostgresqlUnusedReplicationSlot + eval_time: 1m + exp_alerts: [] + + # 2.2.15 + - name: PostgresqlTooManyDeadTuples fires when dead/live >=0.1 + interval: 1m + input_series: + - series: 'pg_stat_user_tables_n_dead_tup{instance="pg1",relname="orders"}' + values: '20000 20000' + - series: 'pg_stat_user_tables_n_live_tup{instance="pg1",relname="orders"}' + values: '100000 100000' + alert_rule_test: + - alertname: PostgresqlTooManyDeadTuples + eval_time: 2m + exp_alerts: + - exp_labels: + alertname: PostgresqlTooManyDeadTuples + severity: warning + instance: pg1 + relname: orders + exp_annotations: + summary: PostgreSQL instance pg1 has too many dead tuples. + description: | + Double-check your VACUUM settings. + LABELS = map[instance:pg1 relname:orders] + + - name: PostgresqlTooManyDeadTuples does not fire when dead/live <0.1 + interval: 1m + input_series: + - series: 'pg_stat_user_tables_n_dead_tup{instance="pg2",relname="customers"}' + values: '100 100' + - series: 'pg_stat_user_tables_n_live_tup{instance="pg2",relname="customers"}' + values: '10000 10000' + alert_rule_test: + - alertname: PostgresqlTooManyDeadTuples + eval_time: 2m + exp_alerts: [] + + # 2.2.16 + - name: PostgresqlConfigurationChanged fires when settings metric differs + interval: 1m + input_series: + - series: 'pg_settings_max_connections{instance="pg1"}' + values: '100 100 100 100 100 100' + - series: 'pg_settings_work_mem{instance="pg1"}' + values: '4096 4096 8192 8192 8192 8192' + alert_rule_test: + - alertname: PostgresqlConfigurationChanged + eval_time: 5m + exp_alerts: + - exp_labels: + alertname: PostgresqlConfigurationChanged + severity: info + instance: pg1 + exp_annotations: + summary: PostgreSQL instance pg1 configuration has changed. + description: | + PostgreSQL database configuration has changed. + LABELS = map[__name__:pg_settings_work_mem instance:pg1] + + - name: PostgresqlConfigurationChanged does not fire if no diff + interval: 1m + input_series: + - series: 'pg_settings_max_connections{instance="pg2"}' + values: '100' + alert_rule_test: + - alertname: PostgresqlConfigurationChanged + eval_time: 5m + exp_alerts: [] + + # 2.2.17 + - name: PostgresqlSslCompressionActive fires if sum>0 + interval: 1m + input_series: + - series: 'pg_stat_ssl_compression{instance="pg1"}' + values: '1' + alert_rule_test: + - alertname: PostgresqlSslCompressionActive + eval_time: 0m + exp_alerts: + - exp_labels: + alertname: PostgresqlSslCompressionActive + severity: warning + instance: pg1 + exp_annotations: + summary: PostgreSQL instance pg1 SSL compression is active. + description: | + Database connections with SSL compression are enabled. This may add significant jitter in replication delay. + Replicas should turn off SSL compression via `sslcompression=0` in `recovery.conf`. + LABELS = map[instance:pg1] + + - name: PostgresqlSslCompressionActive does not fire if sum==0 + interval: 1m + input_series: + - series: 'pg_stat_ssl_compression{instance="pg2"}' + values: '0' + alert_rule_test: + - alertname: PostgresqlSslCompressionActive + eval_time: 0m + exp_alerts: [] + + # 2.2.18 + - name: PostgresqlTooManyLocksAcquired fires if ratio >0.2 + interval: 1m + input_series: + - series: 'pg_locks_count{instance="pg1"}' + values: '100 150 200 250 300 350' + - series: 'pg_settings_max_locks_per_transaction{instance="pg1"}' + values: '30 30 30 30 30 30' + - series: 'pg_settings_max_connections{instance="pg1"}' + values: '30 30 30 30 30 30' + alert_rule_test: + - alertname: PostgresqlTooManyLocksAcquired + eval_time: 4m + exp_alerts: + - exp_labels: + alertname: PostgresqlTooManyLocksAcquired + severity: warning + instance: pg1 + exp_annotations: + summary: PostgreSQL instance pg1 has acquired too many locks. + description: | + If this alert happens frequently, you may need to increase the PostgreSQL setting max_locks_per_transaction. + LABELS = map[instance:pg1] + + - name: PostgresqlTooManyLocksAcquired does not fire if ratio <=0.2 + interval: 1m + input_series: + - series: 'pg_locks_count{instance="pg2"}' + values: '100' + - series: 'pg_settings_max_locks_per_transaction{instance="pg2"}' + values: '50' + - series: 'pg_settings_max_connections{instance="pg2"}' + values: '100' + alert_rule_test: + - alertname: PostgresqlTooManyLocksAcquired + eval_time: 2m + exp_alerts: [] + + + # 2.2.19 + - name: PostgresqlBloatIndexHigh fires if bloat_pct >80 and size>100MB + interval: 5m + input_series: + - series: 'pg_bloat_btree_bloat_pct{idxname="idx_orders",instance="pg1"}' + values: '85 85 85 85 85 85 85 85 85 85 85 85 85 85 85 85 85 85 85 85' + - series: 'pg_bloat_btree_real_size{idxname="idx_orders",instance="pg1"}' + values: '200000000 200000000 200000000 200000000 200000000 200000000 200000000 200000000 200000000 200000000 200000000 200000000 200000000 200000000 200000000 200000000' + alert_rule_test: + - alertname: PostgresqlBloatIndexHigh(>80%) + eval_time: 1h10m + exp_alerts: + - exp_labels: + alertname: PostgresqlBloatIndexHigh(>80%) + severity: warning + instance: pg1 + idxname: idx_orders + exp_annotations: + summary: PostgreSQL instance pg1 has a high bloat index (> 80%). + description: | + The index idx_orders is bloated. + Consider running `REINDEX INDEX CONCURRENTLY idx_orders;` + LABELS = map[__name__:pg_bloat_btree_bloat_pct idxname:idx_orders instance:pg1] + + - name: PostgresqlBloatIndexHigh does not fire if bloat_pct <=80 + interval: 1h + input_series: + - series: 'pg_bloat_btree_bloat_pct{instance="pg2",idxname="idx_customers"}' + values: '70 70' + - series: 'pg_bloat_btree_real_size{instance="pg2",idxname="idx_customers"}' + values: '200000000 200000000' + alert_rule_test: + - alertname: PostgresqlBloatIndexHigh(>80%) + eval_time: 1h + exp_alerts: [] + + # 2.2.20 + - name: PostgresqlBloatTableHigh fires if bloat_pct >80 and size>200MB + interval: 5m + input_series: + - series: 'pg_bloat_table_bloat_pct{instance="pg1",relname="orders"}' + values: '90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90' + - series: 'pg_bloat_table_real_size{instance="pg1",relname="orders"}' + values: '500000000 500000000 500000000 500000000 500000000 500000000 500000000 500000000 500000000 500000000 500000000 500000000 500000000 500000000 500000000' + alert_rule_test: + - alertname: PostgresqlBloatTableHigh(>80%) + eval_time: 1h10m + exp_alerts: + - exp_labels: + alertname: PostgresqlBloatTableHigh(>80%) + severity: warning + instance: pg1 + relname: orders + exp_annotations: + summary: PostgreSQL instance pg1 has a high bloat table (> 80%). + description: | + The table orders is bloated. + Consider running `VACUUM orders;` + LABELS = map[__name__:pg_bloat_table_bloat_pct instance:pg1 relname:orders] + + - name: PostgresqlBloatTableHigh does not fire if bloat_pct <=80 + interval: 1h + input_series: + - series: 'pg_bloat_table_bloat_pct{instance="pg2",relname="customers"}' + values: '60 60' + - series: 'pg_bloat_table_real_size{instance="pg2",relname="customers"}' + values: '500000000 500000000' + alert_rule_test: + - alertname: PostgresqlBloatTableHigh(>80%) + eval_time: 1h + exp_alerts: [] + + # 2.2.21 + - name: PostgresqlInvalidIndex fires after 6h if index present + interval: 5m + input_series: + - series: 'pg_general_index_info_pg_relation_size{indexrelname="orders_ccnew_idx",instance="pg1",relname="orders"}' + values: '1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' + alert_rule_test: + - alertname: PostgresqlInvalidIndex + eval_time: 7h + exp_alerts: + - exp_labels: + alertname: PostgresqlInvalidIndex + severity: critical + instance: pg1 + relname: orders + indexrelname: orders_ccnew_idx + exp_annotations: + summary: PostgreSQL instance pg1)= has an invalid index. + description: | + The table orders has an invalid index: orders_ccnew_idx. + Consider running `DROP INDEX orders_ccnew_idx;` + LABELS = map[__name__:pg_general_index_info_pg_relation_size indexrelname:orders_ccnew_idx instance:pg1 relname:orders] + + - name: PostgresqlInvalidIndex does not fire if no ccnew index + interval: 1h + input_series: + - series: 'pg_general_index_info_pg_relation_size{instance="pg2",relname="orders",indexrelname="orders_valid_idx"}' + values: '1 1 1 1 1 1 1' + alert_rule_test: + - alertname: PostgresqlInvalidIndex + eval_time: 6h + exp_alerts: [] From 4462a03504b8feff8b84c1dfa190971b3166b8bc Mon Sep 17 00:00:00 2001 From: Deezzir Date: Fri, 26 Sep 2025 20:13:05 -0400 Subject: [PATCH 06/10] docs: update alert rules list --- docs/reference/alert-rules.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/reference/alert-rules.md b/docs/reference/alert-rules.md index 809d452be6..faf048131d 100644 --- a/docs/reference/alert-rules.md +++ b/docs/reference/alert-rules.md @@ -48,8 +48,17 @@ This page contains a markdown version of the alert rules described in the `postg | `PatroniPostgresqlDown` | ![critical] | Patroni PostgreSQL instance is down.
Check for errors in the Loki logs. | | `PatroniHasNoLeader` | ![critical] | Patroni instance has no leader node.
A leader node (neither primary nor standby) cannot be found inside a cluster.
Check for errors in the Loki logs. | +## `PgbackrestExporterK8s` + +| Alert | Severity | Notes | +| ----- | -------- | ----- | +| `PgBackRestBackupError` | ![critical] | Backup failed for a stanza.
The last pgBackRest backup ended with error status > 0.
Check the pgBackRest logs for the stanza. | +| `PgBackRestBackupTooOld` | ![warning] | No recent backup available.
The last pgBackRest backup is older than 7 days.
Consider checking your backup schedule, capacity, and logs. | +| `PgBackRestStanzaError` | ![warning] | A stanza has reported errors.
Status > 0 indicates problems such as missing stanza path or no valid backups.
Check pgBackRest logs for details. | +| `PgBackRestRepoError` | ![warning] | A repository has reported errors.
Status > 0 indicates the repo may be inaccessible, out of space, or otherwise unhealthy.
Check pgBackRest logs and storage system. | +| `PgBackRestExporterError` | ![critical] | The pgBackRest exporter failed to fetch data.
Metric `pgbackrest_exporter_status == 0` indicates exporter-side issues.
This may be a misconfiguration or runtime error; check exporter logs. | + [info]: https://img.shields.io/badge/info-blue [warning]: https://img.shields.io/badge/warning-yellow [critical]: https://img.shields.io/badge/critical-red - From 89b4c53f1427b4f7c8871ec08a71c7e3b5b04989 Mon Sep 17 00:00:00 2001 From: Deezzir Date: Fri, 26 Sep 2025 20:22:30 -0400 Subject: [PATCH 07/10] fix: path for alerts in `alert-test` ci job --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 05db40e295..34cf6b5554 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -57,7 +57,7 @@ jobs: - name: Install prometheus snap run: sudo snap install prometheus - name: Check validity of prometheus alert rules - run: promtool check rules src/alert_rules/prometheus/* + run: promtool check rules src/prometheus_alert_rules/* - name: Run unit tests for prometheus alert rules run: promtool test rules tests/alerts/*.yaml From 6bd839616d11867df4955f628d470efb55603d4b Mon Sep 17 00:00:00 2001 From: Deezzir Date: Fri, 26 Sep 2025 20:28:37 -0400 Subject: [PATCH 08/10] fix: `test_charm` unit test --- src/charm.py | 2 +- tests/unit/test_charm.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index e6d8d914a3..2c581432d0 100755 --- a/src/charm.py +++ b/src/charm.py @@ -214,7 +214,7 @@ def __init__(self, *args): self.pgbackrest_server_service = "pgbackrest server" self.ldap_sync_service = "ldap-sync" self.metrics_service = "metrics_server" - self.pgbackrest_metrics_service = "pgbackrest_metrics_exporter" + self.pgbackrest_metrics_service = "pgbackrest_metrics_service" self._unit = self.model.unit.name self._name = self.model.app.name self._namespace = self.model.name diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 84759e94b3..89340516bd 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -33,6 +33,7 @@ POSTGRESQL_SERVICE = "postgresql" LDAP_SYNC_SERVICE = "ldap-sync" METRICS_SERVICE = "metrics_server" +PGBACKREST_METRICS_SERVICE = "pgbackrest_metrics_service" PGBACKREST_SERVER_SERVICE = "pgbackrest server" ROTATE_LOGS_SERVICE = "rotate-logs" @@ -995,6 +996,15 @@ def test_postgresql_layer(harness): ), }, }, + PGBACKREST_METRICS_SERVICE: { + "override": "replace", + "summary": "pgbackrest metrics exporter", + "command": "/usr/bin/pgbackrest_exporter", + "startup": "enabled", + "after": [POSTGRESQL_SERVICE], + "user": "postgres", + "group": "postgres", + }, ROTATE_LOGS_SERVICE: { "override": "replace", "summary": "rotate logs", From db8f13a4c07b513554faebac4a896fe29160011f Mon Sep 17 00:00:00 2001 From: Deezzir Date: Wed, 15 Oct 2025 13:06:09 -0400 Subject: [PATCH 09/10] chore: correct pgbackrest name --- src/grafana_dashboards/postgresql-pgbackrest-metrics.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/grafana_dashboards/postgresql-pgbackrest-metrics.json b/src/grafana_dashboards/postgresql-pgbackrest-metrics.json index 3b71fb4f80..3bfa349829 100644 --- a/src/grafana_dashboards/postgresql-pgbackrest-metrics.json +++ b/src/grafana_dashboards/postgresql-pgbackrest-metrics.json @@ -7107,7 +7107,7 @@ }, "timepicker": {}, "timezone": "", - "title": "PgBackrest K8s", + "title": "pgBackRest K8s", "tags": [ "postgres", "pgbackrest", From ea9a4590592626fcae36a61e772a4cf00532b9ac Mon Sep 17 00:00:00 2001 From: Deezzir Date: Wed, 15 Oct 2025 16:44:56 -0400 Subject: [PATCH 10/10] test: fix `test_restart` --- tests/integration/ha_tests/test_restart.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/ha_tests/test_restart.py b/tests/integration/ha_tests/test_restart.py index 26a93da597..e3a2a83997 100644 --- a/tests/integration/ha_tests/test_restart.py +++ b/tests/integration/ha_tests/test_restart.py @@ -38,6 +38,8 @@ async def test_deploy(ops_test: OpsTest, charm) -> None: if not await app_name(ops_test, APPLICATION_NAME): await ops_test.model.deploy(APPLICATION_NAME, num_units=1) + await ops_test.model.relate(DATABASE_APP_NAME, f"{APPLICATION_NAME}:database") + async with ops_test.fast_forward(): await ops_test.model.wait_for_idle( apps=[DATABASE_APP_NAME, APPLICATION_NAME],