Add volume heal entries count metric (#21634)

* Add volume heal entries count metric

* Use case-insensitive check for connected status

Author: ricmatsui

glusterfs/changelog.d/21634.added

@@ -0,0 +1 @@
+Add volume heal entries count metric

glusterfs/datadog_checks/glusterfs/check.py

@@ -20,7 +20,7 @@
 except ImportError:
     from datadog_checks.base.stubs import datadog_agent
 
-from .metrics import BRICK_STATS, CLUSTER_STATS, PARSE_METRICS, VOL_SUBVOL_STATS, VOLUME_STATS
+from .metrics import BRICK_STATS, CLUSTER_STATS, HEAL_INFO_STATS, PARSE_METRICS, VOL_SUBVOL_STATS, VOLUME_STATS
 
 GLUSTER_VERSION = 'glfs_version'
 CLUSTER_STATUS = 'cluster_status'
@@ -156,6 +156,9 @@ def parse_volume_summary(self, output):
             if 'subvols' in volume:
                 self.parse_subvols_stats(volume.get('subvols', []), volume_tags)
 
+            if 'healinfo' in volume:
+                self.parse_healinfo_stats(volume.get('healinfo', []), volume_tags)
+
             self.submit_service_check(self.VOLUME_SC, volume.get('health'), volume_tags)
 
     def parse_subvols_stats(self, subvols, volume_tags):
@@ -181,6 +184,21 @@ def parse_subvols_stats(self, subvols, volume_tags):
                 tags.extend(subvol_tags)
                 self.submit_metrics(brick, 'brick', BRICK_STATS, tags)
 
+    def parse_healinfo_stats(self, healinfo, volume_tags):
+        for info in healinfo:
+            if info['status'].lower() != 'connected':
+                continue
+
+            brick_name = info['name'].split(":")
+            brick_server = brick_name[0]
+            brick_export = brick_name[1]
+            tags = [
+                'brick_server:{}'.format(brick_server),
+                'brick_export:{}'.format(brick_export),
+            ]
+            tags.extend(volume_tags)
+            self.submit_metrics(info, 'heal_info', HEAL_INFO_STATS, tags)
+
     def submit_metrics(self, payload, prefix, metric_mapping, tags):
         """
         Parse a payload with a given metric_mapping and submit metric for valid values.

glusterfs/datadog_checks/glusterfs/metrics.py

@@ -41,5 +41,7 @@
 BRICK_STATS = {'block_size': 'block_size'}
 BRICK_STATS.update(GENERAL_STATS)
 
+HEAL_INFO_STATS = {'nr_entries': 'entries.count'}
+
 # Parse metric values that contain measurements, e.g "2.00 GiB"
 PARSE_METRICS = ['v_size_used', 'v_size']

glusterfs/metadata.csv

@@ -11,6 +11,7 @@ glusterfs.cluster.nodes.active,gauge,,node,,Current active nodes,0,glusterfs,clu
 glusterfs.cluster.nodes.count,gauge,,node,,Total number of nodes in cluster,0,glusterfs,cluster node total,
 glusterfs.cluster.volumes.count,gauge,,unit,,Number of volumes in cluster,0,glusterfs,cluster volume count,
 glusterfs.cluster.volumes.started,gauge,,unit,,Number of volumes started in cluster,0,glusterfs,cluster volume start,
+glusterfs.heal_info.entries.count,gauge,,unit,,Number of entries requiring healing,0,glusterfs,entry count,
 glusterfs.subvol.disperse,gauge,,unit,,Disperse count of subvolume,0,glusterfs,subvol disperse,
 glusterfs.subvol.disperse_redundancy,gauge,,unit,,Disperse redundancy of subvolume,0,glusterfs,subvol disperse redundancy,
 glusterfs.subvol.replica,gauge,,unit,,Replicas in subvolume,0,glusterfs,subvol replica,

glusterfs/tests/common.py

@@ -23,6 +23,7 @@
     "glusterfs.cluster.nodes.count",
     "glusterfs.cluster.volumes.count",
     "glusterfs.cluster.volumes.started",
+    "glusterfs.heal_info.entries.count",
     "glusterfs.subvol.disperse",
     "glusterfs.subvol.disperse_redundancy",
     "glusterfs.subvol.replica",

glusterfs/tests/fixtures/gstatus.txt

@@ -1,2 +1,2 @@
 Note: Unable to get self-heal status for one or more volumes
-{"last_updated": "2024-08-29 20:27:02.124097", "data": {"cluster_status": "Healthy", "glfs_version": "7.1", "node_count": 2, "nodes_active": 2, "volume_count": 1, "volumes_started": 1, "volume_summary": [{"name": "gv0", "uuid": "07d66a0b-a91a-40d6-bf9f-61929d580469", "type": "REPLICATE", "status": "Started", "num_bricks": 2, "distribute": 2, "stripe": 1, "replica": 2, "disperse": 0, "disperse_redundancy": 0, "transport": "TCP", "snapshot_count": 0, "options": [{"name": "transport.address-family", "value": "inet"}, {"name": "storage.fips-mode-rchecksum", "value": "on"}, {"name": "nfs.disable", "value": "on"}, {"name": "performance.client-io-threads", "value": "off"}], "subvols": [{"name": "gv0-replicate-0", "replica": 2, "disperse": 0, "disperse_redundancy": 0, "type": "REPLICATE", "bricks": [{"name": "gluster-node-1:/export-test", "uuid": "8012e8ac-7cf5-4b3d-baa1-267bb3e295cc", "online": false, "pid": "-1", "size_total": 124678225920, "size_free": 68029804544, "inodes_total": 15482880, "inodes_free": 14286246, "device": "overlay", "block_size": "4096", "mnt_options": "rw,relatime,lowerdir=/var/lib/docker/overlay2/l/LESL5QOHZYQNDD25KZZHVRX3OB:/var/lib/docker/overlay2/l/PII5LMBVHYV4L4EALWEHCKWPQ3:/var/lib/docker/overlay2/l/OERRABS5DQBFV4INMOMQQ3FX5O:/var/lib/docker/overlay2/l/AYNOUZOPONWWW75WH2RPMP4DOI:/var/lib/docker/overlay2/l/45Q3YDG43GBCMKIJPZDITUUXFX:/var/lib/docker/overlay2/l/G3IKPJOI3F3RIVZEBPTMX5BLPO:/var/lib/docker/overlay2/l/TDLCX2SGGKG7ABDGJDMJC3ONQ6:/var/lib/docker/overlay2/l/Y7CHXYLWEF6HRJBJDI4M5I4GFD:/var/lib/docker/overlay2/l/7TPEWGIZAH3S5U5B72V2K7M3WH:/var/lib/docker/overlay2/l/CKZCWXAMXB6RMLU4A4VFDCOEYK:/var/lib/docker/overlay2/l/NNQ72DJMBC7TUAK277FA5PR4IK:/var/lib/docker/overlay2/l/44LZI42GLK5IHSPPMXPLIEQLHE:/var/lib/docker/overlay2/l/4JUASJHM3RCXVY7LJ77YCXBHWN:/var/lib/docker/overlay2/l/BC4LWP5MFS25IHS7XF2K6LOCZT:/var/lib/docker/overlay2/l/UMY7IRPY4F66A2UYYHWC2XSTZQ:/var/lib/docker/overlay2/l/UQSEQUD36GO7377GNEW5Y5IRR2:/var/lib/docker/overlay2/l/M4WVVFWYC4MNNWA3KUCN2IX64U:/var/lib/docker/overlay2/l/J2RQAAZX2CJNGGPVOVXI253ZVL:/var/lib/docker/overlay2/l/V5VCR6O3VJMTJFBVQ3V2XFBHQ6:/var/lib/docker/overlay2/l/PGLAKHDGTPFYYIXS4ZXF5BCSZZ,upperdir=/var/lib/docker/overlay2/6b5c18e3ea5f4ebbf2d14e6b3d7134177fc8d4a548c83db85d48dc9916a2996e/diff,workdir=/var/lib/docker/overlay2/6b5c18e3ea5f4ebbf2d14e6b3d7134177fc8d4a548c83db85d48dc9916a2996e/work,nouserxattr", "fs_name": "overlay", "size_used": 56648421376, "inodes_used": 1196634, "ports": {"tcp": "N/A", "rdma": "N/A"}, "type": "Brick"}, {"name": "gluster-node-2:/export-test", "uuid": "0e6d296e-eee1-4d9f-9c94-cc8492eb41a6", "online": false, "pid": "-1", "size_total": 124678225920, "size_free": 68029804544, "inodes_total": 15482880, "inodes_free": 14286246, "device": "overlay", "block_size": "4096", "mnt_options": "rw,relatime,lowerdir=/var/lib/docker/overlay2/l/RTX55737OX6K4UJR6OILVMJSEZ:/var/lib/docker/overlay2/l/PII5LMBVHYV4L4EALWEHCKWPQ3:/var/lib/docker/overlay2/l/OERRABS5DQBFV4INMOMQQ3FX5O:/var/lib/docker/overlay2/l/AYNOUZOPONWWW75WH2RPMP4DOI:/var/lib/docker/overlay2/l/45Q3YDG43GBCMKIJPZDITUUXFX:/var/lib/docker/overlay2/l/G3IKPJOI3F3RIVZEBPTMX5BLPO:/var/lib/docker/overlay2/l/TDLCX2SGGKG7ABDGJDMJC3ONQ6:/var/lib/docker/overlay2/l/Y7CHXYLWEF6HRJBJDI4M5I4GFD:/var/lib/docker/overlay2/l/7TPEWGIZAH3S5U5B72V2K7M3WH:/var/lib/docker/overlay2/l/CKZCWXAMXB6RMLU4A4VFDCOEYK:/var/lib/docker/overlay2/l/NNQ72DJMBC7TUAK277FA5PR4IK:/var/lib/docker/overlay2/l/44LZI42GLK5IHSPPMXPLIEQLHE:/var/lib/docker/overlay2/l/4JUASJHM3RCXVY7LJ77YCXBHWN:/var/lib/docker/overlay2/l/BC4LWP5MFS25IHS7XF2K6LOCZT:/var/lib/docker/overlay2/l/UMY7IRPY4F66A2UYYHWC2XSTZQ:/var/lib/docker/overlay2/l/UQSEQUD36GO7377GNEW5Y5IRR2:/var/lib/docker/overlay2/l/M4WVVFWYC4MNNWA3KUCN2IX64U:/var/lib/docker/overlay2/l/J2RQAAZX2CJNGGPVOVXI253ZVL:/var/lib/docker/overlay2/l/V5VCR6O3VJMTJFBVQ3V2XFBHQ6:/var/lib/docker/overlay2/l/PGLAKHDGTPFYYIXS4ZXF5BCSZZ,upperdir=/var/lib/docker/overlay2/c811601994dbcdd9a206a7cff359adb935bbf302e0a74327e1c486067642dc1e/diff,workdir=/var/lib/docker/overlay2/c811601994dbcdd9a206a7cff359adb935bbf302e0a74327e1c486067642dc1e/work,nouserxattr", "fs_name": "overlay", "size_used": 56648421376, "inodes_used": 1196634, "ports": {"tcp": "N/A", "rdma": "N/A"}, "type": "Brick"}], "health": "down"}], "size_total": 124678225920, "size_free": 68029804544, "size_used": 56648421376, "inodes_total": 15482880, "inodes_free": 14286246, "inodes_used": 1196634, "health": "down", "v_size": "116.12 GiB", "v_size_used": "52.76 GiB", "v_used_percent": "45.44", "online": 0, "voltype": "", "quota": ""}]}}
+{"last_updated": "2024-08-29 20:27:02.124097", "data": {"cluster_status": "Healthy", "glfs_version": "7.1", "node_count": 2, "nodes_active": 2, "volume_count": 1, "volumes_started": 1, "volume_summary": [{"name": "gv0", "uuid": "07d66a0b-a91a-40d6-bf9f-61929d580469", "type": "REPLICATE", "status": "Started", "num_bricks": 2, "distribute": 2, "stripe": 1, "replica": 2, "disperse": 0, "disperse_redundancy": 0, "transport": "TCP", "snapshot_count": 0, "options": [{"name": "transport.address-family", "value": "inet"}, {"name": "storage.fips-mode-rchecksum", "value": "on"}, {"name": "nfs.disable", "value": "on"}, {"name": "performance.client-io-threads", "value": "off"}], "subvols": [{"name": "gv0-replicate-0", "replica": 2, "disperse": 0, "disperse_redundancy": 0, "type": "REPLICATE", "bricks": [{"name": "gluster-node-1:/export-test", "uuid": "8012e8ac-7cf5-4b3d-baa1-267bb3e295cc", "online": false, "pid": "-1", "size_total": 124678225920, "size_free": 68029804544, "inodes_total": 15482880, "inodes_free": 14286246, "device": "overlay", "block_size": "4096", "mnt_options": "rw,relatime,lowerdir=/var/lib/docker/overlay2/l/LESL5QOHZYQNDD25KZZHVRX3OB:/var/lib/docker/overlay2/l/PII5LMBVHYV4L4EALWEHCKWPQ3:/var/lib/docker/overlay2/l/OERRABS5DQBFV4INMOMQQ3FX5O:/var/lib/docker/overlay2/l/AYNOUZOPONWWW75WH2RPMP4DOI:/var/lib/docker/overlay2/l/45Q3YDG43GBCMKIJPZDITUUXFX:/var/lib/docker/overlay2/l/G3IKPJOI3F3RIVZEBPTMX5BLPO:/var/lib/docker/overlay2/l/TDLCX2SGGKG7ABDGJDMJC3ONQ6:/var/lib/docker/overlay2/l/Y7CHXYLWEF6HRJBJDI4M5I4GFD:/var/lib/docker/overlay2/l/7TPEWGIZAH3S5U5B72V2K7M3WH:/var/lib/docker/overlay2/l/CKZCWXAMXB6RMLU4A4VFDCOEYK:/var/lib/docker/overlay2/l/NNQ72DJMBC7TUAK277FA5PR4IK:/var/lib/docker/overlay2/l/44LZI42GLK5IHSPPMXPLIEQLHE:/var/lib/docker/overlay2/l/4JUASJHM3RCXVY7LJ77YCXBHWN:/var/lib/docker/overlay2/l/BC4LWP5MFS25IHS7XF2K6LOCZT:/var/lib/docker/overlay2/l/UMY7IRPY4F66A2UYYHWC2XSTZQ:/var/lib/docker/overlay2/l/UQSEQUD36GO7377GNEW5Y5IRR2:/var/lib/docker/overlay2/l/M4WVVFWYC4MNNWA3KUCN2IX64U:/var/lib/docker/overlay2/l/J2RQAAZX2CJNGGPVOVXI253ZVL:/var/lib/docker/overlay2/l/V5VCR6O3VJMTJFBVQ3V2XFBHQ6:/var/lib/docker/overlay2/l/PGLAKHDGTPFYYIXS4ZXF5BCSZZ,upperdir=/var/lib/docker/overlay2/6b5c18e3ea5f4ebbf2d14e6b3d7134177fc8d4a548c83db85d48dc9916a2996e/diff,workdir=/var/lib/docker/overlay2/6b5c18e3ea5f4ebbf2d14e6b3d7134177fc8d4a548c83db85d48dc9916a2996e/work,nouserxattr", "fs_name": "overlay", "size_used": 56648421376, "inodes_used": 1196634, "ports": {"tcp": "N/A", "rdma": "N/A"}, "type": "Brick"}, {"name": "gluster-node-2:/export-test", "uuid": "0e6d296e-eee1-4d9f-9c94-cc8492eb41a6", "online": false, "pid": "-1", "size_total": 124678225920, "size_free": 68029804544, "inodes_total": 15482880, "inodes_free": 14286246, "device": "overlay", "block_size": "4096", "mnt_options": "rw,relatime,lowerdir=/var/lib/docker/overlay2/l/RTX55737OX6K4UJR6OILVMJSEZ:/var/lib/docker/overlay2/l/PII5LMBVHYV4L4EALWEHCKWPQ3:/var/lib/docker/overlay2/l/OERRABS5DQBFV4INMOMQQ3FX5O:/var/lib/docker/overlay2/l/AYNOUZOPONWWW75WH2RPMP4DOI:/var/lib/docker/overlay2/l/45Q3YDG43GBCMKIJPZDITUUXFX:/var/lib/docker/overlay2/l/G3IKPJOI3F3RIVZEBPTMX5BLPO:/var/lib/docker/overlay2/l/TDLCX2SGGKG7ABDGJDMJC3ONQ6:/var/lib/docker/overlay2/l/Y7CHXYLWEF6HRJBJDI4M5I4GFD:/var/lib/docker/overlay2/l/7TPEWGIZAH3S5U5B72V2K7M3WH:/var/lib/docker/overlay2/l/CKZCWXAMXB6RMLU4A4VFDCOEYK:/var/lib/docker/overlay2/l/NNQ72DJMBC7TUAK277FA5PR4IK:/var/lib/docker/overlay2/l/44LZI42GLK5IHSPPMXPLIEQLHE:/var/lib/docker/overlay2/l/4JUASJHM3RCXVY7LJ77YCXBHWN:/var/lib/docker/overlay2/l/BC4LWP5MFS25IHS7XF2K6LOCZT:/var/lib/docker/overlay2/l/UMY7IRPY4F66A2UYYHWC2XSTZQ:/var/lib/docker/overlay2/l/UQSEQUD36GO7377GNEW5Y5IRR2:/var/lib/docker/overlay2/l/M4WVVFWYC4MNNWA3KUCN2IX64U:/var/lib/docker/overlay2/l/J2RQAAZX2CJNGGPVOVXI253ZVL:/var/lib/docker/overlay2/l/V5VCR6O3VJMTJFBVQ3V2XFBHQ6:/var/lib/docker/overlay2/l/PGLAKHDGTPFYYIXS4ZXF5BCSZZ,upperdir=/var/lib/docker/overlay2/c811601994dbcdd9a206a7cff359adb935bbf302e0a74327e1c486067642dc1e/diff,workdir=/var/lib/docker/overlay2/c811601994dbcdd9a206a7cff359adb935bbf302e0a74327e1c486067642dc1e/work,nouserxattr", "fs_name": "overlay", "size_used": 56648421376, "inodes_used": 1196634, "ports": {"tcp": "N/A", "rdma": "N/A"}, "type": "Brick"}], "health": "down"}], "size_total": 124678225920, "size_free": 68029804544, "size_used": 56648421376, "inodes_total": 15482880, "inodes_free": 14286246, "inodes_used": 1196634, "health": "down", "v_size": "116.12 GiB", "v_size_used": "52.76 GiB", "v_used_percent": "45.44", "online": 0, "voltype": "", "quota": "", "healinfo": [{ "name": "gluster-node-1:/export-test", "status": "Connected", "host_uuid": "8012e8ac-7cf5-4b3d-baa1-267bb3e295cc", "nr_entries": "0" }, { "name": "gluster-node-2:/export-test", "status": "Unknown error", "host_uuid": "0e6d296e-eee1-4d9f-9c94-cc8492eb41a6", "nr_entries": "-" }]}]}}