First the setup.
For each of our xcp-ng hosts (5 for this cluster) I setup the following.

cat <<EOF > /etc/drbd-reactor.d/prometheus.toml
[[prometheus]]
enums = true
address = "10.2.0.1[0-4]:9942"
EOF

systemctl restart drbd-reactor

nano /etc/sysconfig/iptables 

#add 
-A RH-Firewall-1-INPUT -s 10.1.8.0/22 -p tcp -m conntrack --ctstate NEW -m tcp --dport 9942 -j ACCEPT
-A RH-Firewall-1-INPUT -s 10.1.8.0/22 -p tcp -m conntrack --ctstate NEW -m tcp --dport 9100 -j ACCEPT

systemctl restart iptables



wget https://github.com/prometheus/node_exporter/releases/download/v1.12.1/node_exporter-1.12.1.linux-amd64.tar.gz
tar xvfz node_exporter-*.linux-amd64.tar.gz
mv node_exporter-*.linux-amd64/node_exporter /usr/local/bin/


cat <<EOF > /etc/systemd/system/node_exporter.service
[Unit]
Description=Node Exporter
After=network.target

[Service]
User=root
ExecStart=/usr/local/bin/node_exporter --web.listen-address=10.2.0.1[0-4]:9100

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable --now node_exporter

Then because I wanted to reuse my existing setup, I deployed the following in k8s in a new namespace for organization.

---
apiVersion: v1
kind: Endpoints
metadata:
  name: xcp-ng-metrics
  namespace: xcp-ng-monitoring
subsets:
  - addresses:
      - ip: 10.2.0.10
      - ip: 10.2.0.11
      - ip: 10.2.0.12
      - ip: 10.2.0.13
      - ip: 10.2.0.14
    ports:
      - name: drbd
        port: 9942
        protocol: TCP
      - name: node
        port: 9100
        protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
  name: xcp-ng-metrics
  namespace: xcp-ng-monitoring
  labels:
    app: xcp-ng-metrics
spec:
  clusterIP: None
  ports:
    - name: drbd
      port: 9942
      targetPort: 9942
    - name: node
      port: 9100
      targetPort: 9100
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: xcp-ng
  namespace: xcp-ng-monitoring
spec:
  selector:
    matchLabels:
      app: xcp-ng-metrics
  endpoints:
    - port: drbd
      interval: 30s
    - port: node
      interval: 30s

Then I went and nabbed these two dashboards linked in the GUI

Node-exporter is as expected

I have already built out the DRBD reactor dashboard, cause I want to spec out some prometheus rules.

Possible prometheus rules for alerting, gotten by taking https://github.com/LINBIT/drbd-reactor/blob/master/doc/prometheus.md and feeding it into an llm. Then going over each one and created a panel for it and tinkered to ensure it functioned as desired. A few generated were nonsense but that is how it goes lol.

---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: xcp-ng-metrics-rules
  namespace: xcp-ng-monitoring
spec:
  groups:
  - name: drbd_process_health
    rules:
    - alert: DRBDReactorDaemonDown
      expr: drbdreactor_up == 0 or up{job="drbd-reactor"} == 0
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: "DRBD Reactor telemetry agent is unreachable on {{ $labels.instance }}"
        description: "The DRBD Prometheus endpoint has failed. Metrics for this node are stale, blinding the cluster manager to storage faults."

    - alert: DRBDResourceCountAnomaly
      expr: changes(drbd_resource_resources[15m]) > 0
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "DRBD resource count fluctuated on {{ $labels.instance }}"
        description: "The number of active DRBD resources has changed unexpectedly. Verify that no resources were erroneously deleted or unconfigured."


  - name: drbd_connection_integrity
    rules:
    - alert: DRBDConnectionSplitBrain
      expr: drbd_connection_state{drbd_connection_state="StandAlone"} == 1
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: "DRBD Split-Brain detected on {{ $labels.instance }} (StandAlone State)"
        description: "The DRBD resource has entered the StandAlone state, refusing to reconnect. This signifies failed authentication, manual disconnection, or an active split-brain partition."

    - alert: DRBDNetworkFailure
      expr: drbd_connection_state{drbd_connection_state="NetworkFailure"} == 1 or drbd_connection_state{drbd_connection_state="BrokenPipe"} == 1
      for: 2m
      labels:
        severity: critical
      annotations:
        summary: "DRBD peer connection lost on {{ $labels.instance }}"
        description: "The TCP/IP connection to the DRBD peer has been severed. Replication is currently halted."

    - alert: DRBDNetworkCongestion
      expr: drbd_connection_congested == 1
      for: 10m
      labels:
        severity: warning
      annotations:
        summary: "TCP Send Buffer saturated on {{ $labels.instance }}"
        description: "The DRBD connection's TCP send buffer has remained over 80% full for 10 minutes, indicating systemic network bandwidth exhaustion."

    - alert: DRBDApplicationInFlightHigh
      expr: drbd_connection_apinflight_bytes > 536870912 # 500 MB
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "Excessive unacknowledged application I/O on {{ $labels.instance }}"
        description: "Over 500MB of application data is in flight to the peer. The peer storage or network link is severely bottlenecking synchronous replication."


  - name: drbd_device_diagnostics
    rules:
    - alert: DRBDDisklessCatastrophe
      expr: drbd_device_unintentionaldiskless == 1 and drbd_device_client == 0
      for: 30s
      labels:
        severity: critical
      annotations:
        summary: "DRBD physical backing device failed on {{ $labels.instance }}"
        description: "The underlying storage device has suffered an I/O fault and detached. The node is operating in a failed diskless state."

    - alert: DRBDDeviceStateDegraded
      expr: drbd_device_state{drbd_device_state="Failed"} == 1 or drbd_device_state{drbd_device_state="Inconsistent"} == 1
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: "DRBD local disk state is degraded on {{ $labels.instance }}"
        description: "The local block device is marked as {{ $labels.drbd_device_state }}. Data redundancy is compromised."

    - alert: DRBDActivityLogFrozen
      expr: drbd_device_alsuspended == 1
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: "DRBD metadata Activity Log suspended on {{ $labels.instance }}"
        description: "The metadata sector is unwritable, suspending the Activity Log. All application writes are blocked to prevent corruption."

    - alert: DRBDLocalDiskLatency
      expr: drbd_device_lowerpending > 1000 and drbd_device_upperpending > 1000
      for: 3m
      labels:
        severity: warning
      annotations:
        summary: "Severe local disk latency on {{ $labels.instance }}"
        description: "Over 1000 requests are queued from DRBD to the local block layer, causing upstream application I/O to stall."

  - name: drbd_cluster_consensus
    rules:
    - alert: DRBDQuorumLost
      expr: drbd_device_quorum == 0
      for: 15s
      labels:
        severity: critical
      annotations:
        summary: "DRBD Quorum lost for resource on {{ $labels.instance }}"
        description: "The node has lost contact with the cluster majority. To prevent split-brain data corruption, the volume has likely suspended I/O."

    - alert: DRBDResourceSuspended
      expr: drbd_resource_suspended == 1
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: "DRBD Resource is fully suspended on {{ $labels.instance }}"
        description: "All read and write system calls are blocked. Applications utilizing this block device will hang in D-state."

    - alert: DRBDStuckSynchronization
      expr: drbd_peerdevice_outofsync_bytes > 0 and rate(drbd_connection_rsinflight_bytes[5m]) == 0
      for: 15m
      labels:
        severity: critical
      annotations:
        summary: "DRBD background synchronization is stalled on {{ $labels.instance }}"
        description: "Data divergence exists between peers ({{ $value }} bytes), but no resynchronization traffic has been detected for 5 minutes."