From fa2528a083d98fe085471c9db39b792adb1e44c8 Mon Sep 17 00:00:00 2001
From: Micah Anderson <micah@riseup.net>
Date: Wed, 23 Jun 2021 13:52:23 -0400
Subject: [PATCH] Add basic prometheus metrics for backup status.

This requires that `prometheus-node-exporter` is installed.

If `reportprom` is set, then during the reporting phase, metrics will be written
to `${prom_textfile_dir}/backupninja.prom`, which will then be available for
scraping by prometheus.

The metrics simply report the hostname and the number of actions run, the number
of warnings, errors, fails, and halts that were produced in the backup run.

These are written to the prometheus-node-exporter directory in an atomic way.
---
 etc/backupninja.conf.in |  4 ++++
 src/backupninja.in      | 51 +++++++++++++++++++++++++++++++++++++++--
 2 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/etc/backupninja.conf.in b/etc/backupninja.conf.in
index f2366a3..60bd90d 100644
--- a/etc/backupninja.conf.in
+++ b/etc/backupninja.conf.in
@@ -13,6 +13,10 @@
 # 1 -- Fatal errors           (only)
 loglevel = 4
 
+# Produce prometheus metrics of backup status (default = no).
+# Requires `prometheus-node-exporter` to be installed
+reportprom = false
+
 # send a summary of the backup status to
 # this email address:
 reportemail = root
diff --git a/src/backupninja.in b/src/backupninja.in
index 10a8c66..8a95b62 100755
--- a/src/backupninja.in
+++ b/src/backupninja.in
@@ -513,6 +513,8 @@ setfile $conffile
 getconf configdirectory @CFGDIR@/backup.d
 getconf scriptdirectory @datadir@
 getconf reportdirectory
+getconf reportprom
+getconf prom_textfile_dir /var/lib/prometheus/node-exporter
 getconf reportemail
 getconf reporthost
 getconf reportspace
@@ -601,10 +603,10 @@ for file in $files; do
    fi
 done
 
-## mail the messages to the report address
+## reporting
 
 if [ $actions_run == 0 ]; then doit=0
-elif [ "$reportemail" == "" ]; then doit=0
+elif [ "$reportemail" == "" -a "$reportprom" == "" ]; then doit=0
 elif [ $fatals != 0 ]; then doit=1
 elif [ $errors != 0 ]; then doit=1
 elif [ $halts != 0 ]; then doit=1
@@ -614,6 +616,51 @@ else doit=0
 fi
 
 if [ $doit == 1 ]; then
+   if [ ! -z "$reportprom" ]; then
+      if [ -d "$prom_textfile_dir" ]; then
+         debug "reporting to prometheus"
+         hostname=`hostname`
+
+         # set some defaults, so there aren't empty strings
+         [ ! -z $warnings ] || warnings=0
+         [ ! -z $errors ] || errors=0
+         [ ! -z $fatals ] || fatals=0
+         [ ! -z $halts ] || halts=0
+         [ ! -z $actions_run ] || actions_run=0
+
+         output_file="${prom_textfile_dir}/backupninja.prom"
+         tmp_file="${output_file}.$$"
+         trap "rm -f $tmp_file 2>/dev/null" EXIT INT TERM
+         cat <<EOF > "$tmp_file"
+# HELP backupninja_warnings Number of warnings reported by Backupninja
+# TYPE backupninja_warnings gauge
+# HELP backupninja_errors Number of errors reported by Backupninja
+# TYPE backupninja_errors gauge
+# HELP backupninja_fatals Number of fatals reported by Backupninja
+# TYPE backupninja_fatals gauge
+# HELP backupninja_halts Number of halts reported by Backupninja
+# TYPE backupninja_halts gauge
+# HELP backupninja_actions Number of actions run by Backupninja
+# TYPE backupninja_actions gauge
+backupninja_warnings{host="$hostname"} $warnings
+backupninja_errors{host="$hostname"} $errors
+backupninja_fatals{host="$hostname"} $fatals
+backupninja_halts{host="$hostname"} $halts
+backupninja_actions{host="$hostname"} $actions_run
+EOF
+         if [ $? -gt 0 ]; then
+            rm -f "$tmp_file" 2>/dev/null
+            error "could not write metrics to ${prom_textfile_dir}!"
+            let "errors +-1"
+         else
+            mv -f "$tmp_file" "$output_file"
+            chmod 0644 "$output_file"
+         fi
+      else
+         error "$prom_textfile_dir does not exist!"
+         let "errors +-1"
+      fi
+   fi
    if [ -x "$(which mail 2>/dev/null)" ]; then
       debug "send report to $reportemail"
       hostname=`hostname`