Merge branch 'borg_support' into 'master'

Borg support (rebased)

See merge request riseuplabs/backupninja!10
This commit is contained in:
lavamind 2018-06-27 17:30:54 -07:00
commit 177b4267e9
11 changed files with 765 additions and 7 deletions

2
.gitignore vendored
View File

@ -20,6 +20,8 @@ etc/cron.d/backupninja
etc/logrotate.d/backupninja
examples/Makefile
handlers/Makefile
handlers/borg
handlers/borg.helper
handlers/dup
handlers/dup.helper
handlers/ldap

View File

@ -59,3 +59,5 @@ Daniel Lo Nigro <daniel@dan.cx> -- Dropbox support for Duplicity
Matthijs Wensveen <matthijs.wensveen@gmail.com> -- fix symmetric encryption in dup handler
ulrich <ulrich@habmalnefrage.de> -- Added validation check for when
Romain Dessort <romain@univers-libre.net> -- Fix list of devices when dumping partition tables
Guillaume Subiron <ben@wainei.net> -- borg handler
Jerome Charaoui <jerome@riseup.net> -- borg handler

View File

@ -2,6 +2,9 @@ version 1.0.3 -- UNRELEASED
documentation changes
· Fix typos in README.md and manpages. Thank you, Lintian!
handler changes
borg:
. Added initial support
version 1.0.2 -- September 05, 2017
handler changes

View File

@ -15,7 +15,7 @@ Requirements:
Recommended:
rdiff-backup duplicity rsync gzip hwinfo sfdisk cryptsetup flashrom hwinfo
rdiff-backup duplicity rsync borgbackup gzip hwinfo sfdisk cryptsetup flashrom hwinfo
To install backupninja, simply do the following:

View File

@ -33,7 +33,7 @@ The following backup types are supported:
with an unpriviledged backup user
- backup of mysql databases (via mysqlhotcopy and mysqldump)
- basic system and hardware info
- encrypted remote backups (via duplicity)
- encrypted remote backups (via duplicity or borgbackup)
- backup of subversion repositories
Installation
@ -107,6 +107,7 @@ file in `/etc/backup.d` according to the file's suffix:
- `.sh`: run this file as a shell script.
- `.rdiff`: filesystem backup (using rdiff-backup)
- `.dup`: filesystem backup (using duplicity)
- `.borg`: filesystem backup (using borg)
- `.mysql`: backup mysql databases
- `.pgsql`: backup PostgreSQL databases
- `.sys`: general hardware, partition, and system reports.

View File

@ -1,7 +1,7 @@
EXAMPLES = example.dup example.maildir example.makecd example.mysql \
example.pgsql example.rdiff example.rsync example.sh \
example.svn example.sys example.trac
EXAMPLES = example.borg example.dup example.maildir example.makecd \
example.mysql example.pgsql example.rdiff example.rsync \
example.sh example.svn example.sys example.trac
EXTRA_DIST = $(EXAMPLES)

182
examples/example.borg Normal file
View File

@ -0,0 +1,182 @@
##
## This is an example borgbackup configuration file.
##
## Here you can find all the possible borgbackup options, details of
## what the options provide and possible settings. The defaults are set
## as the commented out option, uncomment and change when
## necessary. Options which are uncommented in this example do not have
## defaults, and the settings provided are recommended.
##
## The defaults are useful in most cases, just make sure to configure the
## destination host and user.
##
## default is 0, but set to 19 if you want to lower the priority.
## an example setting would be:
## nicelevel = 19
##
## Default
# nicelevel = 0
## default is yes. set to no to skip the test if the remote host is alive
##
## Default:
# testconnect = yes
## default is not to limit bandwidth.
## set to a number in kiBytes/second to limit bandwidth usage.
##
## Default:
# bwlimit = 0
######################################################
## source section
## (where the files to be backed up are coming from)
[source]
## default is to initialize the backup repository if absent
## set to no to skip this step
##
## Default:
# init = yes
## A few notes about includes and excludes:
## 1. include paths do not support any kind of pattern matching
## 2. exclude paths support several types of pattern matching, the default being
## shell-style matching, where, for example, '*' matches any number of
## characters. for more info, see : borg help patterns
## 3. Symlinks are preserved (not followed).
##
## for more info see : borg help patterns
##
## files to include in the backup
include = /opt
include = /srv
include = /etc
include = /root
include = /home
include = /usr/local
include = /var
## files to exclude from the backup
exclude = /var/lock
exclude = /var/run
exclude = /var/cache
exclude = /var/tmp
exclude = /var/lib/mongodb/journal
exclude = /var/lib/clamav
exclude = /var/lib/mlocate
exclude = /var/lib/postgresql
exclude = /var/lib/mysql
## define extra command-line options for the "borg create" operation.
##
## Example:
## create_options = --exclude-caches
##
## for more info see : borg help create
##
## Default:
# create_options =
## whether to prune (remove) older backups
##
## Default:
# prune = yes
## keep all backups within this time frame.
## must be defined as a number followed by one of the
## following characters: "H", "d", "w", "m", "y"
##
## this option will be ignored if set to 0
##
## the default is to keep all backups made within the
## last 30 days
##
## Default:
# keep = 30d
## define extra command-line options for the "borg prune" operation.
##
## Example:
## prune_options = --keep-daily=7 --keep-weekly=4 --keep-monthly=6
##
## for more info see : borg help prune
##
## Default:
# prune_options =
######################################################
## destination section
## (where the files are copied to)
[dest]
## put the backups under this directory, this must be set!
## an example setting would be:
## directory = /backups
##
## Default:
# directory =
## the machine which will receive the backups.
## an example setting would be:
## host = backuphost
##
## set host = localhost for local backups (no ssh)
##
## Default
# host =
## make the files owned by this user. you must be able to
## `su -c "ssh backupuser@backhost"` without specifying a password.
## an example setting would be:
## user = backupuser
##
## Default:
# user =
## archive name, should be unique every day.
## Format tags available :
## {now}, {utcnow}, {fqdn}, {hostname}, {user}, {pid}
##
## for more info see : borg help create
##
## Default:
# archive = {now:%Y-%m-%dT%H:%M:%S}
## compression algorithm
## can be "none", "lz4", "zstd[,L]", "zlib[,L]", "lzma[,L]", "auto,C[,L]".
## - "none" stands for "no compression"
## - "lz4" offers very high speed, very low compression
## - "zstd" is a modern wide-range algorithm
## - "zlib" offers medium speed, medium compression
## - "lzma" offers low speed, high compression
## - "auto,C" will use a heuristic to decide whether to compress using
## selected algorithm C
##
## for more info see : borg help compression
##
## Default:
# compression = lz4
## encryption mode to use for repository creation
##
## common options are "none", "repokey" or "keyfile"
## - "none" = no encryption
## - "repokey" = encryption with key stored inside the repository
## - "keyfile" = encryption with key stored in ~/.config/borg/keys/
##
## for other options and more info, see : borg help init
##
## Default:
# encryption = none
## passphrase of the key used for repository encryptions
##
## must be set if encryption is not "none"
## encryption is disabled by default
##
## Default:
# passphrase =

View File

@ -1,9 +1,9 @@
HANDLERS = dup dup.helper maildir makecd \
HANDLERS = borg borg.helper dup dup.helper maildir makecd \
makecd.helper mysql mysql.helper pgsql pgsql.helper rdiff \
rdiff.helper rsync sh svn sys sys.helper trac tar tar.helper
DIST_HANDLERS = dup.in dup.helper.in maildir.in makecd.in \
DIST_HANDLERS = borg.in borg.helper.in dup.in dup.helper.in maildir.in makecd.in \
makecd.helper.in mysql.in mysql.helper.in pgsql.in pgsql.helper.in rdiff.in \
rdiff.helper.in rsync.in sh.in svn.in sys.in sys.helper.in trac.in tar.in tar.helper.in wget
@ -18,6 +18,14 @@ edit = sed \
pkgdata_DATA = $(HANDLERS)
borg: $(srcdir)/borg.in
rm -f borg
$(edit) $(srcdir)/borg.in > borg
borg.helper: $(srcdir)/borg.helper.in
rm -f borg.helper
$(edit) $(srcdir)/borg.helper.in > borg.helper
dup: $(srcdir)/dup.in
rm -f dup
$(edit) $(srcdir)/dup.in > dup

389
handlers/borg.helper.in Normal file
View File

@ -0,0 +1,389 @@
# -*- mode: sh; sh-basic-offset: 3; indent-tabs-mode: nil; -*-
# vim: set filetype=sh sw=3 sts=3 expandtab autoindent:
#
# Copyright 2016 Benjamin Maisonnas <ben@wainei.net>
#
# This work is free. You can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
HELPERS="$HELPERS borg:deduplicated_archive_based_backup"
declare -a borg_includes
declare -a borg_excludes
# FUNCTIONS
do_borg_host_includes() {
set -o noglob
# choose the files to backup
REPLY=
while [ -z "$REPLY" ]; do
formBegin "$borg_title - host system: includes"
for ((i=0; i < ${#borg_includes[@]} ; i++)); do
formItem include ${borg_includes[$i]}
done
formItem include
formItem include
formItem include
formItem include
formItem include
formItem include
formItem include
formItem include
formDisplay
[ $? = 0 ] || return
unset borg_includes
borg_includes=($REPLY)
done
set +o noglob
}
do_borg_excludes() {
set -o noglob
formBegin "$borg_title: host system: excludes"
for ((i=0; i < ${#borg_excludes[@]} ; i++))
do
formItem exclude ${borg_excludes[$i]}
done
formItem exclude
formItem exclude
formItem exclude
formItem exclude
formItem exclude
formItem exclude
formItem exclude
formItem exclude
formDisplay
[ $? = 0 ] || return
unset borg_excludes
borg_excludes=($REPLY)
set +o noglob
}
do_borg_src() {
do_borg_host_includes
[ $? = 0 ] || return 1
do_borg_excludes
[ $? = 0 ] || return 1
_src_done="(DONE)"
setDefault dest
}
do_borg_dest() {
declare -a tmp_array
set -o noglob
REPLY=
while [ -z "$REPLY" -o -z "$borg_directory" -o -z "$borg_host" -o -z "$borg_user" -o -z "$borg_archive" -o -z "$borg_compression" ]
do
formBegin "$borg_title - destination"
formItem "directory" "$borg_directory"
formItem "host" "$borg_host"
formItem "user" "$borg_user"
formItem "archive_name" "$borg_archive"
formItem "compression" "$borg_compression"
formDisplay
[ $? = 0 ] || return
tmp_array=($REPLY)
borg_directory=${tmp_array[0]}
borg_host=${tmp_array[1]}
borg_user=${tmp_array[2]}
borg_archive=${tmp_array[3]}
borg_compression=${tmp_array[4]}
done
set +o noglob
_dest_done="(DONE)"
setDefault conn
}
do_borg_enc() {
radioBox "$borg_title" "Encryption mode" \
"none" "no encryption and no authentication" on \
"repokey" "encryption with a passphrase" off
[ $? = 1 ] && return;
borg_encryption="$REPLY"
if [ "$borg_encryption" = "repokey" ]; then
local question="Enter the passphrase needed to encrypt/decrypt the repository:"
REPLY=
while [ -z "$REPLY" -o -z "$borg_passphrase" ]; do
passwordBox "$borg_title - " "$question"
[ $? = 0 ] || return 1
borg_passphrase="$REPLY"
done
fi
_enc_done="(DONE)"
setDefault prune
}
do_borg_local_dir() {
local dir_status="ok"
IFS=$' \t\n'
if [ "$_dest_done" = "" ]; then
msgBox "$borg_title: error" "You must first configure the destination."
return 1
fi
echo "Testing to see if the borg backup directory exists and is writable"
if [ test -d "$borg_directory" ]; then
if [ test -w "$borg_directory" ]; then
msgBox "destination directory is not writable!" "The destination directory is not writable by the user you specified. Please fix the permissions on the directory and then try again."
dir_status=failed
fi
else
booleanBox "Destination does not exist" "The destination backup directory does not exist, do you want me to create it for you?"
if [ $? = 0 ]; then
if mkdir -p "$borg_directory"; then
msgBox "$borg_title: success" "Creation of the destination directory was a success!"
else
msgBox "$borg_title: error" "Creation of the destination directory failed, check the directory permissions."
dir_status=failed
fi
fi
fi
[ "$dir_status" = "ok" ] || return 1
}
do_borg_ssh_con() {
local remote_status="ok"
IFS=$' \t\n'
if [ "$_dest_done" = "" ]; then
msgBox "$borg_title: error" "You must first configure the destination."
return 1
elif [ "$borg_user" = "" ]; then
msgBox "$borg_title: error" "You must first configure the destination user."
return 1
elif [ "$borg_host" = "" ]; then
msgBox "$borg_title: error" "You must first configure the destination host."
return 1
else
booleanBox "$borg_title" "This step will create a ssh key for the local root user with no passphrase (if one does not already exist), and attempt to copy root's public ssh key to authorized_keys file of $borg_user@$borg_host. This will allow the local root to make unattended backups to $borg_user@$borg_host.\n\n\nAre you sure you want to continue?"
[ $? = 0 ] || return 1
fi
if [ ! -f /root/.ssh/id_dsa.pub -a ! -f /root/.ssh/id_rsa.pub ]; then
echo "Creating local root's ssh key"
ssh-keygen -t rsa -b 4096 -f /root/.ssh/id_rsa -N ""
echo "Done. hit return to continue"
read
fi
ssh -o PreferredAuthentications=publickey $borg_host -l $borg_user "exit" 2> /dev/null
if [ $? -ne 0 ]; then
echo "Copying root's public ssh key to authorized_keys of $borg_user@$borg_host. When prompted, specify the password for user $borg_user@$borg_host."
pubkeys=( /root/.ssh/id_[rd]sa.pub )
if ! ssh-copy-id -i ${pubkeys[0]} $borg_user@$borg_host; then
echo "FAILED: Couldn't copy root's public ssh key to authorized_keys of $borg_user@$borg_host."
ssh $borg_user@$borg_host 'test -w .ssh || test -w .'
result=$?
echo "Hit return to continue."
read
case $result in
0 ) msgBox "$borg_title: error" "Directories are writable: Probably just a typo the first time." ;;
1 ) msgBox "$borg_title: error" "Connected successfully to $borg_user@$borg_host, but unable to write. Check ownership and modes of ~$borg_user on $borg_host." ;;
255 ) msgBox "$borg_title: error" "Failed to connect to $borg_user@$borg_host. Check hostname, username, and password. Also, make sure sshd is running on the destination host." ;;
* ) msgBox "$borg_title: error" "Unexpected error (return code ${result})." ;;
esac
return
else
echo "Done. hit return to continue"
read
fi
else
echo "root@localhost is already in authorized_keys of $borg_user@$borg_host."
echo "Hit return to continue."
read
fi
# test to see if the remote borg backup directory exists and is writable
echo "Testing to see if remote borg backup directory exists and is writable"
ssh $borg_user@$borg_host "test -d ${borg_directory}"
if [ $? = 0 ]; then
ssh $borg_user@$borg_host "test -w $borg_directory"
if [ $? != 0 ]; then
msgBox "destination directory is not writable!" "The remote destination directory is not writable by the user you specified. Please fix the permissions on the directory and then try again."
remote_status=failed
fi
else
booleanBox "Remote directory does not exist" "The destination backup directory does not exist, do you want me to create it for you?"
if [ $? = 0 ]; then
ssh $borg_user@$borg_host "mkdir -p ${borg_directory}"
result=$?
case $result in
0) msgBox "$borg_title: success" "Creation of the remote destination directory was a success!";;
1) msgBox "$borg_title: error" "Connected successfully to $borg_user@$borg_host, but was unable to create the destination directory, check the directory permissions."
remote_status=failed;;
255) msgBox "$borg_title: error" "Failed to connect to $borg_user@$borg_host. Check hostname, username, and password. Also, make sure sshd is running on the destination host."
remote_status=failed;;
*) msgBox "$borg_title: error" "Unexpected error."
remote_status=failed;;
esac
fi
fi
[ "$remote_status" = "ok" ] || return 1
}
do_borg_con() {
echo "Checking for local install of borg"
which borg
if [ $? != 0 ]; then
msgBox "$borg_title: error" "borg executable not found, please install borg ($borg_docs/installation.html)."
return
fi
if [ "$borg_host" != "localhost" ]; then
do_borg_ssh_con
else
do_borg_local_dir
fi
[ $? = 0 ] || return
echo "SUCCESS: Everything looks good!"
echo "Hit return to continue."
read
_con_done="(DONE)"
setDefault enc
}
do_borg_prune() {
radioBox "$borg_title" "pruning (how many backups to keep" \
"yes" "regularly prune old backups" on \
"no" "keep all backups" off
[ $? = 1 ] && return;
borg_prune="$REPLY"
if [ "$borg_prune" = "yes" ]; then
declare -a tmp_array
set -o noglob
REPLY=
formBegin "$borg_title - keep all backups made within this number of days"
formItem "keep" "$borg_keep"
formDisplay
[ $? = 0 ] || return
tmp_array=($REPLY)
borg_keep=${tmp_array[0]}
set +o noglob
fi
_prune_done="(DONE)"
setDefault finish
}
do_borg_finish() {
get_next_filename $configdirectory/90.borg
cat > $next_filename <<EOF
## for more options see
## - example.borg
## - $borg_docs
[source]
EOF
## includes ##
set -o noglob
for ((i=0; i < ${#borg_includes[@]} ; i++)); do
echo "include = ${borg_includes[$i]}" >> $next_filename
done
set +o noglob
## excludes ##
set -o noglob
for ((i=0; i < ${#borg_excludes[@]} ; i++)); do
echo exclude = ${borg_excludes[$i]} >> $next_filename
done
set +o noglob
cat >> $next_filename <<EOF
## for more info see : borg prune -h
prune = $borg_prune
keep = "${borg_keep}d"
[dest]
directory = $borg_directory
host = $borg_host
user = $borg_user
archive = $borg_archive
compression = $borg_compression
encryption = $borg_encryption
passphrase = $borg_passphrase
EOF
chmod 600 $next_filename
}
borg_main_menu() {
while true; do
srcitem="choose files to include & exclude $_src_done"
destitem="configure backup destination $_dest_done"
conitem="test connection and destination dir $_con_done"
encitem="configure encryption mode $_enc_done"
pruneitem="configure pruning (optional) $_prune_done"
menuBox "$borg_title" "choose a step:" \
src "$srcitem" \
dest "$destitem" \
conn "$conitem" \
enc "$encitem" \
prune "$pruneitem" \
finish "finish and create config file"
[ $? = 0 ] || return
result="$REPLY"
case "$result" in
"src") do_borg_src;;
"dest") do_borg_dest;;
"conn") do_borg_con;;
"enc") do_borg_enc;;
"prune") do_borg_prune;;
"finish")
if [[ "$_con_done$_dest_done$_enc_done$_src_done" != "(DONE)(DONE)(DONE)(DONE)" ]]; then
msgBox "$borg_title" "You cannot create the configuration file until mandatory steps are completed."
else
do_borg_finish
return
fi
;;
esac
done
}
borg_wizard() {
# Global variables
borg_title="borg action wizard"
borg_docs="http://borgbackup.readthedocs.io/en/stable"
_src_done=
_dest_done=
_enc_done=
_con_done=
borg_directory=/backup/`hostname`
borg_user=root
borg_host=localhost
borg_archive='{now:%Y-%m-%dT%H:%M:%S}'
borg_compression=lz4
borg_encryption=none
borg_passphrase=
borg_keep=30
# Global variables whose '*' shall not be expanded
set -o noglob
borg_includes=(/var/spool/cron/crontabs /var/backups /etc /root /home /usr/local/*bin)
borg_excludes=(/home/*/.gnupg /home/*/.local/share/Trash /home/*/.Trash /home/*/.thumbnails)
set +o noglob
borg_main_menu
}

169
handlers/borg.in Normal file
View File

@ -0,0 +1,169 @@
# -*- mode: sh; sh-basic-offset: 3; indent-tabs-mode: nil; -*-
# vim: set filetype=sh sw=3 sts=3 expandtab autoindent:
#
# borg handler script for backupninja
# requires borgbackup
#
# Guillaume Subiron, Sysnove, 2016
#
# Copyright 2016 Guillaume Subiron <guillaume@sysnove.fr>
#
# This work is free. You can redistribute it and/or modify it under the
# terms of the Do What The Fuck You Want To Public License, Version 2,
# as published by Sam Hocevar. See the http://www.wtfpl.net/ file for more details.
#
#
export BORG_RELOCATED_REPO_ACCESS_IS_OK=yes
export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes
### GET CONFIG ###
getconf testconnect yes
getconf nicelevel 0
getconf bwlimit
setsection source
getconf init yes
getconf include
getconf exclude
getconf create_options
getconf prune yes
getconf keep 30d
getconf prune_options
setsection dest
getconf user
getconf host
getconf directory
# strip trailing /
directory=${directory%/}
getconf archive {now:%Y-%m-%dT%H:%M:%S}
getconf compression lz4
getconf encryption none
getconf passphrase
export BORG_PASSPHRASE="$passphrase"
### CHECK CONFIG ###
# destination specific checks
[ "$directory" != "" ] || fatal "Destination directory not set"
if [ "$host" != "localhost" ]; then
execstr_repository="ssh://${user}@${host}${directory}"
else
execstr_repository="$directory"
fi
execstr_archive="$archive"
# check the connection at the source and destination
[ -n "$test" ] || test=0
if [ "$host" != "localhost" ] && ([ "$testconnect" = "yes" ] || [ "${test}" -eq 1 ]); then
debug "ssh -o PasswordAuthentication=no $host -l $user 'echo -n 1'"
local ret=`ssh -o PasswordAuthentication=no $host -l $user 'echo -n 1'`
if [ "$ret" = 1 ]; then
debug "Connected to $host as $user successfully"
else
teststr="borg list --show-rc -v $execstr_repository"
debug "$teststr"
output=`su -c "$teststr" 2>&1`
if echo "$output" | grep "terminating with success status" ; then
debug "Connected to $host as $user successfully (forced command)"
else
if echo "$output" | grep -E "Repository.+does not exist" ; then
debug "Connected to $host as $user successfully (forced command)"
else
fatal "Can't connect to $host as $user."
fi
fi
fi
fi
### INIT IF NEEDED ###
if [ "$init" == "yes" ]; then
initstr="borg init --encryption=$encryption $execstr_repository"
debug "$initstr"
if [ $test = 0 ]; then
output="`su -c "$initstr" 2>&1`"
if [ $? = 2 ]; then
debug $output
info "Repository was already initialized"
else
warning $output
warning "Repository has been initialized"
fi
fi
fi
### EXECUTE ###
execstr="borg create --stats --compression $compression"
set -o noglob
# includes
SAVEIFS=$IFS
IFS=$(echo -en "\n\b")
for i in $include; do
includes="${includes} '$i'"
done
IFS=$SAVEIFS
# excludes
SAVEIFS=$IFS
IFS=$(echo -en "\n\b")
for i in $exclude; do
excludes="${excludes} --exclude '$i'"
done
IFS=$SAVEIFS
set +o noglob
if [ ! -z $bwlimit ]; then
execstr="${execstr} --remote-ratelimit=${bwlimit}"
fi
if [ ! -z $create_options ]; then
execstr="${execstr} ${create_options}"
fi
# include client-part and server-part
execstr="${execstr} ${excludes} $execstr_repository::$execstr_archive ${includes}"
debug "$execstr"
if [ $test = 0 ]; then
output=`nice -n $nicelevel su -c "$execstr" 2>&1`
if [ $? = 0 ]; then
debug $output
info "Successfully finished backing up source $label"
else
error $output
fatal "Failed backuping up source $label"
fi
fi
### REMOVE OLD BACKUPS ###
# borg prune
if [ "$prune" == "yes" ]; then
if [ ! "$keep" == "0" ]; then
prune_options="${prune_options} --keep-within=${keep}"
fi
prunestr="borg prune $prune_options $execstr_repository"
debug "$prunestr"
if [ $test = 0 ]; then
output="`su -c "$prunestr" 2>&1`"
if [ $? = 0 ]; then
debug $output
info "Removing old backups succeeded."
else
warning $output
warning "Failed removing old backups."
fi
fi
fi
unset BORG_PASSPHRASE
return 0

View File

@ -33,6 +33,8 @@ run this file as a shell script.
backup action for rdiff-backup.
.IP .dup
backup action for duplicity.
.IP .borg
backup action for borgbackup.
.IP .maildir
backup action for slow, incremental rsyncs of tens of thousands of maildirs.
.IP .mysql