mirror of
https://0xacab.org/liberate/backupninja.git
synced 2024-09-19 23:21:40 +02:00
eff97c8235
ssh identity file, defaulting to the expected /root/.ssh/id_rsa. This is important to have if you are trying to backup to a different user on a backuphost than other handlers are using. For example, if the rdiff handler is connecting to backalbatross@backupserver-pn using /root/.ssh/id_rsa for public key authentication, and you want to configure the maildir handler to connect as user backmaildir@backupserver-pn you will might want to specify a different identity file for that authentication (if you do not want to share the id file between the 'backalbatross' and 'backmaildir' users). Note: this cannot be accomplished with a ssh config file because that file is host based, ie. you can only specify one user per host specified in the config
351 lines
10 KiB
Plaintext
351 lines
10 KiB
Plaintext
# -*- mode: sh; sh-basic-offset: 3; indent-tabs-mode: nil; -*-
|
|
|
|
###############################################################
|
|
#
|
|
# This handler slowly creates a backup of each user's maildir
|
|
# to a remote server. It is designed to be run with low overhead
|
|
# in terms of cpu and bandwidth so it runs pretty slow.
|
|
# Hardlinking is used to save storage space.
|
|
#
|
|
# each users maildir will contain these files:
|
|
# daily.1
|
|
# daily.2
|
|
# daily.3
|
|
# weekly.1
|
|
# weekly.2
|
|
# monthly.1
|
|
# if keepdaily is 3, keepweekly is 2, and keepmonthly is 1.
|
|
# the actual maildir is stored within each snapshot directory.
|
|
#
|
|
# The basic algorithm is to rsync each maildir individually,
|
|
# and to use hard links for retaining historical data.
|
|
#
|
|
# We handle each maildir individually because it becomes very
|
|
# unweldy to hardlink and rsync many hundreds of thousands
|
|
# of files at once. It is much faster to take on smaller
|
|
# chunks at a time.
|
|
#
|
|
# For the backup rotation to work, destuser must be able to run
|
|
# arbitrary bash commands on the desthost.
|
|
#
|
|
# Any maildir which is deleted from the source will be moved to
|
|
# "deleted" directory in the destination. It is up to you to
|
|
# periodically remove this directory or old maildirs in it.
|
|
#
|
|
##############################################################
|
|
|
|
getconf rotate yes
|
|
getconf remove yes
|
|
getconf backup yes
|
|
|
|
getconf loadlimit 5
|
|
getconf speedlimit 0
|
|
getconf keepdaily 5
|
|
getconf keepweekly 3
|
|
getconf keepmonthly 1
|
|
|
|
getconf srcdir /var/maildir
|
|
getconf destdir
|
|
getconf desthost
|
|
getconf destport 22
|
|
getconf destuser
|
|
getconf destid_file /root/.ssh/id_rsa
|
|
|
|
getconf multiconnection notset
|
|
|
|
letters="0 1 2 3 4 5 6 7 8 9 a b c d e f g h i j k l m n o p q r s t u v w x y z"
|
|
failedcount=0
|
|
# strip trailing /
|
|
destdir=${destdir%/}
|
|
srcdir=${srcdir%/}
|
|
|
|
[ -d $srcdir ] || fatal "source directory $srcdir doesn't exist"
|
|
|
|
[ "$multiconnection" == "notset" ] && fatal "The maildir handler uses a very different destination format. See the example .maildir for more information"
|
|
|
|
if [ $test ]; then
|
|
testflags="--dry-run -v"
|
|
fi
|
|
|
|
rsyncflags="$testflags -e 'ssh -p $destport -i $destid_file' -r -v --ignore-existing --delete --size-only --bwlimit=$speedlimit"
|
|
excludes="--exclude '.Trash/\*' --exclude '.Mistakes/\*' --exclude '.Spam/\*'"
|
|
|
|
##################################################################
|
|
### FUNCTIONS
|
|
|
|
function do_user() {
|
|
local user=$1
|
|
local btype=$2
|
|
local letter=${user:0:1}
|
|
local source="$srcdir/$letter/$user/"
|
|
local target="$destdir/$letter/$user/$btype.1"
|
|
if [ ! -d $source ]; then
|
|
warning "maildir $source not found"
|
|
return
|
|
fi
|
|
|
|
debug "syncing"
|
|
ret=`$RSYNC -e "ssh -p $destport -i $destid_file" -r \
|
|
--links --ignore-existing --delete --size-only --bwlimit=$speedlimit \
|
|
--exclude '.Trash/*' --exclude '.Mistakes/*' --exclude '.Spam/*' \
|
|
$source $destuser@$desthost:$target \
|
|
2>&1`
|
|
ret=$?
|
|
# ignore 0 (success) and 24 (file vanished before it could be copied)
|
|
if [ $ret != 0 -a $ret != 24 ]; then
|
|
warning "rsync $user failed"
|
|
warning " returned: $ret"
|
|
let "failedcount = failedcount + 1"
|
|
if [ $failedcount -gt 100 ]; then
|
|
fatal "100 rsync errors -- something is not working right. bailing out."
|
|
fi
|
|
fi
|
|
ssh -o PasswordAuthentication=no $desthost -l $destuser -i $destid_file "date +%c%n%s > $target/created"
|
|
}
|
|
|
|
# remove any maildirs from backup which might have been deleted
|
|
# and add new ones which have just been created.
|
|
# (actually, it just moved them to the directory "deleted")
|
|
|
|
function do_remove() {
|
|
local tmp1=`maketemp maildir-tmp-file`
|
|
local tmp2=`maketemp maildir-tmp-file`
|
|
|
|
ssh -p $destport -i $estid_file $destuser@$desthost mkdir -p "$destdir/deleted"
|
|
for i in 0 1 2 3 4 5 6 7 8 9 a b c d e f g h i j k l m n o p q r s t u v w x y z; do
|
|
ls -1 "$srcdir/$i/" | sort > $tmp1
|
|
ssh -p $destport $destuser@$desthost ls -1 "$destdir/$i/" | sort > $tmp2
|
|
for deluser in `join -v 2 $tmp1 $tmp2`; do
|
|
[ "$deluser" != "" ] || continue
|
|
info "removing $destuser@$desthost:$destdir/$i/$deluser/"
|
|
ssh -p $destport -i $destid_file $destuser@$desthost mv "$destdir/$i/$deluser/" "$destdir/deleted"
|
|
ssh -p $destport -i $destid_file $destuser@$desthost "date +%c%n%s > '$destdir/$i/$deluser/deleted_on'"
|
|
done
|
|
done
|
|
rm $tmp1
|
|
rm $tmp2
|
|
}
|
|
|
|
function do_rotate() {
|
|
[ "$rotate" == "yes" ] || return;
|
|
local user=$1
|
|
local letter=${user:0:1}
|
|
local backuproot="$destdir/$letter/$user"
|
|
(
|
|
ssh -T -o PasswordAuthentication=no $desthost -l $destuser -i $destid_file <<EOF
|
|
##### BEGIN REMOTE SCRIPT #####
|
|
seconds_daily=86400
|
|
seconds_weekly=604800
|
|
seconds_monthly=2628000
|
|
keepdaily=$keepdaily
|
|
keepweekly=$keepweekly
|
|
keepmonthly=$keepmonthly
|
|
now=\`date +%s\`
|
|
|
|
if [ ! -d "$backuproot" ]; then
|
|
echo "Debug: skipping rotate of $user. $backuproot doesn't exist."
|
|
exit
|
|
fi
|
|
for rottype in daily weekly monthly; do
|
|
seconds=\$((seconds_\${rottype}))
|
|
|
|
dir="$backuproot/\$rottype"
|
|
if [ ! -d \$dir.1 ]; then
|
|
echo "Debug: \$dir.1 does not exist, skipping."
|
|
continue 1
|
|
elif [ ! -f \$dir.1/created ]; then
|
|
echo "Warning: \$dir.1/created does not exist. This backup may be only partially completed. Skipping rotation."
|
|
continue 1
|
|
fi
|
|
|
|
# Rotate the current list of backups, if we can.
|
|
oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1\`
|
|
#echo "Debug: oldest \$oldest"
|
|
[ "\$oldest" == "" ] && oldest=0
|
|
for (( i=\$oldest; i > 0; i-- )); do
|
|
if [ -d \$dir.\$i ]; then
|
|
if [ -f \$dir.\$i/created ]; then
|
|
created=\`tail -1 \$dir.\$i/created\`
|
|
else
|
|
created=0
|
|
fi
|
|
cutoff_time=\$(( now - (seconds*(i-1)) ))
|
|
if [ ! \$created -gt \$cutoff_time ]; then
|
|
next=\$(( i + 1 ))
|
|
if [ ! -d \$dir.\$next ]; then
|
|
echo "Debug: \$rottype.\$i --> \$rottype.\$next"
|
|
mv \$dir.\$i \$dir.\$next
|
|
date +%c%n%s > \$dir.\$next/rotated
|
|
else
|
|
echo "Debug: skipping rotation of \$dir.\$i because \$dir.\$next already exists."
|
|
fi
|
|
else
|
|
echo "Debug: skipping rotation of \$dir.\$i because it was created" \$(( (now-created)/86400)) "days ago ("\$(( (now-cutoff_time)/86400))" needed)."
|
|
fi
|
|
fi
|
|
done
|
|
done
|
|
|
|
max=\$((keepdaily+1))
|
|
if [ \( \$keepweekly -gt 0 -a -d $backuproot/daily.\$max \) -a ! -d $backuproot/weekly.1 ]; then
|
|
echo "Debug: daily.\$max --> weekly.1"
|
|
mv $backuproot/daily.\$max $backuproot/weekly.1
|
|
date +%c%n%s > $backuproot/weekly.1/rotated
|
|
fi
|
|
|
|
max=\$((keepweekly+1))
|
|
if [ \( \$keepmonthly -gt 0 -a -d $backuproot/weekly.\$max \) -a ! -d $backuproot/monthly.1 ]; then
|
|
echo "Debug: weekly.\$max --> monthly.1"
|
|
mv $backuproot/weekly.\$max $backuproot/monthly.1
|
|
date +%c%n%s > $backuproot/monthly.1/rotated
|
|
fi
|
|
|
|
for rottype in daily weekly monthly; do
|
|
max=\$((keep\${rottype}+1))
|
|
dir="$backuproot/\$rottype"
|
|
oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1\`
|
|
[ "\$oldest" == "" ] && oldest=0
|
|
# if we've rotated the last backup off the stack, remove it.
|
|
for (( i=\$oldest; i >= \$max; i-- )); do
|
|
if [ -d \$dir.\$i ]; then
|
|
if [ -d $backuproot/rotate.tmp ]; then
|
|
echo "Debug: removing rotate.tmp"
|
|
rm -rf $backuproot/rotate.tmp
|
|
fi
|
|
echo "Debug: moving \$rottype.\$i to rotate.tmp"
|
|
mv \$dir.\$i $backuproot/rotate.tmp
|
|
fi
|
|
done
|
|
done
|
|
####### END REMOTE SCRIPT #######
|
|
EOF
|
|
) | (while read a; do passthru $a; done)
|
|
|
|
}
|
|
|
|
|
|
function setup_remote_dirs() {
|
|
local user=$1
|
|
local backuptype=$2
|
|
local letter=${user:0:1}
|
|
local dir="$destdir/$letter/$user/$backuptype"
|
|
local tmpdir="$destdir/$letter/$user/rotate.tmp"
|
|
(
|
|
ssh -T -o PasswordAuthentication=no $desthost -l $destuser -i $destid_file <<EOF
|
|
if [ ! -d $destdir ]; then
|
|
echo "Fatal: Destination directory $destdir does not exist on host $desthost."
|
|
exit 1
|
|
elif [ -d $dir.1 ]; then
|
|
if [ -f $dir.1/created ]; then
|
|
echo "Warning: $dir.1 already exists. Overwriting contents."
|
|
else
|
|
echo "Warning: we seem to be resuming a partially written $dir.1"
|
|
fi
|
|
else
|
|
if [ -d $tmpdir ]; then
|
|
mv $tmpdir $dir.1
|
|
if [ \$? == 1 ]; then
|
|
echo "Fatal: could mv $destdir/rotate.tmp $dir.1 on host $desthost"
|
|
exit 1
|
|
fi
|
|
else
|
|
mkdir --parents $dir.1
|
|
if [ \$? == 1 ]; then
|
|
echo "Fatal: could not create directory $dir.1 on host $desthost"
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [ -d $dir.2 ]; then
|
|
echo "Debug: update links $backuptype.2 --> $backuptype.1"
|
|
cp -alf $dir.2/. $dir.1
|
|
#if [ \$? == 1 ]; then
|
|
# echo "Fatal: could not create hard links to $dir.1 on host $desthost"
|
|
# exit 1
|
|
#fi
|
|
fi
|
|
fi
|
|
[ -f $dir.1/created ] && rm $dir.1/created
|
|
[ -f $dir.1/rotated ] && rm $dir.1/rotated
|
|
exit 0
|
|
EOF
|
|
) | (while read a; do passthru $a; done)
|
|
|
|
if [ $? == 1 ]; then exit; fi
|
|
}
|
|
|
|
function start_mux() {
|
|
if [ "$multiconnection" == "yes" ]; then
|
|
debug "Starting dummy ssh connection"
|
|
ssh -p $destport -i $destid_file $destuser@$desthost sleep 1d &
|
|
sleep 1
|
|
fi
|
|
}
|
|
|
|
function end_mux() {
|
|
if [ "$multiconnection" == "yes" ]; then
|
|
debug "Stopping dummy ssh connection"
|
|
ssh -p $destport -i $destid_file $destuser@$desthost pkill sleep
|
|
fi
|
|
}
|
|
|
|
###
|
|
##################################################################
|
|
|
|
# see if we can login
|
|
debug "ssh -o PasswordAuthentication=no $desthost -l $destuser -i $destid_file 'echo -n 1'"
|
|
if [ ! $test ]; then
|
|
result=`ssh -o PasswordAuthentication=no $desthost -l $destuser -i $destid_file 'echo -n 1' 2>&1`
|
|
if [ "$result" != "1" ]; then
|
|
fatal "Can't connect to $desthost as $destuser using $destid_file."
|
|
fi
|
|
fi
|
|
|
|
end_mux
|
|
start_mux
|
|
|
|
## SANITY CHECKS ##
|
|
status=`ssh -p $destport -i $destid_file $destuser@$desthost "[ -d \"$destdir\" ] && echo 'ok'"`
|
|
if [ "$status" != "ok" ]; then
|
|
end_mux
|
|
fatal "Destination directory $destdir doesn't exist!"
|
|
exit
|
|
fi
|
|
|
|
### REMOVE OLD MAILDIRS ###
|
|
|
|
if [ "$remove" == "yes" ]; then
|
|
do_remove
|
|
fi
|
|
|
|
### MAKE BACKUPS ###
|
|
|
|
if [ "$backup" == "yes" ]; then
|
|
if [ $keepdaily -gt 0 ]; then btype=daily
|
|
elif [ $keepweekly -gt 0 ]; then btype=weekly
|
|
elif [ $keepmonthly -gt 0 ]; then btype=monthly
|
|
else fatal "keeping no backups"; fi
|
|
|
|
if [ "$testuser" != "" ]; then
|
|
cd "$srcdir/${user:0:1}"
|
|
do_rotate $testuser
|
|
setup_remote_dirs $testuser $btype
|
|
do_user $testuser $btype
|
|
else
|
|
for i in $letters; do
|
|
[ -d "$srcdir/$i" ] || fatal "directory $srcdir/$i not found."
|
|
cd "$srcdir/$i"
|
|
debug $i
|
|
for user in `ls -1`; do
|
|
[ "$user" != "" ] || continue
|
|
debug $user
|
|
do_rotate $user
|
|
setup_remote_dirs $user $btype
|
|
do_user $user $btype
|
|
done
|
|
done
|
|
fi
|
|
fi
|
|
|
|
end_mux
|
|
|