mirror of
https://0xacab.org/liberate/backupninja.git
synced 2024-09-19 23:21:40 +02:00
85f4ea886a
The code added in the previous commit is only relevant for the mirror format. With the long format, directories are created correctly in the rotation routine. For the short format, the directory is initially created, but only for a local destination, so adding a short snippet to the rotation routine does it.
1184 lines
31 KiB
Bash
1184 lines
31 KiB
Bash
# -*- mode: sh; sh-basic-offset: 3; indent-tabs-mode: nil; -*-
|
|
# vim: set filetype=sh sw=3 sts=3 expandtab autoindent:
|
|
#
|
|
# backupninja handler for incremental backups using rsync and hardlinks
|
|
# feedback: rhatto at riseup.net
|
|
#
|
|
# rsync handler is free software; you can redistribute it and/or modify it
|
|
# under the terms of the GNU General Public License as published by the Free
|
|
# Software Foundation; either version 2 of the License, or any later version.
|
|
#
|
|
# rsync handler is distributed in the hope that it will be useful, but WITHOUT
|
|
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
# more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along with
|
|
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
|
|
# Place - Suite 330, Boston, MA 02111-1307, USA
|
|
#
|
|
# Inspiration
|
|
# -----------
|
|
#
|
|
# - http://www.mikerubel.org/computers/rsync_snapshots/
|
|
# - rsnap handler by paulv at bikkel.org
|
|
# - maildir handler from backupninja
|
|
#
|
|
# Config file options
|
|
# -------------------
|
|
#
|
|
# [general]
|
|
# log = rsync log file
|
|
# partition = partition where the backup lives
|
|
# fscheck = set to 1 if fsck should run on $partition after the backup is made
|
|
# read_only = set to 1 if $partition is mounted read-only
|
|
# mountpoint = backup partition mountpoint or backup main folder (either local or remote)
|
|
# backupdir = folder relative do $mountpoint where the backup should be stored (local or remote)
|
|
# format = specify backup storage format: short, long or mirror (i.e, no rotations)
|
|
# days = for short storage format, specify the number of backup increments (min = 2, set to 1 or less to disable)
|
|
# keepdaily = for long storage format, specify the number of daily backup increments
|
|
# keepweekly = for long storage format, specify the number of weekly backup increments
|
|
# keepmonthly = for long storage format, specify the number of monthly backup increments
|
|
# nicelevel = rsync command nice level
|
|
# ionicelevel = best-effort nice level applied to rsync and other commands
|
|
# enable_mv_timestamp_bug = set to "yes" if your system isnt handling timestamps correctly
|
|
# tmp = temp folder
|
|
# multiconnection = set to "yes" if you want to use multiconnection ssh support
|
|
#
|
|
# [source]
|
|
# from = local or remote
|
|
# host = source hostname or ip, if remote backup
|
|
# port = remote port number (remote source only)
|
|
# user = remote user name (remote source only)
|
|
# testconnect = when "yes", test the connection for a remote source before backup
|
|
# include = include folder on backup
|
|
# exclude = exclude folder on backup
|
|
# ssh = ssh command line (remote source only)
|
|
# protocol = ssh or rsync (remote source only)
|
|
# rsync = rsync program
|
|
# rsync_options = rsync command options
|
|
# numericids = when set to 1, use numeric ids instead of user/group mappings on rsync
|
|
# compress = if set to 1, compress data on rsync (remote source only)
|
|
# bandwidthlimit = set a bandwidth limit in KB/s (remote source only)
|
|
# remote_rsync = remote rsync program (remote source only)
|
|
# id_file = ssh key file (remote source only)
|
|
# batch = set to "yes" to rsync use a batch file as source
|
|
# batchbase = folder where the batch file is located
|
|
# filelist = set yes if you want rsync to use a file list source
|
|
# filelistbase = folder where the file list is placed
|
|
#
|
|
# [dest]
|
|
# dest = backup destination type (local or remote)
|
|
# testconnect = when "yes", test the connection for a remote source before backup
|
|
# ssh = ssh command line (remote dest only)
|
|
# protocol = ssh or rsync (remote dest only)
|
|
# numericids = when set to 1, use numeric ids instead of user/group mappings on rsync
|
|
# compress = if set to 1, compress data on rsync (remote source only)
|
|
# host = destination host name (remote destination only)
|
|
# port = remote port number (remote destination only)
|
|
# user = remote user name (remote destination only)
|
|
# id_file = ssh key file (remote destination only)
|
|
# bandwidthlimit = set a bandwidth limit in KB/s (remote destination only)
|
|
# remote_rsync = remote rsync program (remote dest only)
|
|
# batch = set to "yes" to rsync write a batch file from the changes
|
|
# batchbase = folder where the batch file should be written
|
|
# fakesuper = set to yes so rsync use the --fake-super flag (remote destination only)
|
|
#
|
|
# [services]
|
|
# initscripts = absolute path where scripts are located
|
|
# service = script name to be stoped at the begining of the backup and started at its end
|
|
#
|
|
# You can also specify some system comands if you don't want the default system values:
|
|
#
|
|
# [system]
|
|
# rm = rm command
|
|
# cp = cp command
|
|
# touch = touch command
|
|
# mv = mv command
|
|
# fsck = fsck command
|
|
#
|
|
|
|
# function definitions
|
|
|
|
function eval_config {
|
|
|
|
# system section
|
|
|
|
setsection system
|
|
getconf rm rm
|
|
getconf cp cp
|
|
getconf touch touch
|
|
getconf mv mv
|
|
getconf fsck fsck
|
|
|
|
# general section
|
|
|
|
setsection general
|
|
getconf log /var/log/backup/rsync.log
|
|
getconf partition
|
|
getconf fscheck
|
|
getconf read_only
|
|
getconf mountpoint
|
|
getconf backupdir
|
|
getconf format short
|
|
getconf days 7
|
|
getconf keepdaily 5
|
|
getconf keepweekly 3
|
|
getconf keepmonthly 1
|
|
getconf nicelevel 0
|
|
getconf ionicelevel
|
|
getconf enable_mv_timestamp_bug no
|
|
getconf tmp /tmp
|
|
getconf multiconnection no
|
|
|
|
# source section
|
|
|
|
setsection source
|
|
getconf from local
|
|
getconf rsync $RSYNC
|
|
getconf rsync_options "-av --delete --recursive"
|
|
|
|
if [ "$from" == "remote" ]; then
|
|
getconf testconnect no
|
|
getconf protocol ssh
|
|
getconf ssh ssh
|
|
getconf host
|
|
|
|
if [ "$protocol" == "ssh" ]; then
|
|
# sshd default listen port
|
|
getconf port 22
|
|
else
|
|
# rsyncd default listen port
|
|
getconf port 873
|
|
fi
|
|
|
|
getconf user
|
|
getconf bandwidthlimit
|
|
getconf remote_rsync rsync
|
|
getconf id_file /root/.ssh/id_rsa
|
|
fi
|
|
|
|
getconf batch no
|
|
|
|
if [ "$batch" == "yes" ]; then
|
|
getconf batchbase
|
|
if [ ! -z "$batchbase" ]; then
|
|
batch="read"
|
|
fi
|
|
fi
|
|
|
|
getconf filelist no
|
|
getconf filelistbase
|
|
getconf include
|
|
getconf exclude
|
|
getconf numericids 0
|
|
getconf compress 0
|
|
|
|
# dest section
|
|
|
|
setsection dest
|
|
getconf dest local
|
|
getconf fakesuper no
|
|
|
|
if [ "$dest" == "remote" ]; then
|
|
getconf testconnect no
|
|
getconf protocol ssh
|
|
getconf ssh ssh
|
|
getconf host
|
|
|
|
if [ "$protocol" == "ssh" ]; then
|
|
# sshd default listen port
|
|
getconf port 22
|
|
else
|
|
# rsyncd default listen port
|
|
getconf port 873
|
|
fi
|
|
|
|
getconf user
|
|
getconf bandwidthlimit
|
|
getconf remote_rsync rsync
|
|
getconf id_file /root/.ssh/id_rsa
|
|
fi
|
|
|
|
getconf batch no
|
|
|
|
if [ "$batch" != "yes" ]; then
|
|
getconf batch no
|
|
if [ "$batch" == "yes" ]; then
|
|
getconf batchbase
|
|
if [ ! -z "$batchbase" ]; then
|
|
batch="write"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
getconf numericids 0
|
|
getconf compress 0
|
|
|
|
# services section
|
|
|
|
setsection services
|
|
getconf initscripts /etc/init.d
|
|
getconf service
|
|
|
|
# config check
|
|
|
|
if [ "$dest" != "local" ] && [ "$from" == "remote" ]; then
|
|
fatal "When source is remote, destination should be local."
|
|
fi
|
|
|
|
if [ "$from" != "local" ] && [ "$from" != "remote" ]; then
|
|
fatal "Invalid source $from"
|
|
fi
|
|
|
|
backupdir="$mountpoint/$backupdir"
|
|
|
|
if [ "$dest" == "local" ] && [ ! -d "$backupdir" ]; then
|
|
fatal "Backupdir $backupdir does not exist"
|
|
fi
|
|
|
|
if [ ! -z "$log" ]; then
|
|
mkdir -p `dirname $log`
|
|
fi
|
|
|
|
if [ "$format" == "short" ]; then
|
|
if [ -z "$days" ]; then
|
|
keep="4"
|
|
else
|
|
keep=$[$days - 1]
|
|
fi
|
|
fi
|
|
|
|
if [ ! -z "$nicelevel" ]; then
|
|
nice="nice -n $nicelevel"
|
|
else
|
|
nice=""
|
|
fi
|
|
|
|
if [ -n "$ionicelevel" ]; then
|
|
nice="ionice -c2 -n $ionicelevel $nice"
|
|
fi
|
|
|
|
ssh_cmd_base="ssh -T -o PasswordAuthentication=no -p $port -i $id_file"
|
|
ssh_cmd="$ssh_cmd_base $user@$host"
|
|
|
|
if [ "$from" == "remote" ] || [ "$dest" == "remote" ]; then
|
|
if [ "$testconnect" == "yes" ] && [ "$protocol" == "ssh" ]; then
|
|
test_connect $host $port $user $id_file
|
|
fi
|
|
fi
|
|
|
|
if [ "$multiconnection" == "yes" ]; then
|
|
ssh_cmd="$ssh_cmd -S $tmp/%r@%h:%p"
|
|
fi
|
|
|
|
if [ $enable_mv_timestamp_bug == "yes" ]; then
|
|
mv=move_files
|
|
fi
|
|
|
|
set -o noglob
|
|
SAVEIFS=$IFS
|
|
IFS=$(echo -en "\n\b")
|
|
for i in $exclude; do
|
|
str="${i//__star__/*}"
|
|
excludes="${excludes} --exclude='$str'"
|
|
done
|
|
IFS=$SAVEIFS
|
|
set +o noglob
|
|
|
|
# Make sure we'll run bash at the destination
|
|
ssh_cmd="$ssh_cmd /bin/bash"
|
|
|
|
}
|
|
|
|
function rotate_short {
|
|
|
|
local dest
|
|
local folder="$1"
|
|
local keep="$2"
|
|
local metadata="`dirname $folder`/metadata"
|
|
|
|
# No rotations
|
|
if [[ "$keep" -lt 1 ]]; then
|
|
return
|
|
fi
|
|
|
|
if [ -d $folder.$keep ]; then
|
|
$nice $mv /$folder.$keep /$folder.tmp
|
|
fi
|
|
|
|
for ((n=$[$keep - 1]; n >= 0; n--)); do
|
|
if [ -d $folder.$n ]; then
|
|
dest=$[$n + 1]
|
|
$nice $mv /$folder.$n /$folder.$dest
|
|
$touch /$folder.$dest
|
|
mkdir -p $metadata/`basename $folder`.$dest
|
|
date +%c%n%s > $metadata/`basename $folder`.$dest/rotated
|
|
fi
|
|
done
|
|
|
|
if [ -d $folder.tmp ]; then
|
|
$nice $mv /$folder.tmp /$folder.0
|
|
fi
|
|
|
|
if [ -d $folder.1 ]; then
|
|
$nice $cp -alf /$folder.1/. /$folder.0
|
|
fi
|
|
|
|
# Cleanup orphaned metadata
|
|
for file in `ls $metadata`; do
|
|
if [ ! -d "`dirname $folder`/$file" ]; then
|
|
debug "removing orphaned metadata $file"
|
|
rm -rf $metadata/$file
|
|
fi
|
|
done
|
|
|
|
}
|
|
|
|
function rotate_short_remote {
|
|
|
|
local folder="$1"
|
|
local metadata="`dirname $folder`/metadata"
|
|
local keep="$2"
|
|
|
|
# No rotations
|
|
if [[ "$keep" -lt 1 ]]; then
|
|
return
|
|
fi
|
|
|
|
(
|
|
$ssh_cmd <<EOF
|
|
##### BEGIN REMOTE SCRIPT #####
|
|
|
|
if [ -d $folder.$keep ]; then
|
|
$nice mv /$folder.$keep /$folder.tmp
|
|
fi
|
|
|
|
for ((n=$(($keep - 1)); n >= 0; n--)); do
|
|
if [ -d $folder.\$n ]; then
|
|
dest=\$((\$n + 1))
|
|
$nice mv /$folder.\$n /$folder.\$dest
|
|
touch /$folder.\$dest
|
|
mkdir -p $metadata/`basename $folder`.\$dest
|
|
date +%c%n%s > $metadata/`basename $folder`.\$dest/rotated
|
|
fi
|
|
done
|
|
|
|
if [ -d $folder.tmp ]; then
|
|
$nice mv /$folder.tmp /$folder.0
|
|
fi
|
|
|
|
if [ -d $folder.1 ]; then
|
|
$nice $cp -alf /$folder.1/. /$folder.0
|
|
fi
|
|
|
|
# Cleanup orphaned metadata
|
|
for file in \`ls $metadata\`; do
|
|
if [ ! -d "`dirname $folder`/\$file" ]; then
|
|
echo "Debug: removing orphaned metadata \$file"
|
|
rm -rf $metadata/\$file
|
|
fi
|
|
done
|
|
|
|
# Create first directory if needed
|
|
if [ -d $backupdir ]; then
|
|
[ -d $folder.0 ] || mkdir -p $folder.0
|
|
fi
|
|
##### END REMOTE SCRIPT #######
|
|
EOF
|
|
) | (while read a; do passthru $a; done)
|
|
|
|
}
|
|
|
|
function rotate_long {
|
|
|
|
backuproot="$1"
|
|
seconds_daily=86400
|
|
seconds_weekly=604800
|
|
seconds_monthly=2628000
|
|
keepdaily=$keepdaily
|
|
keepweekly=$keepweekly
|
|
keepmonthly=$keepmonthly
|
|
now=`date +%s`
|
|
|
|
local metadata
|
|
|
|
if [ ! -d "$backuproot" ]; then
|
|
warning "Skipping rotate of $backuproot as it doesn't exist."
|
|
return
|
|
fi
|
|
|
|
for rottype in daily weekly monthly; do
|
|
seconds=$((seconds_${rottype}))
|
|
dir="$backuproot/$rottype"
|
|
metadata="$backuproot/metadata/$rottype"
|
|
|
|
mkdir -p $metadata.1
|
|
if [ ! -d $dir.1 ]; then
|
|
echo "Debug: $dir.1 does not exist, skipping."
|
|
continue 1
|
|
elif [ ! -f $metadata.1/created ] && [ ! -f $metadata.1/rotated ]; then
|
|
warning "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation."
|
|
continue 1
|
|
fi
|
|
|
|
# Rotate the current list of backups, if we can.
|
|
oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1`
|
|
[ "$oldest" == "" ] && oldest=0
|
|
for (( i=$oldest; i > 0; i-- )); do
|
|
if [ -d $dir.$i ]; then
|
|
if [ -f $metadata.$i/created ]; then
|
|
created=`tail -1 $metadata.$i/created`
|
|
elif [ -f $metadata.$i/rotated ]; then
|
|
created=`tail -1 $metadata.$i/rotated`
|
|
else
|
|
created=0
|
|
fi
|
|
# Validate created date
|
|
if [ -z "$created" ] || echo $created | grep -v -q -e '^[0-9]*$'; then
|
|
warning "Invalid metadata $created. Skipping rotation."
|
|
break
|
|
fi
|
|
cutoff_time=$(( now - (seconds*(i-1)) ))
|
|
if [ ! $created -gt $cutoff_time ]; then
|
|
next=$(( i + 1 ))
|
|
if [ ! -d $dir.$next ]; then
|
|
debug "$rottype.$i --> $rottype.$next"
|
|
$nice mv $dir.$i $dir.$next
|
|
mkdir -p $metadata.$next
|
|
date +%c%n%s > $metadata.$next/rotated
|
|
if [ -f $metadata.$i/created ]; then
|
|
$nice mv $metadata.$i/created $metadata.$next
|
|
fi
|
|
else
|
|
debug "skipping rotation of $dir.$i because $dir.$next already exists."
|
|
fi
|
|
else
|
|
debug "skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)."
|
|
fi
|
|
fi
|
|
done
|
|
done
|
|
|
|
max=$((keepdaily+1))
|
|
if [ $keepweekly -gt 0 -a -d $backuproot/daily.$max -a ! -d $backuproot/weekly.1 ]; then
|
|
debug "daily.$max --> weekly.1"
|
|
$nice mv $backuproot/daily.$max $backuproot/weekly.1
|
|
mkdir -p $backuproot/metadata/weekly.1
|
|
# Update increment folder date and setup metadata
|
|
$touch $backuproot/weekly.1
|
|
date +%c%n%s > $backuproot/metadata/weekly.1/rotated
|
|
#if [ -f $backuproot/metadata/daily.$max/created ]; then
|
|
# $nice mv $backuproot/metadata/daily.$max/created $backuproot/metadata/weekly.1/
|
|
#fi
|
|
fi
|
|
|
|
max=$((keepweekly+1))
|
|
if [ $keepmonthly -gt 0 -a -d $backuproot/weekly.$max -a ! -d $backuproot/monthly.1 ]; then
|
|
debug "weekly.$max --> monthly.1"
|
|
$nice mv $backuproot/weekly.$max $backuproot/monthly.1
|
|
mkdir -p $backuproot/metadata/monthly.1
|
|
# Update increment folder date and setup metadata
|
|
$touch $backuproot/monthly.1
|
|
date +%c%n%s > $backuproot/metadata/monthly.1/rotated
|
|
#if [ -f $backuproot/metadata/weekly.$max/created ]; then
|
|
# $nice mv $backuproot/metadata/weekly.$max/created $backuproot/metadata/weekly.1/
|
|
#fi
|
|
fi
|
|
|
|
for rottype in daily weekly monthly; do
|
|
max=$((keep${rottype}+1))
|
|
dir="$backuproot/$rottype"
|
|
oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1`
|
|
[ "$oldest" == "" ] && oldest=0
|
|
# if we've rotated the last backup off the stack, remove it.
|
|
for (( i=$oldest; i >= $max; i-- )); do
|
|
if [ -d $dir.$i ]; then
|
|
if [ -d $backuproot/rotate.tmp ]; then
|
|
debug "removing rotate.tmp"
|
|
$nice rm -rf $backuproot/rotate.tmp
|
|
fi
|
|
debug "moving $rottype.$i to rotate.tmp"
|
|
$nice mv $dir.$i $backuproot/rotate.tmp
|
|
fi
|
|
done
|
|
done
|
|
|
|
# Cleanup orphaned metadata
|
|
for file in `ls $backuproot/metadata`; do
|
|
if [ ! -d "$backuproot/$file" ]; then
|
|
debug "removing orphaned metadata $file"
|
|
rm -rf $backuproot/metadata/$file
|
|
fi
|
|
done
|
|
|
|
}
|
|
|
|
function rotate_long_remote {
|
|
|
|
local backuproot="$1"
|
|
|
|
(
|
|
$ssh_cmd <<EOF
|
|
##### BEGIN REMOTE SCRIPT #####
|
|
|
|
seconds_daily=86400
|
|
seconds_weekly=604800
|
|
seconds_monthly=2628000
|
|
keepdaily=$keepdaily
|
|
keepweekly=$keepweekly
|
|
keepmonthly=$keepmonthly
|
|
now=\`date +%s\`
|
|
|
|
if [ ! -d "$backuproot" ]; then
|
|
echo "Fatal: skipping rotate of $backuproot as it doesn't exist."
|
|
exit
|
|
fi
|
|
|
|
for rottype in daily weekly monthly; do
|
|
seconds=\$((seconds_\${rottype}))
|
|
dir="$backuproot/\$rottype"
|
|
metadata="$backuproot/metadata/\$rottype"
|
|
|
|
mkdir -p \$metadata.1
|
|
if [ ! -d \$dir.1 ]; then
|
|
echo "Debug: \$dir.1 does not exist, skipping."
|
|
continue 1
|
|
elif [ ! -f \$metadata.1/created ] && [ ! -f \$metadata.1/rotated ]; then
|
|
echo "Warning: metadata does not exist for \$dir.1. This backup may be only partially completed. Skipping rotation."
|
|
continue 1
|
|
fi
|
|
|
|
# Rotate the current list of backups, if we can.
|
|
oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1\`
|
|
[ "\$oldest" == "" ] && oldest=0
|
|
for (( i=\$oldest; i > 0; i-- )); do
|
|
if [ -d \$dir.\$i ]; then
|
|
if [ -f \$metadata.\$i/created ]; then
|
|
created=\`tail -1 \$metadata.\$i/created\`
|
|
elif [ -f \$metadata.\$i/rotated ]; then
|
|
created=\`tail -1 \$metadata.\$i/rotated\`
|
|
else
|
|
created=0
|
|
fi
|
|
# Validate created date
|
|
if [ -z "\$created" ] || echo \$created | grep -v -q -e '^[0-9]*$'; then
|
|
echo "Warning: Invalid metadata \$created. Skipping rotation."
|
|
break
|
|
fi
|
|
cutoff_time=\$(( now - (seconds*(i-1)) ))
|
|
if [ ! \$created -gt \$cutoff_time ]; then
|
|
next=\$(( i + 1 ))
|
|
if [ ! -d \$dir.\$next ]; then
|
|
echo "Debug: \$rottype.\$i --> \$rottype.\$next"
|
|
$nice mv \$dir.\$i \$dir.\$next
|
|
mkdir -p \$metadata.\$next
|
|
date +%c%n%s > \$metadata.\$next/rotated
|
|
if [ -f \$metadata.\$i/created ]; then
|
|
$nice mv \$metadata.\$i/created \$metadata.\$next
|
|
fi
|
|
else
|
|
echo "Debug: skipping rotation of \$dir.\$i because \$dir.\$next already exists."
|
|
fi
|
|
else
|
|
echo "Debug: skipping rotation of \$dir.\$i because it was created" \$(( (now-created)/86400)) "days ago ("\$(( (now-cutoff_time)/86400))" needed)."
|
|
fi
|
|
fi
|
|
done
|
|
done
|
|
|
|
max=\$((keepdaily+1))
|
|
if [ \$keepweekly -gt 0 -a -d $backuproot/daily.\$max -a ! -d $backuproot/weekly.1 ]; then
|
|
echo "Debug: daily.\$max --> weekly.1"
|
|
$nice mv $backuproot/daily.\$max $backuproot/weekly.1
|
|
mkdir -p $backuproot/metadata/weekly.1
|
|
# Update increment folder date and setup metadata
|
|
$touch $backuproot/weekly.1
|
|
date +%c%n%s > $backuproot/metadata/weekly.1/rotated
|
|
#if [ -f $backuproot/metadata/daily.\$max/created ]; then
|
|
# $nice mv $backuproot/metadata/daily.\$max/created $backuproot/metadata/weekly.1/
|
|
#fi
|
|
fi
|
|
|
|
max=\$((keepweekly+1))
|
|
if [ \$keepmonthly -gt 0 -a -d $backuproot/weekly.\$max -a ! -d $backuproot/monthly.1 ]; then
|
|
echo "Debug: weekly.\$max --> monthly.1"
|
|
$nice mv $backuproot/weekly.\$max $backuproot/monthly.1
|
|
mkdir -p $backuproot/metadata/monthly.1
|
|
# Update increment folder date and setup metadata
|
|
$touch $backuproot/monthly.1
|
|
date +%c%n%s > $backuproot/metadata/monthly.1/rotated
|
|
#if [ -f $backuproot/metadata/weekly.\$max/created ]; then
|
|
# $nice mv $backuproot/metadata/weekly.\$max/created $backuproot/metadata/weekly.1/
|
|
#fi
|
|
fi
|
|
|
|
for rottype in daily weekly monthly; do
|
|
max=\$((keep\${rottype}+1))
|
|
dir="$backuproot/\$rottype"
|
|
oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1\`
|
|
[ "\$oldest" == "" ] && oldest=0
|
|
# if we've rotated the last backup off the stack, remove it.
|
|
for (( i=\$oldest; i >= \$max; i-- )); do
|
|
if [ -d \$dir.\$i ]; then
|
|
if [ -d $backuproot/rotate.tmp ]; then
|
|
echo "Debug: removing rotate.tmp"
|
|
$nice rm -rf $backuproot/rotate.tmp
|
|
fi
|
|
echo "Debug: moving \$rottype.\$i to rotate.tmp"
|
|
$nice mv \$dir.\$i $backuproot/rotate.tmp
|
|
fi
|
|
done
|
|
done
|
|
|
|
# Cleanup orphaned metadata
|
|
for file in \`ls $backuproot/metadata\`; do
|
|
if [ ! -d "$backuproot/\$file" ]; then
|
|
echo "Debug: removing orphaned metadata \$file"
|
|
rm -rf $backuproot/metadata/\$file
|
|
fi
|
|
done
|
|
##### END REMOTE SCRIPT #######
|
|
EOF
|
|
) | (while read a; do passthru $a; done)
|
|
|
|
}
|
|
|
|
function setup_long_dirs {
|
|
|
|
local destdir=$1
|
|
local backuptype=$2
|
|
local dir="$destdir/$backuptype"
|
|
local tmpdir="$destdir/rotate.tmp"
|
|
local metadata="$destdir/metadata/$backuptype.1"
|
|
|
|
if [ ! -d $destdir ]; then
|
|
echo "Creating destination directory $destdir..."
|
|
mkdir -p $destdir
|
|
fi
|
|
|
|
if [ -d $dir.1 ]; then
|
|
if [ -f $metadata/created ]; then
|
|
echo "Warning: $dir.1 already exists. Overwriting contents."
|
|
else
|
|
echo "Warning: we seem to be resuming a partially written $dir.1"
|
|
fi
|
|
else
|
|
if [ -d $tmpdir ]; then
|
|
mv $tmpdir $dir.1
|
|
if [ $? == 1 ]; then
|
|
fatal "Could not move $tmpdir to $dir.1 on host $host"
|
|
fi
|
|
else
|
|
mkdir --parents $dir.1
|
|
if [ $? == 1 ]; then
|
|
fatal "Could not create directory $dir.1 on host $host"
|
|
fi
|
|
fi
|
|
if [ -d $dir.2 ]; then
|
|
echo "Debug: update links $backuptype.2 --> $backuptype.1"
|
|
cp -alf $dir.2/. $dir.1
|
|
#if [ $? == 1 ]; then
|
|
# fatal "Could not create hard links to $dir.1 on host $host"
|
|
#fi
|
|
fi
|
|
fi
|
|
[ -f $metadata/created ] && rm $metadata/created
|
|
[ -f $metadata/rotated ] && rm $metadata/rotated
|
|
|
|
}
|
|
|
|
function setup_long_dirs_remote {
|
|
|
|
local destdir=$1
|
|
local backuptype=$2
|
|
local dir="$destdir/$backuptype"
|
|
local tmpdir="$destdir/rotate.tmp"
|
|
local metadata="$destdir/metadata/$backuptype.1"
|
|
|
|
(
|
|
$ssh_cmd <<EOF
|
|
##### BEGIN REMOTE SCRIPT #####
|
|
if [ ! -d $destdir ]; then
|
|
echo "Creating destination directory $destdir on $host..."
|
|
mkdir -p $destdir
|
|
fi
|
|
|
|
if [ -d $dir.1 ]; then
|
|
if [ -f $metadata/created ]; then
|
|
echo "Warning: $dir.1 already exists. Overwriting contents."
|
|
else
|
|
echo "Warning: we seem to be resuming a partially written $dir.1"
|
|
fi
|
|
else
|
|
if [ -d $tmpdir ]; then
|
|
mv $tmpdir $dir.1
|
|
if [ \$? == 1 ]; then
|
|
echo "Fatal: could mv $destdir/rotate.tmp $dir.1 on host $host"
|
|
exit 1
|
|
fi
|
|
else
|
|
mkdir --parents $dir.1
|
|
if [ \$? == 1 ]; then
|
|
echo "Fatal: could not create directory $dir.1 on host $host"
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [ -d $dir.2 ]; then
|
|
echo "Debug: update links $backuptype.2 --> $backuptype.1"
|
|
cp -alf $dir.2/. $dir.1
|
|
#if [ \$? == 1 ]; then
|
|
# echo "Fatal: could not create hard links to $dir.1 on host $host"
|
|
# exit 1
|
|
#fi
|
|
fi
|
|
fi
|
|
[ -f $metadata/created ] && rm $metadata/created
|
|
[ -f $metadata/rotated ] && rm $metadata/rotated
|
|
##### END REMOTE SCRIPT #######
|
|
EOF
|
|
) | (while read a; do passthru $a; done)
|
|
|
|
}
|
|
|
|
function move_files {
|
|
|
|
ref=$tmp/makesnapshot-mymv-$$;
|
|
$touch -r $1 $ref;
|
|
$mv $1 $2;
|
|
$touch -r $ref $2;
|
|
$rm $ref;
|
|
|
|
}
|
|
|
|
function prepare_storage {
|
|
|
|
section="`basename $SECTION`"
|
|
|
|
if [ "$format" == "short" ]; then
|
|
|
|
suffix="$section.0"
|
|
|
|
if [ ! "$test" ]; then
|
|
info "Rotating $backupdir/$SECTION..."
|
|
echo "Rotating $backupdir/$SECTION..." >> $log
|
|
|
|
if [ "$dest" == "remote" ]; then
|
|
rotate_short_remote $backupdir/$SECTION/$section $keep
|
|
else
|
|
rotate_short $backupdir/$SECTION/$section $keep
|
|
if [ ! -d "$backupdir/$SECTION/$section.0" ]; then
|
|
mkdir -p $backupdir/$SECTION/$section.0
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
elif [ "$format" == "long" ]; then
|
|
|
|
if [ $keepdaily -gt 0 ]; then
|
|
btype=daily
|
|
elif [ $keepweekly -gt 0 ]; then
|
|
btype=weekly
|
|
elif [ $keepmonthly -gt 0 ]; then
|
|
btype=monthly
|
|
else
|
|
fatal "keeping no backups";
|
|
fi
|
|
|
|
suffix="$btype.1"
|
|
|
|
if [ ! "$test" ]; then
|
|
info "Rotating $backupdir/$SECTION/..."
|
|
echo "Rotating $backupdir/$SECTION/..." >> $log
|
|
|
|
if [ "$dest" == "remote" ]; then
|
|
rotate_long_remote $backupdir/$SECTION
|
|
setup_long_dirs_remote $backupdir/$SECTION $btype
|
|
else
|
|
rotate_long $backupdir/$SECTION
|
|
setup_long_dirs $backupdir/$SECTION $btype
|
|
fi
|
|
fi
|
|
|
|
elif [ "$format" == "mirror" ]; then
|
|
|
|
# create $SECTION directories, but not $backupdir
|
|
if [ "$dest" == "local" ]; then
|
|
if [ -d "$backupdir" ]; then
|
|
mkdir --parents $backupdir/$SECTION
|
|
else
|
|
fatal "The destination $backupdir does not exist"
|
|
exit 1
|
|
fi
|
|
else
|
|
debug "$ssh_cmd"
|
|
debug "$backupdir/$SECTION"
|
|
(
|
|
$ssh_cmd <<EOF
|
|
##### BEGIN REMOTE SCRIPT #####
|
|
if [ -d "$backupdir" ]; then
|
|
mkdir --parents $backupdir/$SECTION
|
|
else
|
|
echo "FATAL: The destination $backupdir does not exist"
|
|
exit 1
|
|
fi
|
|
##### END REMOTE SCRIPT #######
|
|
EOF
|
|
) | (while read a; do passthru $a; done)
|
|
fi
|
|
|
|
suffix=""
|
|
|
|
else
|
|
fatal "Invalid backup format $format"
|
|
fi
|
|
|
|
}
|
|
|
|
function set_orig {
|
|
|
|
if [ "$from" == "local" ]; then
|
|
orig="/$SECTION/"
|
|
elif [ "$from" == "remote" ]; then
|
|
if [ "$protocol" == "rsync" ]; then
|
|
orig="rsync://$user@$host:$port/$SECTION/"
|
|
else
|
|
orig="$user@$host:/$SECTION/"
|
|
fi
|
|
fi
|
|
|
|
}
|
|
|
|
function set_dest {
|
|
|
|
if [ "$dest" == "local" ]; then
|
|
dest_path="$backupdir/$SECTION/$suffix/"
|
|
else
|
|
if [ "$protocol" == "rsync" ]; then
|
|
dest_path="rsync://$user@$host:$port/$backupdir/$SECTION/$suffix/"
|
|
else
|
|
dest_path="$user@$host:$backupdir/$SECTION/$suffix/"
|
|
fi
|
|
fi
|
|
|
|
}
|
|
|
|
function set_batch_mode {
|
|
|
|
local batch_file="$batchbase/$SECTION/$suffix"
|
|
|
|
if [ "$batch" == "read" ]; then
|
|
if [ -e "$batch_file" ]; then
|
|
orig=""
|
|
excludes=""
|
|
batch_option="--read-batch=$batch_file"
|
|
else
|
|
fatal "Batch file not found: $batch_file"
|
|
fi
|
|
elif [ "$batch" == "write" ]; then
|
|
mkdir -p `dirname $batch_file`
|
|
batch_option="--write-batch=$batch_file"
|
|
fi
|
|
|
|
}
|
|
|
|
function update_metadata {
|
|
|
|
local metadata
|
|
local folder
|
|
|
|
if [ "$dest" == "local" ]; then
|
|
metadata="`dirname $dest_path`/metadata/`basename $dest_path`"
|
|
mkdir -p $metadata
|
|
# Use the backup start time and not the time the backup was
|
|
# finished, otherwise daily rotations might not take place.
|
|
# If we used backup end time, in the next handler run
|
|
# we might not have $now - $created >= 24:00
|
|
echo "$starttime" > $metadata/created
|
|
$touch $backupdir/$SECTION/$suffix
|
|
else
|
|
folder="`echo $dest_path | cut -d : -f 2`"
|
|
metadata="`dirname $folder`/metadata/`basename $folder`"
|
|
|
|
(
|
|
$ssh_cmd <<EOF
|
|
##### BEGIN REMOTE SCRIPT #####
|
|
mkdir -p $metadata
|
|
# Use the backup start time and not the time the backup was
|
|
# finished, otherwise daily rotations might not take place.
|
|
# If we used backup end time, in the next handler run
|
|
# we might not have $now - $created >= 24:00
|
|
echo "$starttime" > $metadata/created
|
|
##### END REMOTE SCRIPT #######
|
|
EOF
|
|
) | (while read a; do passthru $a; done)
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
function test_connect {
|
|
|
|
local host="$1"
|
|
local port="$2"
|
|
local user="$3"
|
|
local id_file="$4"
|
|
|
|
if [ -z "$host" ] || [ -z "$user" ]; then
|
|
fatal "Remote host or user not set"
|
|
fi
|
|
|
|
debug "$ssh_cmd 'echo -n 1'"
|
|
result=`$ssh_cmd 'echo -n 1'`
|
|
|
|
if [ "$result" != "1" ]; then
|
|
fatal "Can't connect to $host as $user."
|
|
else
|
|
debug "Connected to $host successfully."
|
|
fi
|
|
|
|
}
|
|
|
|
function set_filelist {
|
|
|
|
filelist_flag=""
|
|
|
|
if [ "$filelist" == "yes" ]; then
|
|
if [ ! -z "$filelistbase" ]; then
|
|
if [ -e "$filelistbase/$SECTION/$suffix" ]; then
|
|
filelist_flag="--files-from=$filelistbase/$SECTION/$suffix"
|
|
else
|
|
warning "File list $filelistbase/$SECTION/$suffix not found."
|
|
fi
|
|
else
|
|
warning "No filelistbase set."
|
|
fi
|
|
fi
|
|
|
|
}
|
|
|
|
function set_rsync_options {
|
|
|
|
if [ "$test" ]; then
|
|
rsync_options="$rsync_options --dry-run"
|
|
fi
|
|
|
|
if [ "$numericids" != "0" ]; then
|
|
rsync_options="$rsync_options --numeric-ids"
|
|
fi
|
|
|
|
if [ "$from" == "local" ] || [ "$dest" == "local" ]; then
|
|
# rsync options for local sources or destinations
|
|
true
|
|
fi
|
|
|
|
if [ "$from" == "remote" ] || [ "$dest" == "remote" ]; then
|
|
|
|
# rsync options for remote sources or destinations
|
|
|
|
if [ "$compress" == "1" ]; then
|
|
rsync_options="$rsync_options --compress"
|
|
fi
|
|
|
|
if [ ! -z "$bandwidthlimit" ]; then
|
|
rsync_options="$rsync_options --bwlimit=$bandwidthlimit"
|
|
fi
|
|
|
|
if [ "$fakesuper" == "yes" ]; then
|
|
remote_rsync="$remote_rsync --fake-super"
|
|
fi
|
|
|
|
if [ "$protocol" == "ssh" ]; then
|
|
if [ ! -e "$id_file" ]; then
|
|
fatal "SSH identity file $id_file not found."
|
|
else
|
|
debug RSYNC_RSH=\"$ssh_cmd_base\"
|
|
echo RSYNC_RSH=\"$ssh_cmd_base\" >> $log
|
|
export RSYNC_RSH="$ssh_cmd_base"
|
|
fi
|
|
fi
|
|
|
|
fi
|
|
|
|
# Mangle rsync_options so we can use quotes after all other
|
|
# options were evaluated.
|
|
if [ "$from" == "local" ] && [ "$dest" == "local" ]; then
|
|
rsync_options=($rsync_options)
|
|
else
|
|
rsync_options=($rsync_options --rsync-path="$remote_rsync")
|
|
fi
|
|
|
|
}
|
|
|
|
function stop_services {
|
|
|
|
if [ ! -z "$service" ]; then
|
|
for daemon in $service; do
|
|
info "Stopping service $daemon..."
|
|
$initscripts/$daemon stop
|
|
done
|
|
fi
|
|
|
|
}
|
|
|
|
function start_services {
|
|
|
|
# restart services
|
|
|
|
if [ ! -z "$service" ]; then
|
|
for daemon in $service; do
|
|
info "Starting service $daemon..."
|
|
$initscripts/$daemon start
|
|
done
|
|
fi
|
|
|
|
}
|
|
|
|
function mount_rw {
|
|
|
|
# mount backup destination folder as read-write
|
|
|
|
if [ "$dest" == "local" ]; then
|
|
if [ "$read_only" == "1" ] || [ "$read_only" == "yes" ]; then
|
|
if [ -d "$mountpoint" ]; then
|
|
mount -o remount,rw $mountpoint
|
|
if (($?)); then
|
|
fatal "Could not mount $mountpoint"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
}
|
|
|
|
function mount_ro {
|
|
|
|
# remount backup destination as read-only
|
|
|
|
if [ "$dest" == "local" ]; then
|
|
if [ "$read_only" == "1" ] || [ "$read_only" == "yes" ]; then
|
|
mount -o remount,ro $mountpoint
|
|
fi
|
|
fi
|
|
|
|
}
|
|
|
|
function run_fsck {
|
|
|
|
# check partition for errors
|
|
|
|
if [ "$dest" == "local" ]; then
|
|
if [ "$fscheck" == "1" ] || [ "$fscheck" == "yes" ]; then
|
|
umount $mountpoint
|
|
if (($?)); then
|
|
warning "Could not umount $mountpoint to run fsck"
|
|
else
|
|
$nice $fsck -v -y $partition >> $log
|
|
mount $mountpoint
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
}
|
|
|
|
function start_mux {
|
|
|
|
if [ "$multiconnection" == "yes" ]; then
|
|
debug "Starting master ssh connection"
|
|
$ssh_cmd -M sleep 1d &
|
|
sleep 1
|
|
fi
|
|
|
|
}
|
|
|
|
function end_mux {
|
|
|
|
if [ "$multiconnection" == "yes" ]; then
|
|
debug "Stopping master ssh connection"
|
|
$ssh_cmd pkill sleep
|
|
fi
|
|
|
|
}
|
|
|
|
function set_pipefail {
|
|
|
|
# Save initial pipefail status for later restoration
|
|
if echo "$SHELLOPTS" | grep -q ":pipefail"; then
|
|
pipefail="-o"
|
|
else
|
|
pipefail="+o"
|
|
fi
|
|
|
|
# Ensure that a non-zero rsync exit status is caught by our handler
|
|
set -o pipefail
|
|
|
|
}
|
|
|
|
function restore_pipefail {
|
|
|
|
if [ ! -z "$pipefail" ]; then
|
|
set $pipefail pipefail
|
|
fi
|
|
|
|
}
|
|
|
|
function check_rsync_exit_status {
|
|
|
|
if [ -z "$1" ]; then
|
|
return
|
|
fi
|
|
|
|
case $1 in
|
|
0)
|
|
debug "Rsync transfer of $SECTION finished successfully."
|
|
return
|
|
;;
|
|
1|2|3|4|5|6|10|11|12|13|14|21)
|
|
fatal "Rsync error $1 when trying to transfer $SECTION"
|
|
;;
|
|
*)
|
|
warning "Rsync error $1 when trying to transfer $SECTION"
|
|
;;
|
|
esac
|
|
|
|
}
|
|
|
|
# the backup procedure
|
|
|
|
eval_config
|
|
set_rsync_options
|
|
start_mux
|
|
stop_services
|
|
mount_rw
|
|
|
|
starttime="`date +%c%n%s`"
|
|
echo "Starting backup at `echo "$starttime" | head -n 1`" >> $log
|
|
|
|
for SECTION in $include; do
|
|
|
|
prepare_storage
|
|
set_orig
|
|
set_batch_mode
|
|
set_filelist
|
|
set_dest
|
|
|
|
info "Syncing $SECTION on $dest_path..."
|
|
command="$rsync ${rsync_options[@]} --delete-excluded $filelist_flag $excludes $batch_option $orig $dest_path"
|
|
debug $nice su -c "$command"
|
|
set_pipefail
|
|
$nice su -c "$command" | tee -a $log
|
|
|
|
check_rsync_exit_status $?
|
|
restore_pipefail
|
|
update_metadata
|
|
|
|
done
|
|
|
|
mount_ro
|
|
run_fsck
|
|
start_services
|
|
end_mux
|
|
|
|
echo "Finished backup at `date`" >> $log
|
|
|