mirror of
https://0xacab.org/liberate/backupninja.git
synced 2024-11-10 04:42:31 +01:00
Merge branch 'master' of labs.riseup.net:backupninja
This commit is contained in:
Micah Anderson
commit
f8e69f738e
1
AUTHORS
1
AUTHORS
@ -43,3 +43,4 @@ aihtdikh -- Allow 'when = XXX' with spaces in .sh files.
|
||||
Chris Lamb <lamby@debian.org> -- rdiff.helper bugfix
|
||||
Yuval Kogman <nothingmuch@woobling.org> -- RackSpace's CloudFiles support for duplicity
|
||||
exobuzz - mysql bugfixes
|
||||
Glennie Vignarajah <glennie@glennie.fr> -- mysql bugfix
|
||||
|
||||
30
ChangeLog
30
ChangeLog
@ -1,4 +1,12 @@
|
||||
version 1.0 -- UNRELEASED
|
||||
handler changes
|
||||
mysql:
|
||||
. Use --skip-events when backing up the performance_schema database.
|
||||
(Closes: #673572)
|
||||
backupninja changes
|
||||
. Make it clear what lockfile could not be acquired, if any.
|
||||
|
||||
version 1.0-rc1 -- May 15, 2012
|
||||
handler changes
|
||||
dup:
|
||||
. Make the .dup generated by ninjahelper more consistent with
|
||||
@ -10,6 +18,8 @@ version 1.0 -- UNRELEASED
|
||||
(Closes: #657201)
|
||||
. Report failure output at error loglevel so that it is emailed
|
||||
(Closes: #536858)
|
||||
maildir:
|
||||
. Remove 'loadlimit' parameter - it is not used anywhere.
|
||||
mysql:
|
||||
. Don't attempt to dump performance_schema database (Redmine#3741).
|
||||
pgsql:
|
||||
@ -22,15 +32,29 @@ version 1.0 -- UNRELEASED
|
||||
. Fix metadata rotation.
|
||||
. Allow disabling rotation or setting 2 days as minimum for backup
|
||||
increments in rsync short format (Redmine#2107).
|
||||
. Use fatal instead of custom "exit 1" (Redmine#3721).
|
||||
. Abort on rsync error (Redmine#3692).
|
||||
. Cleanup orphaned metadata (Redmine#3727).
|
||||
. Use the backup start time and not the time the backup was finished
|
||||
(Closes: #654192)
|
||||
. Use the backup start time and not the time the backup was finished.
|
||||
(Closes: #654192).
|
||||
. Use 'debug', 'fatal' and 'warning' functions instead of regular echo
|
||||
and exit (Redmine#3840, Redmine#3721).
|
||||
. Quoting $starttime (Redmine#3868).
|
||||
. Validate created date on long_rotation to avoid too many arguments
|
||||
at comparison (Redmine#3868).
|
||||
. Quoting $exclude and $excludes and avoiding a for loop on $exclude
|
||||
to not expand wildcards in beforehand (Redmine#3882).
|
||||
. Quote excludes (Redmine#3882).
|
||||
. Changing remaining 'exit' to 'fatal' at rsync handler (Redmine#3721).
|
||||
. Removing duplicated locking support (Redmine#3838).
|
||||
. Documenting rotation parameters at example.rsync (Redmine#3891).
|
||||
. Ensure that a non-zero rsync exit status is caught (Redmine#3892).
|
||||
build system changes
|
||||
. Workaround automake sanity check that would prevent us from
|
||||
installing lib/* into lib/backupninja/. Where else are be supposed
|
||||
to install such files anyway?
|
||||
. Have "make dist" ship handlers/*.in instead of make results.
|
||||
. Have "make dist" ship the FAQ.
|
||||
. Install handlers as pkgdata_DATA, instead of their .in files.
|
||||
documentation changes
|
||||
. Document what features available to .sh jobs (Redmine #1558).
|
||||
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
# vi: noexpandtab softtabstop=0
|
||||
## Process this file with automake to produce Makefile.in
|
||||
|
||||
EXTRA_DIST = README COPYING AUTHORS INSTALL NEWS ChangeLog \
|
||||
EXTRA_DIST = FAQ README COPYING AUTHORS INSTALL NEWS ChangeLog \
|
||||
backupninja.spec backupninja.spec.in autogen.sh
|
||||
|
||||
SUBDIRS = etc examples handlers lib man src
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
|
||||
# The maintainer mode is causing me grief with newest versions of autotools
|
||||
#AM_MAINTAINER_MODE
|
||||
AC_INIT([backupninja],[0.9.10],[backupninja@lists.riseup.net])
|
||||
AC_INIT([backupninja],[1.0-rc1],[backupninja@lists.riseup.net])
|
||||
AC_CONFIG_SRCDIR([src/backupninja.in])
|
||||
AM_INIT_AUTOMAKE
|
||||
|
||||
|
||||
@ -36,9 +36,33 @@ backupdir = myserver
|
||||
#tmp = /tmp
|
||||
|
||||
# specify backup storage format: short, long or mirror (i.e, no rotations)
|
||||
#
|
||||
# In the short format, incremental backups are rotated every day the handler
|
||||
# runs an by a finite number of times (backup.0, backup.1, backup.1, etc), so
|
||||
# if you want to have incremental backups for longer periods (like months) you
|
||||
# have to configure rotations for 30 or more using the "days" parameter at the
|
||||
# [general] section in the handler config.
|
||||
#
|
||||
# The short format is better described here:
|
||||
# http://www.mikerubel.org/computers/rsync_snapshots/#Incremental
|
||||
#
|
||||
# The long format is inspired by the maildir handler and allows keeping backups
|
||||
# of longer periods (weeks and months) using less rotations as it stores
|
||||
# the increments in folders like daily.1, weekly.1, monthly.1 and has three
|
||||
# rotation parameters:
|
||||
#
|
||||
# keepdaily = number of daily backup increments
|
||||
# keepweekly = number of weekly backup increments
|
||||
# keepmonthly = number of monthly backup increments
|
||||
#
|
||||
format = short
|
||||
|
||||
# for short storage format, specify the number of backup increments (min = 2, set to 1 or less to disable)
|
||||
#
|
||||
# Note that setting days = 0 is almost the same as using format = mirror except
|
||||
# that with the days config your backup gets a .0 suffix at the destination
|
||||
# folder, making it easier to turn it later to an incremental backup.
|
||||
#
|
||||
days = 7
|
||||
|
||||
# for long storage format, specify the number of daily backup increments
|
||||
@ -50,11 +74,6 @@ days = 7
|
||||
# for long storage format, specify the number of monthly backup increments
|
||||
#keepmonthly = 1
|
||||
|
||||
# use this if you need a lockfile to be kept during backup execution
|
||||
# this is an useful feature in case you have some tasks that should
|
||||
# know if the backup is running or not
|
||||
#lockfile =
|
||||
|
||||
# rsync command nice level
|
||||
#nicelevel = 0
|
||||
|
||||
|
||||
@ -3,16 +3,20 @@ HANDLERS = dup dup.helper maildir makecd \
|
||||
makecd.helper mysql mysql.helper pgsql pgsql.helper rdiff \
|
||||
rdiff.helper rsync sh svn sys sys.helper trac tar tar.helper
|
||||
|
||||
DIST_HANDLERS = dup.in dup.helper.in maildir.in makecd.in \
|
||||
makecd.helper.in mysql.in mysql.helper.in pgsql.in pgsql.helper.in rdiff.in \
|
||||
rdiff.helper.in rsync.in sh.in svn.in sys.in sys.helper.in trac.in tar.in tar.helper.in wget
|
||||
|
||||
CLEANFILES = $(HANDLERS)
|
||||
|
||||
EXTRA_DIST = Makefile.am $(HANDLERS)
|
||||
EXTRA_DIST = Makefile.am $(DIST_HANDLERS)
|
||||
|
||||
edit = sed \
|
||||
-e "s,@BASH\@,$(BASH),g" \
|
||||
-e "s,@AWK\@,$(AWK),g" \
|
||||
-e "s,@SED\@,$(SED),g"
|
||||
|
||||
dist_pkgdata_DATA = $(HANDLERS)
|
||||
pkgdata_DATA = $(HANDLERS)
|
||||
|
||||
dup: $(srcdir)/dup.in
|
||||
rm -f dup
|
||||
|
||||
@ -256,10 +256,15 @@ then
|
||||
for db in $databases
|
||||
do
|
||||
DUMP_BASE="$MYSQLDUMP $defaultsfile $sqldumpoptions"
|
||||
if [ "$db" = "information_schema" ] || [ "$db" = "performance_schema" ]
|
||||
then
|
||||
DUMP_BASE="${DUMP_BASE} --skip-lock-tables"
|
||||
fi
|
||||
|
||||
case "$db" in
|
||||
information_schema)
|
||||
DUMP_BASE="${DUMP_BASE} --skip-lock-tables"
|
||||
;;
|
||||
performance_schema)
|
||||
DUMP_BASE="${DUMP_BASE} --skip-lock-tables --skip-events"
|
||||
;;
|
||||
esac
|
||||
|
||||
# Dumping structure and data
|
||||
DUMP="$DUMP_BASE $ignore $db"
|
||||
|
||||
@ -39,7 +39,6 @@
|
||||
# keepdaily = for long storage format, specify the number of daily backup increments
|
||||
# keepweekly = for long storage format, specify the number of weekly backup increments
|
||||
# keepmonthly = for long storage format, specify the number of monthly backup increments
|
||||
# lockfile = lockfile to be kept during backup execution
|
||||
# nicelevel = rsync command nice level
|
||||
# enable_mv_timestamp_bug = set to "yes" if your system isnt handling timestamps correctly
|
||||
# tmp = temp folder
|
||||
@ -129,7 +128,6 @@ function eval_config {
|
||||
getconf keepdaily 5
|
||||
getconf keepweekly 3
|
||||
getconf keepmonthly 1
|
||||
getconf lockfile
|
||||
getconf nicelevel 0
|
||||
getconf enable_mv_timestamp_bug no
|
||||
getconf tmp /tmp
|
||||
@ -277,9 +275,7 @@ function eval_config {
|
||||
mv=move_files
|
||||
fi
|
||||
|
||||
for path in $exclude; do
|
||||
excludes="$excludes --exclude=$path"
|
||||
done
|
||||
excludes=`echo "$exclude" | @SED@ -e "s/^/--exclude='/g" -e "s/ /' --exclude='/g" -e "s/$/'/"`
|
||||
|
||||
}
|
||||
|
||||
@ -391,8 +387,8 @@ function rotate_long {
|
||||
local metadata
|
||||
|
||||
if [ ! -d "$backuproot" ]; then
|
||||
echo "Debug: skipping rotate of $backuproot as it doesn't exist."
|
||||
exit
|
||||
warning "Skipping rotate of $backuproot as it doesn't exist."
|
||||
return
|
||||
fi
|
||||
|
||||
for rottype in daily weekly monthly; do
|
||||
@ -405,12 +401,12 @@ function rotate_long {
|
||||
echo "Debug: $dir.1 does not exist, skipping."
|
||||
continue 1
|
||||
elif [ ! -f $metadata.1/created ] && [ ! -f $metadata.1/rotated ]; then
|
||||
echo "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation."
|
||||
warning "Warning: metadata does not exist for $dir.1. This backup may be only partially completed. Skipping rotation."
|
||||
continue 1
|
||||
fi
|
||||
|
||||
# Rotate the current list of backups, if we can.
|
||||
oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1`
|
||||
oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1`
|
||||
[ "$oldest" == "" ] && oldest=0
|
||||
for (( i=$oldest; i > 0; i-- )); do
|
||||
if [ -d $dir.$i ]; then
|
||||
@ -421,11 +417,16 @@ function rotate_long {
|
||||
else
|
||||
created=0
|
||||
fi
|
||||
# Validate created date
|
||||
if [ -z "$created" ] || echo $created | grep -v -q -e '^[0-9]*$'; then
|
||||
warning "Invalid metadata $created. Skipping rotation."
|
||||
break
|
||||
fi
|
||||
cutoff_time=$(( now - (seconds*(i-1)) ))
|
||||
if [ ! $created -gt $cutoff_time ]; then
|
||||
next=$(( i + 1 ))
|
||||
if [ ! -d $dir.$next ]; then
|
||||
echo "Debug: $rottype.$i --> $rottype.$next"
|
||||
debug "$rottype.$i --> $rottype.$next"
|
||||
$nice mv $dir.$i $dir.$next
|
||||
mkdir -p $metadata.$next
|
||||
date +%c%n%s > $metadata.$next/rotated
|
||||
@ -433,10 +434,10 @@ function rotate_long {
|
||||
$nice mv $metadata.$i/created $metadata.$next
|
||||
fi
|
||||
else
|
||||
echo "Debug: skipping rotation of $dir.$i because $dir.$next already exists."
|
||||
debug "skipping rotation of $dir.$i because $dir.$next already exists."
|
||||
fi
|
||||
else
|
||||
echo "Debug: skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)."
|
||||
debug "skipping rotation of $dir.$i because it was created" $(( (now-created)/86400)) "days ago ("$(( (now-cutoff_time)/86400))" needed)."
|
||||
fi
|
||||
fi
|
||||
done
|
||||
@ -444,7 +445,7 @@ function rotate_long {
|
||||
|
||||
max=$((keepdaily+1))
|
||||
if [ $keepweekly -gt 0 -a -d $backuproot/daily.$max -a ! -d $backuproot/weekly.1 ]; then
|
||||
echo "Debug: daily.$max --> weekly.1"
|
||||
debug "daily.$max --> weekly.1"
|
||||
$nice mv $backuproot/daily.$max $backuproot/weekly.1
|
||||
mkdir -p $backuproot/metadata/weekly.1
|
||||
date +%c%n%s > $backuproot/metadata/weekly.1/rotated
|
||||
@ -455,7 +456,7 @@ function rotate_long {
|
||||
|
||||
max=$((keepweekly+1))
|
||||
if [ $keepmonthly -gt 0 -a -d $backuproot/weekly.$max -a ! -d $backuproot/monthly.1 ]; then
|
||||
echo "Debug: weekly.$max --> monthly.1"
|
||||
debug "weekly.$max --> monthly.1"
|
||||
$nice mv $backuproot/weekly.$max $backuproot/monthly.1
|
||||
mkdir -p $backuproot/metadata/monthly.1
|
||||
date +%c%n%s > $backuproot/metadata/monthly.1/rotated
|
||||
@ -467,16 +468,16 @@ function rotate_long {
|
||||
for rottype in daily weekly monthly; do
|
||||
max=$((keep${rottype}+1))
|
||||
dir="$backuproot/$rottype"
|
||||
oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1`
|
||||
oldest=`find $backuproot -maxdepth 1 -type d -name $rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1`
|
||||
[ "$oldest" == "" ] && oldest=0
|
||||
# if we've rotated the last backup off the stack, remove it.
|
||||
for (( i=$oldest; i >= $max; i-- )); do
|
||||
if [ -d $dir.$i ]; then
|
||||
if [ -d $backuproot/rotate.tmp ]; then
|
||||
echo "Debug: removing rotate.tmp"
|
||||
debug "removing rotate.tmp"
|
||||
$nice rm -rf $backuproot/rotate.tmp
|
||||
fi
|
||||
echo "Debug: moving $rottype.$i to rotate.tmp"
|
||||
debug "moving $rottype.$i to rotate.tmp"
|
||||
$nice mv $dir.$i $backuproot/rotate.tmp
|
||||
fi
|
||||
done
|
||||
@ -509,7 +510,7 @@ function rotate_long_remote {
|
||||
now=\`date +%s\`
|
||||
|
||||
if [ ! -d "$backuproot" ]; then
|
||||
echo "Debug: skipping rotate of $backuproot as it doesn't exist."
|
||||
echo "Fatal: skipping rotate of $backuproot as it doesn't exist."
|
||||
exit
|
||||
fi
|
||||
|
||||
@ -528,7 +529,7 @@ function rotate_long_remote {
|
||||
fi
|
||||
|
||||
# Rotate the current list of backups, if we can.
|
||||
oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1\`
|
||||
oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1\`
|
||||
[ "\$oldest" == "" ] && oldest=0
|
||||
for (( i=\$oldest; i > 0; i-- )); do
|
||||
if [ -d \$dir.\$i ]; then
|
||||
@ -539,6 +540,11 @@ function rotate_long_remote {
|
||||
else
|
||||
created=0
|
||||
fi
|
||||
# Validate created date
|
||||
if [ -z "\$created" ] || echo \$created | grep -v -q -e '^[0-9]*$'; then
|
||||
echo "Warning: Invalid metadata \$created. Skipping rotation."
|
||||
break
|
||||
fi
|
||||
cutoff_time=\$(( now - (seconds*(i-1)) ))
|
||||
if [ ! \$created -gt \$cutoff_time ]; then
|
||||
next=\$(( i + 1 ))
|
||||
@ -585,7 +591,7 @@ function rotate_long_remote {
|
||||
for rottype in daily weekly monthly; do
|
||||
max=\$((keep\${rottype}+1))
|
||||
dir="$backuproot/\$rottype"
|
||||
oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1\`
|
||||
oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ -e 's/^.*\.//' | sort -n | tail -1\`
|
||||
[ "\$oldest" == "" ] && oldest=0
|
||||
# if we've rotated the last backup off the stack, remove it.
|
||||
for (( i=\$oldest; i >= \$max; i-- )); do
|
||||
@ -830,7 +836,7 @@ function update_metadata {
|
||||
# finished, otherwise daily rotations might not take place.
|
||||
# If we used backup end time, in the next handler run
|
||||
# we might not have $now - $created >= 24:00
|
||||
echo $starttime > $metadata/created
|
||||
echo "$starttime" > $metadata/created
|
||||
$touch $backupdir/$SECTION/$suffix
|
||||
else
|
||||
folder="`echo $dest_path | cut -d : -f 2`"
|
||||
@ -844,7 +850,7 @@ function update_metadata {
|
||||
# finished, otherwise daily rotations might not take place.
|
||||
# If we used backup end time, in the next handler run
|
||||
# we might not have $now - $created >= 24:00
|
||||
echo $starttime > $metadata/created
|
||||
echo "$starttime" > $metadata/created
|
||||
##### END REMOTE SCRIPT #######
|
||||
EOF
|
||||
) | (while read a; do passthru $a; done)
|
||||
@ -875,45 +881,6 @@ function test_connect {
|
||||
|
||||
}
|
||||
|
||||
function set_lockfile {
|
||||
|
||||
if [ ! -z "$lockfile" ]; then
|
||||
mkdir -p `dirname $lockfile`
|
||||
if ( set -o noclobber; echo "$$" > "$lockfile" ) &> /dev/null; then
|
||||
trap 'unset_lockfile' INT TERM EXIT
|
||||
else
|
||||
fatal "Could not create lockfile $lockfile, exiting"
|
||||
fi
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
function unset_lockfile {
|
||||
|
||||
if [ ! -z "$lockfile" ]; then
|
||||
$rm -f $lockfile || warning "Could not remove lockfile $lockfile"
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
function check_lockfile {
|
||||
|
||||
local pid process
|
||||
|
||||
if [ ! -z "$lockfile" ] && [ -f "$lockfile" ]; then
|
||||
pid="`cat $lockfile`"
|
||||
process="`ps --no-headers -o comm $pid`"
|
||||
if [ "$?" == "0" ] && [ "`ps --no-headers -o comm $$`" == "$process" ]; then
|
||||
info "Another backup is running for $lockfile, skipping run"
|
||||
exit
|
||||
else
|
||||
info "Found old lockfile $lockfile, removing it"
|
||||
unset_lockfile
|
||||
fi
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
function set_filelist {
|
||||
|
||||
filelist_flag=""
|
||||
@ -1100,18 +1067,38 @@ function end_mux {
|
||||
|
||||
}
|
||||
|
||||
function set_pipefail {
|
||||
|
||||
# Save initial pipefail status for later restoration
|
||||
if echo "$SHELLOPTS" | grep -q ":pipefail"; then
|
||||
pipefail="-o"
|
||||
else
|
||||
pipefail="+o"
|
||||
fi
|
||||
|
||||
# Ensure that a non-zero rsync exit status is caught by our handler
|
||||
set -o pipefail
|
||||
|
||||
}
|
||||
|
||||
function restore_pipefail {
|
||||
|
||||
if [ ! -z "$pipefail" ]; then
|
||||
set $pipefail pipefail
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
# the backup procedure
|
||||
|
||||
eval_config
|
||||
check_lockfile
|
||||
set_lockfile
|
||||
set_rsync_options
|
||||
start_mux
|
||||
stop_services
|
||||
mount_rw
|
||||
|
||||
starttime="`date +%c%n%s`"
|
||||
echo "Starting backup at `echo $starttime | head -n 1`" >> $log
|
||||
echo "Starting backup at `echo "$starttime" | head -n 1`" >> $log
|
||||
|
||||
for SECTION in $include; do
|
||||
|
||||
@ -1122,13 +1109,15 @@ for SECTION in $include; do
|
||||
set_dest
|
||||
|
||||
info "Syncing $SECTION on $dest_path..."
|
||||
debug $nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path
|
||||
$nice $rsync "${rsync_options[@]}" $filelist_flag $excludes $batch_option $orig $dest_path | tee -a $log
|
||||
debug $nice $rsync "${rsync_options[@]}" $filelist_flag "$excludes" $batch_option $orig $dest_path
|
||||
set_pipefail
|
||||
$nice $rsync "${rsync_options[@]}" $filelist_flag "$excludes" $batch_option $orig $dest_path | tee -a $log
|
||||
|
||||
if [ "$?" != "0" ]; then
|
||||
fatal "Rsync error when trying to transfer $SECTION"
|
||||
fi
|
||||
|
||||
restore_pipefail
|
||||
update_metadata
|
||||
|
||||
done
|
||||
@ -1136,7 +1125,6 @@ done
|
||||
mount_ro
|
||||
run_fsck
|
||||
start_services
|
||||
unset_lockfile
|
||||
end_mux
|
||||
|
||||
echo "Finnishing backup at `date`" >> $log
|
||||
|
||||
@ -349,7 +349,7 @@ function process_action() {
|
||||
|
||||
else
|
||||
# a backup is probably ongoing already, so display an error message
|
||||
debug "failed to acquire lock"
|
||||
debug "failed to acquire lock $lockfile"
|
||||
echo "Fatal: Could not acquire lock $lockfile. A backup is probably already running for $file." >>$bufferfile
|
||||
fi
|
||||
} 200> $lockfile
|
||||
|
||||
Loading…
Reference in New Issue
Block a user