Enable use of ionice for all handlers to alleviate IO load

On some workloads, a process creating backups should not become so
aggressive on resources that it prevents normal activity of a machine
from running. It is especially important when writing to disk.

One way to make backups have less impact on the main workload is to
force backups to a lower ionice level.

With the new ionicelevel configuration, it becomes possible for users to
activate the use of ionice and to define the level within the
best-effort class to better suit their needs.
This commit is contained in:
Gabriel Filion 2020-05-16 12:02:43 -04:00
parent 0491432147
commit 5c6c583181
7 changed files with 82 additions and 19 deletions

View File

@ -21,6 +21,7 @@ export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes
getconf testconnect yes
getconf nicelevel 0
getconf ionicelevel
getconf bwlimit
setsection source
@ -72,6 +73,17 @@ else
unset BORG_CACHE_DIR
fi
# Check that the ionicelevel is valid
if [ -n "$ionicelevel" ] && echo "$ionicelevel" | grep -vq "^[0-7]$"; then
fatal "The value of ionicelevel is expected to be either empty or an integer from 0 to 7. Got: $ionicelevel"
fi
# Only use ionice if ionicelevel is not empty
nice="nice -n $nicelevel"
if [ -n "$ionicelevel" ]; then
nice="ionice -c2 -n $ionicelevel $nice"
fi
# check the connection at the source and destination
[ -n "$test" ] || test=0
if [ "$host" != "localhost" ] && ([ "$testconnect" = "yes" ] || [ "${test}" -eq 1 ]); then
@ -146,10 +158,10 @@ fi
# include client-part and server-part
execstr="${execstr} ${excludes} $execstr_repository::$execstr_archive ${includes}"
debug "$execstr"
debug "$nice $execstr"
if [ $test = 0 ]; then
output=`nice -n $nicelevel su -c "$execstr" 2>&1`
output=`$nice su -c "$execstr" 2>&1`
if [ $? = 0 ]; then
debug $output
info "Successfully finished backing up source $label"

View File

@ -259,6 +259,7 @@ do_dup_misc_options() {
formBegin "$dup_title - misc. options"
formItem "nicelevel" "$dup_nicelevel"
formItem "ionicelevel" "$dup_ionicelevel"
formItem "testconnect" "$dup_testconnect"
formItem "options" "$dup_options"
formDisplay
@ -271,8 +272,9 @@ do_dup_misc_options() {
IFS=$' \t\n'
dup_nicelevel=${thereply[0]}
dup_testconnect=${thereply[1]}
dup_options=${thereply[2]}
dup_ionicelevel=${thereply[1]}
dup_testconnect=${thereply[2]}
dup_options=${thereply[3]}
set +o noglob
}
@ -294,6 +296,8 @@ options = $dup_options
# default is 0, but set to 19 if you want to lower the priority.
nicelevel = $dup_nicelevel
# by default (when value is empty), ionice will not be used.
ionicelevel = $dup_ionicelevel
# default is yes. set to no to skip the test if the remote host is alive.
# if 'desturl' is set below, 'testconnect' must be set to 'no' for now.
@ -619,6 +623,7 @@ dup_wizard() {
dup_gpg_password=""
dup_gpg_signpassword=""
dup_nicelevel=19
dup_ionicelevel=
dup_testconnect=yes
dup_options=

View File

@ -8,6 +8,7 @@
getconf options
getconf testconnect yes
getconf nicelevel 0
getconf ionicelevel
getconf tmpdir
setsection gpg
@ -80,6 +81,17 @@ if [ "`echo $desturl | @AWK@ -F ':' '{print $1}'`" == "file" ]; then
fi
fi
# Check that the ionicelevel is valid
if [ -n "$ionicelevel" ] && echo "$ionicelevel" | grep -vq "^[0-7]$"; then
fatal "The value of ionicelevel is expected to be either empty or an integer from 0 to 7. Got: $ionicelevel"
fi
# Only use ionice if ionicelevel is not empty
nice="nice -n $nicelevel"
if [ -n "$ionicelevel" ]; then
nice="ionice -c2 -n $ionicelevel $nice"
fi
### VServers
# If vservers are configured, check that the ones listed in $vsnames do exist.
usevserver=no
@ -305,13 +317,12 @@ fi
### Cleanup commands (duplicity >= 0.4.4)
# cleanup
debug "$execstr_precmd duplicity cleanup --force $execstr_options $execstr_serverpart"
debug "$nice $execstr_precmd duplicity cleanup --force $execstr_options $execstr_serverpart"
if [ ! $test ]; then
export PASSPHRASE=$password
export SIGN_PASSPHRASE=$signpassword
export FTP_PASSWORD=$ftp_password
output=`nice -n $nicelevel \
su -c \
output=`$nice su -c \
"$execstr_precmd duplicity cleanup --force $execstr_options $execstr_serverpart 2>&1"`
exit_code=$?
if [ $exit_code -eq 0 ]; then
@ -325,13 +336,12 @@ fi
# remove-older-than
if [ "$keep" != "yes" ]; then
debug "$execstr_precmd duplicity remove-older-than $keep --force $execstr_options $execstr_serverpart"
debug "$nice $execstr_precmd duplicity remove-older-than $keep --force $execstr_options $execstr_serverpart"
if [ ! $test ]; then
export PASSPHRASE=$password
export SIGN_PASSPHRASE=$signpassword
export FTP_PASSWORD=$ftp_password
output=`nice -n $nicelevel \
su -c \
output=`$nice su -c \
"$execstr_precmd duplicity remove-older-than $keep --force $execstr_options $execstr_serverpart 2>&1"`
exit_code=$?
if [ $exit_code -eq 0 ]; then
@ -348,13 +358,12 @@ fi
if [ "$keep" != "yes" ]; then
if [ "$keepincroffulls" != "all" ]; then
if version_ge "$duplicity_version" '0.6.10'; then
debug "$execstr_precmd duplicity remove-all-inc-of-but-n-full $keepincroffulls --force $execstr_options $execstr_serverpart"
debug "$nice $execstr_precmd duplicity remove-all-inc-of-but-n-full $keepincroffulls --force $execstr_options $execstr_serverpart"
if [ ! $test ]; then
export PASSPHRASE=$password
export SIGN_PASSPHRASE=$signpassword
export FTP_PASSWORD=$ftp_password
output=`nice -n $nicelevel \
su -c \
output=`$nice su -c \
"$execstr_precmd duplicity remove-all-inc-of-but-n-full $keepincroffulls --force $execstr_options $execstr_serverpart 2>&1"`
exit_code=$?
if [ $exit_code -eq 0 ]; then
@ -370,14 +379,13 @@ if [ "$keep" != "yes" ]; then
fi
### Backup command
debug "$execstr_precmd duplicity $execstr_command $execstr_options $execstr_source --exclude '**' / $execstr_serverpart"
debug "$nice $execstr_precmd duplicity $execstr_command $execstr_options $execstr_source --exclude '**' / $execstr_serverpart"
if [ ! $test ]; then
outputfile=`maketemp backupout`
export PASSPHRASE=$password
export SIGN_PASSPHRASE=$signpassword
export FTP_PASSWORD=$ftp_password
output=`nice -n $nicelevel \
su -c \
output=`$nice su -c \
"$execstr_precmd duplicity $execstr_command $execstr_options $execstr_source --exclude '**' / $execstr_serverpart >$outputfile 2>&1"`
exit_code=$?
debug "$output"

View File

@ -12,6 +12,20 @@ getconf isoonly yes
getconf imagefile backup.iso
getconf device
getconf nicelevel 0
getconf ionicelevel
## Check config
# Check that the ionicelevel is valid
if [ -n "$ionicelevel" ] && echo "$ionicelevel" | grep -vq "^[0-7]$"; then
fatal "The value of ionicelevel is expected to be either empty or an integer from 0 to 7. Got: $ionicelevel"
fi
# Only use ionice if ionicelevel is not empty
nice="nice -n $nicelevel"
if [ -n "$ionicelevel" ]; then
nice="ionice -c2 -n $ionicelevel $nice"
fi
# define needed executables:
MKISOFS="/usr/bin/genisoimage"
@ -36,7 +50,7 @@ if [ "$isoonly" == "no" ]; then
fi
outputfile="$backupdir/$imagefile"
execstr="nice -n $nicelevel $MKISOFS --quiet -R -o $outputfile "
execstr="$nice $MKISOFS --quiet -R -o $outputfile "
str=""
# excludes

View File

@ -83,6 +83,7 @@ function check_cstream() {
getconf options
getconf testconnect yes
getconf nicelevel 0
getconf ionicelevel
getconf bwlimit
getconf ignore_version no
getconf output_as_info no
@ -133,6 +134,17 @@ else
[ -z "$vsinclude" ] || warning 'vservers support disabled in backupninja.conf, vsincludes configuration lines will be ignored'
fi
# Check that the ionicelevel is valid
if [ -n "$ionicelevel" ] && echo "$ionicelevel" | grep -vq "^[0-7]$"; then
fatal "The value of ionicelevel is expected to be either empty or an integer from 0 to 7. Got: $ionicelevel"
fi
# Only use ionice if ionicelevel is not empty
nice="nice -n $nicelevel"
if [ -n "$ionicelevel" ]; then
nice="ionice -c2 -n $ionicelevel $nice"
fi
# check the connection at the source and destination
[ -n "$test" ] || test=0
if [ "$testconnect" = "yes" ] || [ "${test}" -eq 1 ]; then
@ -269,9 +281,9 @@ set +o noglob
# include client-part and server-part
execstr="${execstr}$execstr_sourcepart $execstr_destpart"
debug "$execstr"
debug "$nice $execstr"
if [ $test = 0 ]; then
output=`nice -n $nicelevel su -c "$execstr" 2>&1`
output=`$nice su -c "$execstr" 2>&1`
if [ $? = 0 ]; then
if [ "$output_as_info" == "yes" ]; then
info "$output"

View File

@ -40,6 +40,7 @@
# keepweekly = for long storage format, specify the number of weekly backup increments
# keepmonthly = for long storage format, specify the number of monthly backup increments
# nicelevel = rsync command nice level
# ionicelevel = best-effort nice level applied to rsync and other commands
# enable_mv_timestamp_bug = set to "yes" if your system isnt handling timestamps correctly
# tmp = temp folder
# multiconnection = set to "yes" if you want to use multiconnection ssh support
@ -129,6 +130,7 @@ function eval_config {
getconf keepweekly 3
getconf keepmonthly 1
getconf nicelevel 0
getconf ionicelevel
getconf enable_mv_timestamp_bug no
getconf tmp /tmp
getconf multiconnection no
@ -258,6 +260,10 @@ function eval_config {
nice=""
fi
if [ -n "$ionicelevel" ]; then
nice="ionice -c2 -n $ionicelevel $nice"
fi
ssh_cmd_base="ssh -T -o PasswordAuthentication=no -p $port -i $id_file"
ssh_cmd="$ssh_cmd_base $user@$host"

View File

@ -19,6 +19,7 @@
# days = number of backup increments (min = 5)
# lockfile = lockfile to be kept during backup execution
# nicelevel = wget command nice level
# ionicelevel = ionice best-effort level applied to wget command
# enable_mv_timestamp_bug = set to "yes" if your system isnt handling timestamps correctly
# tmp = temp folder
#
@ -63,6 +64,7 @@ getconf rotate
getconf days
getconf lockfile
getconf nicelevel 0
getconf ionicelevel
getconf enable_mv_timestamp_bug no
getconf tmp /tmp
@ -145,6 +147,10 @@ else
nice=""
fi
if [ -n "$ionicelevel" ]; then
nice="ionice -c2 -n $ionicelevel $nice"
fi
# set mv procedure
if [ $enable_mv_timestamp_bug == "yes" ]; then