rzerres/dsnap-sync
1
0
Fork 0
Code Issues Pull Requests Releases 1 Activity

dsnap-sync: handle transfer size

Browse Source 
* new function check_transfer_size()
  will calculate the transfer size send via btrfs-send
* update run_backup()
  - handle dryrun
  - handle crate-pv_cmd()
  - send incremantal size

Signed-off-by: Ralf Zerres <ralf.zerres@networkx.de>
This commit is contained in:
Ralf Zerres
2019-12-18 02:22:44 +01:00
committed by Ralf Zerres
parent 01c9e511a5
commit 48c784b35d
1 changed files with 245 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files

190
bin/dsnap-sync
View File

@@ -27,7 +27,7 @@
# difference for the next incremental snapshot.
progname="${0##*/}"
version="0.6.5"
version="0.6.5.1"
# The following lines are modified by the Makefile or
# find_snapper_config script
@@ -79,6 +79,7 @@ snapper_backup_type='none'
#snapper_config_postfix="."`hostname`
snapper_config_postfix=
snap_cleanup_algorithm="timeline"
transfer_size=0
verbose=0
volume_name=
@@ -161,6 +162,103 @@ check_snapper_failed_ids () {
fi
}
check_transfer_size () {
local source_snapshot=${1##source_snapshot=}
local clone_snapshot=${2##clone_snapshot=}
if [ $verbose -ge 3 ]; then
printf "${MAGENTA}check_transfer_size()...${NO_COLOR}\n"
fi
if [ $dryrun -eq 0 ]; then
transfer_size=0
if [ "$interactive" -eq 1 ]; then
if [ $verbose -ge 2 ]; then
if [ ${#clone_snapshot} -gt 0 ]; then
printf "${MAGENTA}Calculate transfer size for incremental snapshot (clone=${GREEN}'%s'${MAGENTA}, source=${GREEN}'%s'${MAGENTA})${NO_COLOR} ...\n" \
"$clone_snapshot" "$source_snapshot"
else
printf "${MAGENTA}Calculate transfer size for snapshot (source=${GREEN}'%s'${MAGENTA})${NO_COLOR} ...\n" \
"$ssource_snapshot"
fi
fi
if [ $btrfs_quota -eq 1 ]; then
# qgroup for given path, exclude ancestrals
# qgroup identifiers conform to level/id where level 0 is reserved to the qgroups associated with subvolumes
transfer_size=$(btrfs qgroup show -f --raw $source_snapshot 2>/dev/null \
| awk 'FNR>2 {print $2}')
if [ $? -eq 1 ]; then
# subvolume is not configured for quota, (temporary?, expensive?) enable that
if [ $btrfs_quota_tmp -eq 1 ]; then
btrfs quota enable $source_snapshot 2>/dev/null
btrfs quota rescan -w $source_snapshot 2>/dev/null
transfer_size=$(btrfs qgroup show -f --raw $source_snapshot 2>/dev/null \
| awk 'FNR>2 {print $2}')
fi
fi
if [ $verbose -ge 3 ]; then
printf "${MAGENTA}BTRFS qgroup show result: ${GREEN}'%s'\b${NO_COLOR}'%s'\n" \
"$?" "$transfer_size"
fi
# need to substitue btrfs 'x.yyGiB' suffix, since pv will need 'xG'
if [ $transfer_size -ge 1048576 ]; then
transfer_size=$(btrfs qgroup show -f --gbytes $source_snapshot 2>/dev/null \
| awk 'FNR>2 { gsub(/.[0-9][0-9]GiB/,"G"); print $2}')
fi
if [ $verbose -ge 2 ]; then
printf "${MAGENTA}BTRFS quota size for ${GREEN}source snapshot${MAGENTA}: size=${GREEN}'%s'${NO_COLOR} ...\n" \
"$snapper_source_id" "$transfer_size"
fi
# should we disable quota usage again?
if [ $btrfs_quota_tmp -eq 1 ]; then btrfs quota disable $source_snapshot; fi
else
if [ ${#clone_snapshot} -gt 0 ]; then
# WIP: dry run with btrfs send
exec 5>&1
transfer_size=$(btrfs send -v -p $clone_snapshot $source_snapshot \
| pv -f 2>&1 > /dev/null | tee >(cat - >&5) \
| awk -F ' ' '{print $1}' )
else
# filesystem size
transfer_size=$(du --one-file-system --summarize $snapper_source_snapshot 2>/dev/null \
| awk -F ' ' '{print $1}')
fi
if [ $transfer_size -ge 1048576 ]; then
transfer_size=$(($transfer_size / 1024 / 1024))G
fi
if [ $verbose -ge 2 ]; then
printf "${MAGENTA}BTRFS transfer size for ${GREEN}source snapshot${MAGENTA}: size=${GREEN}'%s'${NO_COLOR} ...\n" \
"$transfer_size"
fi
fi
fi
else
if [ $verbose -ge 2 ]; then
printf "${MAGENTA}dryrun: Would calculate transfer size for BTRFS ${GREEN}source snapshot${NO_COLOR} ...\n" \
"$snapper_source_id"
fi
fi
}
create_pv_cmd () {
if [ $verbose -ge 3 ]; then
printf "${MAGENTA}create_pv_cmd()...${NO_COLOR}\n"
fi
# prepare cmdline output settings for interactive progress status
if [ $do_pv_cmd -eq 1 ]; then
pv_options="--delay-start 2 --interval 5 --format \"time elapsed [%t] | avg rate %a | rate %r | transmitted [%b] | %p | time remaining [%e]\" "
cmd_pv="pv --size $transfer_size $pv_options | "
#cmd_pv="pv $pv_options --size ${transfer_size} | dialog --gauge \"$progname: Progress for config '$selected_config'\" 6 85 |"
else
cmd_pv=''
fi
}
create_snapshot () {
if [ $verbose -ge 3 ]; then
printf "${MAGENTA}create_snapshot()...${NO_COLOR}\n" $snapper_config
@@ -1616,64 +1714,6 @@ run_backup () {
continue
fi
if [ $dryrun -eq 0 ]; then
snapper_source_snapshot_size=0
if [ "$interactive" -eq 1 ]; then
if [ $verbose -ge 2 ]; then
printf "${MAGENTA}Get size for given source snapshot (id=${GREEN}'%s'${MAGENTA}, path=${GREEN}'%s'${MAGENTA})${NO_COLOR} ...\n" \
"$snapper_source_id" "$snapper_source_snapshot"
fi
if [ $btrfs_quota -eq 1 ]; then
# qgroup for given path, exclude ancestrals
# qgroup identifiers conform to level/id where level 0 is reserved to the qgroups associated with subvolumes
snapper_source_snapshot_size=$(btrfs qgroup show -f --raw $snapper_source_snapshot 2>/dev/null \
| awk 'FNR>2 {print $2}')
if [ $? -eq 1 ]; then
# subvolume is not configured for quota, (temporary?) enable that
if [ $btrfs_quota_tmp -eq 1 ]; then
btrfs quota enable $snapper_source_snapshot 2>/dev/null
btrfs quota rescan -w $snapper_source_snapshot 2>/dev/null
snapper_source_snapshot_size=$(btrfs qgroup show -f --raw $snapper_source_snapshot 2>/dev/null \
| awk 'FNR>2 {print $2}')
fi
fi
if [ $verbose -ge 3 ]; then
printf "${MAGENTA}BTRFS qgroup show result: ${GREEN}'%s'\b${NO_COLOR}'%s'\n" \
"$?" "$snapper_source_snapshot_size"
fi
# need to substitue btrfs 'x.yyGiB' suffix, since pv will need 'xG'
if [ $snapper_source_snapshot_size -ge 1048576 ]; then
snapper_source_snapshot_size=$(btrfs qgroup show -f --gbytes $snapper_source_snapshot 2>/dev/null \
| awk 'FNR>2 { gsub(/.[0-9][0-9]GiB/,"G"); print $2}')
fi
if [ $verbose -ge 2 ]; then
printf "${MAGENTA}BTRFS quota size for ${GREEN}source snapshot${MAGENTA}: id=${GREEN}'%s'${MAGENTA}, size=${GREEN}'%s'${NO_COLOR} ...\n" \
"$snapper_source_id" "$snapper_source_snapshot_size"
fi
# should we disable quota usage again?
if [ $btrfs_quota_tmp -eq 1 ]; then btrfs quota disable $snapper_source_snapshot; fi
else
snapper_source_snapshot_size=$(du --one-file-system --summarize $snapper_source_snapshot 2>/dev/null \
| awk -F ' ' '{print $1}')
if [ $snapper_source_snapshot_size -ge 1048576 ]; then
snapper_source_snapshot_size=$(($snapper_source_snapshot_size / 1024 / 1024))G
fi
if [ $verbose -ge 2 ]; then
printf "${MAGENTA}BTRFS subvolume size for ${GREEN}source snapshot${MAGENTA}: id=${GREEN}'%s'${MAGENTA}, size=${GREEN}'%s'${NO_COLOR} ...\n" \
"$snapper_source_id" "$snapper_source_snapshot_size"
fi
fi
fi
else
if [ $verbose -ge 2 ]; then
printf "${MAGENTA}dryrun: Would calculate BTRFS subvolume size for ${GREEN}source snapshot${NO_COLOR} ...\n" \
"$snapper_source_id"
fi
fi
# setting process I/O scheduling options
if [ $do_ionice_cmd -eq 1 ]; then
# class: best-efford, priority: medium
@@ -1684,15 +1724,17 @@ run_backup () {
cmd_ionice=''
fi
# settings for interactive progress status
if [ $do_pv_cmd -eq 1 ]; then
pv_options="--delay-start 2 --interval 5 --format \"time elapsed [%t] | avg rate %a | rate %r | transmitted [%b] | %p | time remaining [%e]\" "
cmd_pv="pv --size $snapper_source_snapshot_size $pv_options | "
#cmd_pv="pv $pv_options --size ${snapper_source_snapshot_size} | dialog --gauge \"$progname: Progress for config '$selected_config'\" 6 85 |"
else
cmd_pv=''
fi
# prepare pipe command
if [ "$dryrun" -eq 0 ]; then
if [ "$snapper_source_sync_id" -eq 0 ] \
|| [ "$snapper_target_sync_id" -eq 0 ] \
|| [ "$backup_mode" = "full" ] ; then
# get size of stream that needs to be transfered
check_transfer_size "source_snapshot=$snapper_source_snapshot"
# prepare send pipe command
create_pv_cmd
case $selected_fstype in
btrfs)
cmd="btrfs send $btrfs_verbose_flag $snapper_source_snapshot 2>$BTRFS_PIPE \
@@ -1723,15 +1765,12 @@ run_backup () {
fi
;;
esac
if [ "$dryrun" -eq 0 ]; then
if [ "$snapper_source_sync_id" -eq 0 ] \
|| [ "$snapper_target_sync_id" -eq 0 ] \
|| [ "$backup_mode" = "full" ] ; then
# send full snapshot to target
if [ $verbose -ge 2 ]; then
if [ ${#snapper_source_snapshot_size} -gt 0 ]; then
if [ ${#transfer_size} -gt 0 ]; then
printf "${MAGENTA}Sending ${GREEN}snapshot${NO_COLOR} for snapper config ${GREEN}'%s' ${MAGENTA}(id='%s', size='%s')${NO_COLOR} ...\n" \
"$selected_config" "$snapper_source_id" "$snapper_source_snapshot_size"
"$selected_config" "$snapper_source_id" "$transfer_size"
else
printf "${MAGENTA}Sending ${GREEN}snapshot${NO_COLOR} for snapper config ${GREEN}'%s' ${MAGENTA}(id='%s')${NO_COLOR} ...\n" \
"$selected_config"
@@ -1768,6 +1807,7 @@ run_backup () {
printf "${MAGENTA}Prepare ${GREEN}incremental snapshot${NO_COLOR} (id: ${GREEN}'%s'${NO_COLOR}) for snapper config ${GREEN}'%s'${NO_COLOR} ...\n" \
"$snapper_target_id" "$selected_config"
fi
# verify that we have a matching source and target snapshot-id
if [ $snapper_common_sync_id -eq 0 ]; then
if [ ${snapper_source_id} -eq ${snapper_target_sync_id} ]; then
@@ -1814,6 +1854,10 @@ run_backup () {
cmd="$ssh stat --format %i $snapper_target_snapshot 2>/dev/null"
ret=$(eval $cmd)
if [ $? -eq 0 ]; then
# get size of stream that needs to be transfered
check_transfer_size "source_snapshot=$snapper_source_snapshot" "clone_snapshot=snapper_common_sync_snapshot"
create_pv_cmd
case $selected_fstype in
btrfs)
# Sends the difference between the new snapshot and old synced snapshot.