rzerres/dsnap-sync
1
0
Fork 0
Code Issues Pull Requests Releases 1 Activity

dsnap-sync: rework BTRFS_PIPE handling

Browse Source 
* redirect any output of btrfs send/receive to a PIPE
* on failure, output saved messages to this PIPE
* cleanup PIPE on program termination

Signed-off-by: Ralf Zerres <ralf.zerres@networkx.de>
This commit is contained in:
Ralf Zerres
2018-09-28 14:34:48 +02:00
parent 6dcd0f7b45
commit 3f00ef6702
1 changed files with 54 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
bin/dsnap-sync Normal file → Executable file
View File

@@ -36,21 +36,21 @@ SNAPPER_CONFIG=/etc/default/snapper
SNAPPER_TEMPLATE_DIR=/etc/snapper/config-templates SNAPPER_TEMPLATE_DIR=/etc/snapper/config-templates
SNAPPER_CONFIG_DIR=/etc/snapper/configs SNAPPER_CONFIG_DIR=/etc/snapper/configs
# define fifo pipes # Create TEMPDIR
TMPDIR_PIPE=$(mktemp --tmpdir=$XDG_RUNTIME_DIR/$progname -d)
PIPE=$TMPDIR_PIPE/$progname.out
test ! -d $XDG_RUNTIME_DIR/$progname && mkdir -p $XDG_RUNTIME_DIR/$progname test ! -d $XDG_RUNTIME_DIR/$progname && mkdir -p $XDG_RUNTIME_DIR/$progname
mkfifo $PIPE TMPDIR=$(mktemp --tmpdir=$XDG_RUNTIME_DIR/$progname -d)
systemd-cat --identifier="$progname" < $PIPE &
BTRFS_PIPE=$TMPDIR_PIPE/btrfs.out # define fifo pipes
BTRFS_SEND_PIPE=$TMPDIR_PIPE/btrfs_send.out #PIPE=$TMPDIR/$progname.out
BTRFS_RECIEVE_PIPE=$TMPDIR_PIPE/btrfs_recieve.out #mkfifo $PIPE
#mkfifo $BTRFS_PIPE #systemd-cat --identifier="$progname" < $PIPE &
#systemd-cat --identifier="btrfs-pipe" < $BTRFS_PIPE &
BTRFS_PIPE=$TMPDIR/btrfs.fifo
test -p $BTRFS_PIPE && mkfifo $BTRFS_PIPE
# redirect descriptors to given pipes # redirect descriptors to given pipes
exec 3>$PIPE 4>$BTRFS_PIPE #exec 3>$PIPE
4>$BTRFS_PIPE
# global variables # global variables
args= args=
@@ -1447,6 +1447,7 @@ run_backup () {
if [ $donotify -gt 0 ]; then if [ $donotify -gt 0 ]; then
notify_info "Backup in progress" "NOTE: Skipping '$selected_config' configuration." notify_info "Backup in progress" "NOTE: Skipping '$selected_config' configuration."
fi fi
# go for next configuration
i=$(($i+1)) i=$(($i+1))
continue continue
fi fi
@@ -1497,10 +1498,14 @@ run_backup () {
verify_snapper_structure "backup_root=$backup_root" "snapper_target_config=$snapper_target_config" "snapper_target_id=$snapper_target_id" "remote=$remote" verify_snapper_structure "backup_root=$backup_root" "snapper_target_config=$snapper_target_config" "snapper_target_id=$snapper_target_id" "remote=$remote"
;; ;;
btrfs-archive) btrfs-archive)
# check for last common snapshot if [ $snapper_source_sync_id -eq 0 ]; then
snapper_source_id=$snapper_source_sync_id create_snapshot
snapper_source_snapshot=$SUBVOLUME/.snapshots/$snapper_source_sync_id/$snapper_snapshot_name else
snapper_source_info=$SUBVOLUME/.snapshots/$snapper_source_sync_id/info.xml # check for last common snapshot
snapper_source_id=$snapper_source_sync_id
snapper_source_snapshot=$SUBVOLUME/.snapshots/$snapper_source_sync_id/$snapper_snapshot_name
snapper_source_info=$SUBVOLUME/.snapshots/$snapper_source_sync_id/info.xml
fi
# targets backup location will save the snapshots in the subdirectory (snapshot-id) # targets backup location will save the snapshots in the subdirectory (snapshot-id)
# the snapshot is either the base snapshot (flat-file), or an incremental snapshot (btrfs-send stream) # the snapshot is either the base snapshot (flat-file), or an incremental snapshot (btrfs-send stream)
@@ -1541,7 +1546,6 @@ run_backup () {
#fi #fi
# need to substitue btrfs 'x.yyGiB' suffix, since pv will need 'xG' # need to substitue btrfs 'x.yyGiB' suffix, since pv will need 'xG'
#if [ $snapper_source_snapshot_size -ge 1073741824 ]; then
if [ $snapper_source_snapshot_size -ge 1048576 ]; then if [ $snapper_source_snapshot_size -ge 1048576 ]; then
snapper_source_snapshot_size=$(btrfs qgroup show -f --gbytes $snapper_source_snapshot 2>/dev/null \ snapper_source_snapshot_size=$(btrfs qgroup show -f --gbytes $snapper_source_snapshot 2>/dev/null \
| awk 'FNR>2 { gsub(/.[0-9][0-9]GiB/,"G"); print $2}') | awk 'FNR>2 { gsub(/.[0-9][0-9]GiB/,"G"); print $2}')
@@ -1592,28 +1596,29 @@ run_backup () {
case $selected_fstype in case $selected_fstype in
btrfs) btrfs)
cmd="btrfs send $btrfs_verbose_flag $snapper_source_snapshot 2>$BTRFS_SEND_PIPE \ cmd="btrfs send $btrfs_verbose_flag $snapper_source_snapshot 2>4 \
| $cmd_pv \ | $cmd_pv \
$cmd_ionice $ssh btrfs receive $btrfs_verbose_flag $snapper_target_snapshot/ 2>$BTRFS_RECIEVE_PIPE" $cmd_ionice $ssh btrfs receive $btrfs_verbose_flag $snapper_target_snapshot/ 1>4 2>&1"
;; ;;
*) *)
# Can't use btrfs receive, since target filesystem can't support btrfs snapshot feature # Can't use btrfs receive, since target filesystem can't support btrfs snapshot feature
snapper_target_stream=${snapper_target_id}_${archive_type}.btrfs snapper_target_stream=${snapper_target_id}_${archive_type}.btrfs
if [ ! -f $snapper_target_snapshot/$snapper_target_stream ]; then if [ ! -f $snapper_target_snapshot/$snapper_target_stream ]; then
if [ -z $remote ]; then if [ -z $remote ]; then
cmd="btrfs send $btrfs_verbose_flag $snapper_source_snapshot 2>$BTRFS_SEND_PIPE \ cmd="btrfs send $btrfs_verbose_flag $snapper_source_snapshot 2>/dev/null \
| $cmd_pv \ | $cmd_pv \
$cmd_ionice cat > $snapper_target_snapshot/$snapper_target_stream 2>$BTRFS_RECIEVE_PIPE" $cmd_ionice cat > $snapper_target_snapshot/$snapper_target_stream"
else else
cmd="btrfs send $btrfs_verbose_flag $snapper_source_snapshot 2>$BTRFS_SEND_PIPE \ cmd="btrfs send $btrfs_verbose_flag $snapper_source_snapshot 2>/dev/null \
| $cmd_pv \ | $cmd_pv \
$cmd_ionice $ssh 'cat > $snapper_target_snapshot/$snapper_target_stream' 2>$BTRFS_RECIEVE_PIPE" $cmd_ionice $ssh 'cat > $snapper_target_snapshot/$snapper_target_stream' 2>4"
fi fi
else else
if [ $verbose -ge 2 ]; then if [ $verbose -ge 2 ]; then
printf "${RED}BTRFS_Stream: %s${NO_COLOR} already saved.\n" \ printf "${RED}BTRFS_Stream: %s${NO_COLOR} already saved.\n" \
"$snapper_target_snapshot/$snapper_target_stream" "$snapper_target_snapshot/$snapper_target_stream"
fi fi
# go for next configuration
i=$(($i+1)) i=$(($i+1))
continue continue
fi fi
@@ -1631,11 +1636,13 @@ run_backup () {
if [ $verbose -ge 3 ]; then if [ $verbose -ge 3 ]; then
printf "cmd: '%s'\n" "$cmd" printf "cmd: '%s'\n" "$cmd"
fi fi
$(eval $cmd) 1>/dev/null $(eval $cmd)
if [ "$?" -gt 0 ]; then if [ "$?" -gt 0 ]; then
printf "${RED}BTRFS_SEND_PIPE: %s${NO_COLOR}\n" "$(cat $BTRFS_SEND_PIPE)" printf "${RED}BTRFS_PIPE: %s${NO_COLOR}\n" "$(cat <4)"
printf "${RED}BTRFS_RECIEVE_PIPE: %s${NO_COLOR}\n" "$(cat $BTRFS_RECIEVE_PIPE)" # go for next configuration
die "btrfs pipe error." i=$(($i+1))
continue
#die "btrfs pipe error."
fi fi
else else
# source holds synced snapshots # source holds synced snapshots
@@ -1707,21 +1714,21 @@ run_backup () {
# subvolume on the source and the receiving location (the parent-id). # subvolume on the source and the receiving location (the parent-id).
# using "btrfs send -p" instead of "btrfs send -c", then no parent search would be # using "btrfs send -p" instead of "btrfs send -c", then no parent search would be
# needed (Andreij explained in: https://www.spinics.net/lists/linux-btrfs/msg69369.html) # needed (Andreij explained in: https://www.spinics.net/lists/linux-btrfs/msg69369.html)
cmd="btrfs send $btrfs_verbose_flag -p $snapper_common_sync_snapshot $snapper_source_snapshot 2>$BTRFS_PIPE \ cmd="btrfs send $btrfs_verbose_flag -p $snapper_common_sync_snapshot $snapper_source_snapshot 2>4 \
| $cmd_pv \ | $cmd_pv \
$cmd_ionice $ssh btrfs receive $btrfs_verbose_flag $snapper_target_snapshot/ 2>$BTRFS_PIPE" $cmd_ionice $ssh btrfs receive $btrfs_verbose_flag $snapper_target_snapshot/ 1>4 2>&1"
;; ;;
*) *)
# Can't use btrfs receive, since target filesystem can't support btrfs snapshot feature # Can't use btrfs receive, since target filesystem can't support btrfs snapshot feature
snapper_target_stream=${snapper_target_id}_incremental.btrfs snapper_target_stream=${snapper_target_id}_incremental.btrfs
if [ -z $remote ]; then if [ -z $remote ]; then
cmd="btrfs send $btrfs_verbose_flag -p $snapper_common_sync_snapshot $snapper_source_snapshot 2>$BTRFS_SEND_PIPE \ cmd="btrfs send $btrfs_verbose_flag -p $snapper_common_sync_snapshot $snapper_source_snapshot 2>4 \
| $cmd_pv \ | $cmd_pv \
$cmd_ionice cat > $snapper_target_snapshot/$snapper_target_stream 2>$BTRFS_PIPE" $cmd_ionice cat > $snapper_target_snapshot/$snapper_target_stream 2>4"
else else
cmd="btrfs send $btrfs_verbose_flag -p $snapper_common_sync_snapshot $snapper_source_snapshot 2>$BTRFS_SEND_PIPE \ cmd="btrfs send $btrfs_verbose_flag -p $snapper_common_sync_snapshot $snapper_source_snapshot 2>4 \
| $cmd_pv \ | $cmd_pv \
$cmd_ionice $ssh 'cat > $snapper_target_snapshot/$snapper_target_stream' 2>$BTRFS_PIPE" $cmd_ionice $ssh 'cat > $snapper_target_snapshot/$snapper_target_stream' 2>4"
fi fi
;; ;;
esac esac
@@ -1734,21 +1741,19 @@ run_backup () {
if [ $verbose -ge 3 ]; then if [ $verbose -ge 3 ]; then
printf "${GREEN}btrfs command:${NO_COLOR} '%s'\n" "$cmd" printf "${GREEN}btrfs command:${NO_COLOR} '%s'\n" "$cmd"
fi fi
$(eval $cmd) 2>&1 1>/dev/null $(eval $cmd)
case $? in ret=$?
case $ret in
0) 0)
;; ;;
127) 127)
printf "${MAGENTA}btrfs pipe return-code: ${GREEN}'127'${NO_COLOR}\n" printf "${MAGENTA}btrfs pipe return-code: ${GREEN}'127'${NO_COLOR}\n"
;; ;;
*) *)
printf "${RED}btfs pipe ERROR!\n" printf "${RED}btfs pipe ERROR: '%s'!\n" "$ret"
printf "${RED}BTRFS_PIPE: %s${NO_COLOR}\n" "$(cat $BTRFS_PIPE)" printf "${RED}BTRFS_PIPE: %s${NO_COLOR}\n" "$(cat <4)"
#printf "${RED}BTRFS_SEND_PIPE: %s${NO_COLOR}\n" "$(cat $BTRFS_SEND_PIPE)"
#printf "${RED}BTRFS_RECIEVE_PIPE: %s${NO_COLOR}\n" "$(cat $BTRFS_RECIEVE_PIPE)"
#die "btrfs pipe error."
# go for next configuration
run_cleanup ${selected_config} run_cleanup ${selected_config}
# go for next configuration
i=$(($i+1)) i=$(($i+1))
continue continue
;; ;;
@@ -1863,7 +1868,6 @@ run_finalize () {
snapper_source_info=$(eval echo \$snapper_source_info_$i) snapper_source_info=$(eval echo \$snapper_source_info_$i)
snapper_target_config=$(eval echo \$snapper_target_config_$i) snapper_target_config=$(eval echo \$snapper_target_config_$i)
snapper_target_snapshot=$(eval echo \$snapper_target_snapshot_$i) snapper_target_snapshot=$(eval echo \$snapper_target_snapshot_$i)
#tape_id=$(eval echo \$tape_id_$i)
# It's important not to change the values of the snapper key/value pairs ($userdata) # It's important not to change the values of the snapper key/value pairs ($userdata)
# which is stored in snappers info.xml file of the source snapshot. # which is stored in snappers info.xml file of the source snapshot.
@@ -1894,7 +1898,7 @@ run_finalize () {
local ii_sleep=15 local ii_sleep=15
# Solution2: kill running snapperd # Solution2: kill running snapperd
# -> will restart and sync; any unseen interdependencies # -> will restart and sync any unseen dependencies
snapperd_pid=$(eval $ssh pgrep snapperd) snapperd_pid=$(eval $ssh pgrep snapperd)
if [ $verbose -ge 3 ]; then if [ $verbose -ge 3 ]; then
printf "${MAGENTA}Kill runnint ${GREEN}snapperd${MAGENTA} on target id:${GREEN}'%s'${NO_COLOR} ...\n" \ printf "${MAGENTA}Kill runnint ${GREEN}snapperd${MAGENTA} on target id:${GREEN}'%s'${NO_COLOR} ...\n" \
@@ -2008,7 +2012,6 @@ run_finalize () {
if [ $verbose -ge 3 ]; then if [ $verbose -ge 3 ]; then
printf "${MAGENTA}Tagging snapper metadata${NO_COLOR} for snapper id ${GREEN}'%s'${NO_COLOR} on source for configuration ${GREEN}'%s'${NO_COLOR} ...\n" \ printf "${MAGENTA}Tagging snapper metadata${NO_COLOR} for snapper id ${GREEN}'%s'${NO_COLOR} on source for configuration ${GREEN}'%s'${NO_COLOR} ...\n" \
"$snapper_source_id" "$selected_config" "$snapper_source_id" "$selected_config"
#printf "calling: '%s'\n" "$(eval $cmd)"
printf "calling: '%s'\n" "$cmd" printf "calling: '%s'\n" "$cmd"
fi fi
ret=$(eval "$cmd") ret=$(eval "$cmd")
@@ -2018,7 +2021,7 @@ run_finalize () {
sync sync
fi fi
if [ ${#snapper_source_sync_id} -gt 0 ]; then if [ ${#snapper_source_sync_id} -gt 0 ]; then
# TODO: no method to remove userdata pair, use awk # TODO: no snapper method to remove userdata pair, use awk
cmd="snapper --config $selected_config modify \ cmd="snapper --config $selected_config modify \
--description \"$snap_description_finished\" \ --description \"$snap_description_finished\" \
--userdata \"important=no\" \ --userdata \"important=no\" \
@@ -2952,15 +2955,16 @@ run_config_preparation
run_backup run_backup
# cleanup # cleanup
#if [ -f $PIPE ]; then if [ -d $TMPDIR ]; then
# rm -f $PIPE || die "Failed to cleanup temporary pipe '%s'\n" "$PIPE" rm -rf $TMPDIR || die "Failed to cleanup temporary directory '%s'\n" "$TMPDIR"
#fi #printf "${RED}TODO: ${NO_COLOR}Will remove $TMPDIR}"
if [ -d $TMPDIR_PIPE ]; then
rm -rf $TMPDIR_PIPE || die "Failed to cleanup temporary directory '%s'\n" "$TMPDIR_PIPE"
fi fi
printf "${BLUE}Backups done!${NO_COLOR}\n" printf "${BLUE}Backups done!${NO_COLOR}\n"
exec 3>&-
# close the read file descriptor
#exec 3>&-
exec 4>&-
if [ $donotify -gt 0 ]; then if [ $donotify -gt 0 ]; then
if [ "$uuid_cmdline" != "none" ]; then if [ "$uuid_cmdline" != "none" ]; then