Documentation: update README in line with v6.5.0

* README.md (english)
* README.md (german)
* correction the relative path selection

Signed-off-by: Ralf Zerres <ralf.zerres@networkx.de>

README.md

@@ -5,8 +5,8 @@
 
 <p align="center">
   <span>English</span> |
-  <!-- a href="lang/spanish#dsnap-sync">Spanish</a> | -->
-  <a href="lang/german#dsnap-sync">Deutsch</a>
+  <!-- a href="lang/spanish">Spanish</a> | -->
+  <a href="lang/german">Deutsch</a>
 </p>
 
 ## About
@@ -304,6 +304,34 @@ FUSE). Read and write access can be managed using common OS tools.
 An open-source implementation can be found at
 [LinearTapeFileSystem](https://github.com/LinearTapeFileSystem/ltfs).
 
+## Restore
+
+### From Tape
+
+When `dsnap-sync` did save the data with method `btrfs-archive`, you will find
+the corresponding data in a snapper compatible directory structure on the tape.
+
+The structure may look like:
+
+ └── backups
+	 └── @<server-name>
+		├── archive-<subvol-name>
+		│   └── <subvol-id>
+		│       ├── <subvol-id>_full.btrfs
+		│       └── info.xml
+
+The file `info.xml` provide the metadata corresponding to the snapshot.
+The data of the snapshot is stored in the file `<subvol-id>_full.btrfs`.
+This file has to be decrypted with btrfs tool `btrfs-send` to a btrfs
+restore directory:
+
+  cd /target_btrfs_path
+  cp /path_to_tape_root/backups/@<server-name>/archive-<subvol-name>/<subvol-id>_full.btrfs .
+  cat <subvol-id>_full.btrfs  | btrfs receive -v  .
+  rm <subvol-id>_full.btrfs
+
+Please consult btrfs-send man-page for further info.
+
 ## Contributing
 
 Help is very welcome! Feel free to fork and issue a pull request to add
@@ -331,4 +359,4 @@ This work is licensed under a [Creative Common License 4.0][License-CC_BY]
 ![Creative Common Logo][Logo-CC_BY]
 
 © 2016, 2017  James W. Barnett;
-© 2017 - 2018 Ralf Zerres
+© 2017 - 2019 Ralf Zerres

bin/dsnap-sync

@@ -27,7 +27,7 @@
 # difference for the next incremental snapshot.
 
 progname="${0##*/}"
-version="0.6.5"
+version="0.6.5.1"
 
 # The following lines are modified by the Makefile or
 # find_snapper_config script
@@ -79,6 +79,7 @@ snapper_backup_type='none'
 #snapper_config_postfix="."`hostname`
 snapper_config_postfix=
 snap_cleanup_algorithm="timeline"
+transfer_size=0
 verbose=0
 volume_name=
 
@@ -161,6 +162,103 @@ check_snapper_failed_ids () {
     fi
 }
 
+check_transfer_size () {
+    local source_snapshot=${1##source_snapshot=}
+    local clone_snapshot=${2##clone_snapshot=}
+
+    if [ $verbose -ge 3 ]; then
+	printf "${MAGENTA}check_transfer_size()...${NO_COLOR}\n"
+    fi
+
+    if [ $dryrun -eq 0 ]; then
+	transfer_size=0
+	if [ "$interactive" -eq 1 ]; then
+	    if [ $verbose -ge 2 ]; then
+		if [ ${#clone_snapshot} -gt 0 ]; then
+		    printf "${MAGENTA}Calculate transfer size for incremental snapshot (clone=${GREEN}'%s'${MAGENTA}, source=${GREEN}'%s'${MAGENTA})${NO_COLOR} ...\n" \
+			   "$clone_snapshot" "$source_snapshot"
+		else
+		    printf "${MAGENTA}Calculate transfer size for snapshot (source=${GREEN}'%s'${MAGENTA})${NO_COLOR} ...\n" \
+			   "$ssource_snapshot"
+		fi
+	    fi
+	    if [ $btrfs_quota -eq 1 ]; then
+		# qgroup for given path, exclude ancestrals
+		# qgroup identifiers conform to level/id where level 0 is reserved to the qgroups associated with subvolumes
+		transfer_size=$(btrfs qgroup show -f --raw $source_snapshot 2>/dev/null \
+						   | awk 'FNR>2 {print $2}')
+		if [ $? -eq 1 ]; then
+		    # subvolume is not configured for quota, (temporary?, expensive?) enable that
+		    if [ $btrfs_quota_tmp -eq 1 ]; then
+			btrfs quota enable $source_snapshot 2>/dev/null
+			btrfs quota rescan -w $source_snapshot 2>/dev/null
+			transfer_size=$(btrfs qgroup show -f --raw $source_snapshot 2>/dev/null \
+					    | awk 'FNR>2 {print $2}')
+		    fi
+		fi
+
+		if [ $verbose -ge 3 ]; then
+		    printf "${MAGENTA}BTRFS qgroup show result: ${GREEN}'%s'\b${NO_COLOR}'%s'\n" \
+			   "$?" "$transfer_size"
+		fi
+
+		# need to substitue btrfs 'x.yyGiB' suffix, since pv will need 'xG'
+		if [ $transfer_size -ge 1048576 ]; then
+		    transfer_size=$(btrfs qgroup show -f --gbytes $source_snapshot 2>/dev/null \
+						       | awk 'FNR>2 { gsub(/.[0-9][0-9]GiB/,"G"); print $2}')
+		fi
+		if [ $verbose -ge 2 ]; then
+		    printf "${MAGENTA}BTRFS quota size for ${GREEN}source snapshot${MAGENTA}: size=${GREEN}'%s'${NO_COLOR} ...\n" \
+			   "$snapper_source_id" "$transfer_size"
+		fi
+
+		# should we disable quota usage again?
+		if [ $btrfs_quota_tmp -eq 1 ]; then btrfs quota disable $source_snapshot; fi
+	    else
+		if [ ${#clone_snapshot} -gt 0 ]; then
+		    # WIP: dry run with btrfs send
+		    # need to substitue btrfs 'x.yyGiB' suffix, since pv will need 'xG'
+		    transfer_size=$(btrfs send -v -p $clone_snapshot $source_snapshot 2>$BTRFS_PIPE \
+					| pv -f 2>&1 >/dev/null \
+					| awk -F ' ' '{ gsub(/.[0-9][0-9]MiB/,"M"); gsub(/.[0-9][0-9]GiB/,"G"); print $1 }' )
+		else
+		    # filesystem size
+		    transfer_size=$(du --one-file-system --summarize $snapper_source_snapshot 2>/dev/null \
+					| awk -F ' ' '{print $1}')
+		    if [ $transfer_size -ge 1048576 ]; then
+			transfer_size=$(($transfer_size / 1024 / 1024))G
+		    fi
+		fi
+		if [ $verbose -ge 2 ]; then
+		    printf "${MAGENTA}BTRFS transfer size for ${GREEN}source snapshot${MAGENTA}: size=${GREEN}'%s'${NO_COLOR} ...\n" \
+			   "$transfer_size"
+		fi
+	    fi
+	fi
+    else
+	if [ $verbose -ge 2 ]; then
+	    printf "${MAGENTA}dryrun: Would calculate transfer size for BTRFS ${GREEN}source snapshot${NO_COLOR} ...\n" \
+		   "$snapper_source_id"
+	fi
+    fi
+
+}
+
+create_pv_cmd () {
+    if [ $verbose -ge 3 ]; then
+	printf "${MAGENTA}create_pv_cmd()...${NO_COLOR}\n"
+    fi
+
+    # prepare cmdline output settings for interactive progress status
+    if [ $do_pv_cmd -eq 1 ]; then
+	pv_options="--delay-start 2  --interval 5 --format \"time elapsed [%t] | avg rate %a | rate %r | transmitted [%b] | %p | time remaining [%e]\" "
+	cmd_pv="pv --size $transfer_size $pv_options | "
+	#cmd_pv="pv $pv_options --size ${transfer_size} | dialog --gauge \"$progname: Progress for config '$selected_config'\" 6 85 |"
+    else
+	cmd_pv=''
+    fi
+}
+
 create_snapshot () {
     if [ $verbose -ge 3 ]; then
 	printf "${MAGENTA}create_snapshot()...${NO_COLOR}\n" $snapper_config
@@ -1616,64 +1714,6 @@ run_backup () {
 	    continue
 	fi
 
-	if [ $dryrun -eq 0 ]; then
-	    snapper_source_snapshot_size=0
-	    if [ "$interactive" -eq 1 ]; then
-		if [ $verbose -ge 2 ]; then
-		    printf "${MAGENTA}Get size for given source snapshot (id=${GREEN}'%s'${MAGENTA}, path=${GREEN}'%s'${MAGENTA})${NO_COLOR} ...\n" \
-			   "$snapper_source_id" "$snapper_source_snapshot"
-		fi
-
-		if [ $btrfs_quota -eq 1 ]; then
-		    # qgroup for given path, exclude ancestrals
-		    # qgroup identifiers conform to level/id where level 0 is reserved to the qgroups associated with subvolumes
-		    snapper_source_snapshot_size=$(btrfs qgroup show -f --raw $snapper_source_snapshot 2>/dev/null \
-						       | awk 'FNR>2 {print $2}')
-                    if [ $? -eq 1 ]; then
-                        # subvolume is not configured for quota, (temporary?) enable that
-                        if [ $btrfs_quota_tmp -eq 1 ]; then
-                            btrfs quota enable $snapper_source_snapshot 2>/dev/null
-                            btrfs quota rescan -w $snapper_source_snapshot 2>/dev/null
-                            snapper_source_snapshot_size=$(btrfs qgroup show -f --raw $snapper_source_snapshot 2>/dev/null \
-                                                               | awk 'FNR>2 {print $2}')
-			fi
-		    fi
-
-		    if [ $verbose -ge 3 ]; then
-			printf "${MAGENTA}BTRFS qgroup show result: ${GREEN}'%s'\b${NO_COLOR}'%s'\n" \
-	    		       "$?" "$snapper_source_snapshot_size"
-		    fi
-
-		    # need to substitue btrfs 'x.yyGiB' suffix, since pv will need 'xG'
-		    if [ $snapper_source_snapshot_size -ge 1048576 ]; then
-			snapper_source_snapshot_size=$(btrfs qgroup show -f --gbytes $snapper_source_snapshot 2>/dev/null \
-							   | awk 'FNR>2 { gsub(/.[0-9][0-9]GiB/,"G"); print $2}')
-		    fi
-		    if [ $verbose -ge 2 ]; then
-			printf "${MAGENTA}BTRFS quota size for ${GREEN}source snapshot${MAGENTA}: id=${GREEN}'%s'${MAGENTA}, size=${GREEN}'%s'${NO_COLOR} ...\n" \
-			       "$snapper_source_id" "$snapper_source_snapshot_size"
-		    fi
-
-		    # should we disable quota usage again?
-		    if [ $btrfs_quota_tmp -eq 1 ]; then btrfs quota disable $snapper_source_snapshot; fi
-		else
-		    snapper_source_snapshot_size=$(du --one-file-system --summarize $snapper_source_snapshot 2>/dev/null \
-						       | awk -F ' ' '{print $1}')
-		    if [ $snapper_source_snapshot_size -ge 1048576 ]; then
-			snapper_source_snapshot_size=$(($snapper_source_snapshot_size / 1024 / 1024))G
-		    fi
-		    if [ $verbose -ge 2 ]; then
-			printf "${MAGENTA}BTRFS subvolume size for ${GREEN}source snapshot${MAGENTA}: id=${GREEN}'%s'${MAGENTA}, size=${GREEN}'%s'${NO_COLOR} ...\n" \
-			       "$snapper_source_id" "$snapper_source_snapshot_size"
-		    fi
-		fi
-	    fi
-	else
-	    if [ $verbose -ge 2 ]; then
-		printf "${MAGENTA}dryrun: Would calculate BTRFS subvolume size for ${GREEN}source snapshot${NO_COLOR} ...\n" \
-		       "$snapper_source_id"
-	    fi
-	fi
 	# setting process I/O scheduling options
 	if [ $do_ionice_cmd -eq 1 ]; then
 	    # class: best-efford, priority: medium
@@ -1684,15 +1724,17 @@ run_backup () {
 	    cmd_ionice=''
 	fi
 
-	# settings for interactive progress status
-	if [ $do_pv_cmd -eq 1 ]; then
-	    pv_options="--delay-start 2  --interval 5 --format \"time elapsed [%t] | avg rate %a | rate %r | transmitted [%b] | %p | time remaining [%e]\" "
-	    cmd_pv="pv --size $snapper_source_snapshot_size $pv_options | "
-	    #cmd_pv="pv $pv_options --size ${snapper_source_snapshot_size} | dialog --gauge \"$progname: Progress for config '$selected_config'\" 6 85 |"
-	else
-	    cmd_pv=''
-	fi
+	# prepare pipe command
+	if [ "$dryrun" -eq 0 ]; then
+	    if [ "$snapper_source_sync_id" -eq 0 ] \
+		   || [ "$snapper_target_sync_id" -eq 0 ] \
+		   || [ "$backup_mode" = "full" ] ; then
 
+		# get size of stream that needs to be transfered
+		check_transfer_size "source_snapshot=$snapper_source_snapshot"
+
+		# prepare send pipe command
+		create_pv_cmd
 		case $selected_fstype in
 		    btrfs)
 			cmd="btrfs send $btrfs_verbose_flag $snapper_source_snapshot 2>$BTRFS_PIPE \
@@ -1723,15 +1765,12 @@ run_backup () {
 			fi
 			;;
 		esac
-	if [ "$dryrun" -eq 0 ]; then
-	    if [ "$snapper_source_sync_id" -eq 0 ] \
-		   || [ "$snapper_target_sync_id" -eq 0 ] \
-		   || [ "$backup_mode" = "full" ] ; then
+
 		# send full snapshot to target
 		if [ $verbose -ge 2 ]; then
-		    if [ ${#snapper_source_snapshot_size} -gt 0 ]; then
+		    if [ ${#transfer_size} -gt 0 ]; then
 			printf "${MAGENTA}Sending ${GREEN}snapshot${NO_COLOR} for snapper config ${GREEN}'%s' ${MAGENTA}(id='%s', size='%s')${NO_COLOR} ...\n" \
-			       "$selected_config" "$snapper_source_id" "$snapper_source_snapshot_size"
+			       "$selected_config" "$snapper_source_id" "$transfer_size"
 		    else
 			printf "${MAGENTA}Sending ${GREEN}snapshot${NO_COLOR} for snapper config ${GREEN}'%s' ${MAGENTA}(id='%s')${NO_COLOR} ...\n" \
 			       "$selected_config"
@@ -1768,6 +1807,7 @@ run_backup () {
 		    printf "${MAGENTA}Prepare ${GREEN}incremental snapshot${NO_COLOR} (id: ${GREEN}'%s'${NO_COLOR}) for snapper config ${GREEN}'%s'${NO_COLOR} ...\n" \
 			   "$snapper_target_id" "$selected_config"
 		fi
+
 		# verify that we have a matching source and target snapshot-id
 		if [ $snapper_common_sync_id -eq 0 ]; then
 		    if [ ${snapper_source_id} -eq ${snapper_target_sync_id} ]; then
@@ -1814,6 +1854,10 @@ run_backup () {
 		cmd="$ssh stat --format %i $snapper_target_snapshot 2>/dev/null"
 		ret=$(eval $cmd)
 		if [ $? -eq 0 ]; then
+		    # get size of stream that needs to be transfered
+		    check_transfer_size "source_snapshot=$snapper_source_snapshot" "clone_snapshot=$snapper_common_sync_snapshot"
+		    create_pv_cmd
+
 		    case $selected_fstype in
 			btrfs)
 			    # Sends the difference between the new snapshot and old synced snapshot.

bin/tape-admin

@@ -22,7 +22,7 @@
 # Helper routines for tape handling
 
 progname="${0##*/}"
-version="0.0.14"
+version="0.0.15"
 
 # global variables
 color=0
@@ -901,8 +901,8 @@ ltfs_reformat () {
 	    fi
 	    # wiping: put tape in an unformatted state
 	    ltfs_wipe
-	    # RET = 8 -> error?
 	    if [ $? -eq 0 ]; then
+		# format: create the ltfs partitions
 		ltfs_format $volume_name_active $volume_serial
 		if [ $? -eq 0 ]; then
 		    ltfs_mount
@@ -1091,7 +1091,7 @@ mount_tape () {
 
     ltfs_is_mounted
     if test $? -eq 0; then
-        # get date from last written tape in given mediapool
+	# if no explicit tape should be mounted, get date from last written tape in given mediapool
 	if [ "${#volume_name}" -eq 0 ]; then
 	    get_lastwrite ${mediapool_name}
 	    if [ $? != 0 ]; then
@@ -1128,7 +1128,7 @@ mount_tape () {
 	# check if given tape is poolmember of selected pool
 	get_poolmember ${mediapool_name} ${volume_name}
 	if [ $? -eq 0 ]; then
-	    # volume_name is member of given mediapool
+	    # check the mediapolicy for given volume_name
 	    get_mediapolicy ${mediapool_name} ${volume_name_active}
 	    if [ ${#volume_mediapolicy} -gt 0  ]; then
 		if [ ${volume_mediapolicy} = "append" ] ; then
@@ -1147,6 +1147,8 @@ mount_tape () {
 			      printf "${MAGENTA}Valid tape, retensiondate has exposed: ${GREEN}%s${NO_COLOR}\n" \
 				     "${volume_retensiondate}"
 			  fi
+			  #ltfs_format ${volume_name_active}
+			  ltfs_reformat ${volume_name_active}
 			  return 0
 		      fi
 		      if [ $? -eq 1 ]; then
@@ -1188,7 +1190,7 @@ mount_tape () {
 				     "${volume_name}"
 			  fi
 			  ltfs_wipe
-			  #ltfs_format ${volume_name} ${volume_serial}
+			  #ltfs_reformat ${volume_name_active}
 			  if [ $? -eq 0 ]; then
 			      return 0
 			  fi
@@ -1298,7 +1300,7 @@ mount_tape () {
 		RET=$?
 		if [ $RET -eq 2 ]; then
 		    if [ $verbose -ge 1 ]; then
-			printf "${MAGENTA}RetensionDate has exposed, mount the tape\n"
+			printf "${MAGENTA}RetensionDate has exposed, mount the tape${NO_COLOR}\n"
 		    fi
 		    ltfs_mount
 		    return 0
@@ -1326,7 +1328,7 @@ mount_tape () {
 		if [ $? -eq 2 ]; then
 		    # retensiondate has exposed: wipe given tape
 		    ltfs_wipe
-		    #ltfs_format ${volume_name} ${volume_serial}
+		    ltfs_format ${volume_name} ${volume_serial}
 		    if [ $? -eq 0 ]; then
 			return 0
 		    fi
@@ -1386,7 +1388,7 @@ mtx_getlabel () {
 	    volume_name_active=$(perl -ne '
 		/Data Transfer Element (\d+):Full \(Storage Element (\d+) Loaded\)(:VolumeTag =\s*(.+))?/ && print "$4\n";' ${TMPFILE})
 	    volume_name_active=$(echo $volume_name_active | sed -e 's/ *$//g')
-	    if [ $verbose -ge 2 ]; then
+	    if [ $verbose -ge 2 ] && [ ${#volume_name_active} -gt 0 ]; then
 		printf "${MAGENTA}Tape in slot ${GREEN}%s${MAGENTA} has Label: ${GREEN}%s${NO_COLOR}\n" \
 		       "${slot_source}" "${volume_name_active}"
 	    fi
@@ -2405,76 +2407,70 @@ case $cmd in
 	fi
 	;;
     ltfs-is-mounted)
+	if ! $quiet; then
+	    printf "${MAGENTA}LTFS tape mount checke\n"
+	fi
 	ltfs_is_mounted
 	if test $? -gt 0; then
 	    exit 1
-	else
-	    if ! $quiet; then
-		printf "${MAGENTA}LTFS Tape is-mounted: ${GREEN}true${NO_COLOR}\n"
-	    fi
 	fi
 	;;
     ltfs-mount)
+	if ! $quiet; then
+	    printf "${MAGENTA}LTFS mount Tape\n"
+	fi
 	ltfs_mount
 	if test $? -gt 0; then
 	    exit 1
-	else
-	    if ! $quiet; then
-		printf "${MAGENTA}LTFS Tape mount: ${GREEN}true${NO_COLOR}\n"
-	    fi
 	fi
        ;;
     ltfs-reformat)
-	ltfs_reformat "${volume_name}"
-	if test $? -gt 0; then
-	    return 1
-	else
 	if ! $quiet; then
 	    printf "${MAGENTA}LTFS Tape reformat: ${GREEN}%s${NO_COLOR}\n" \
 		   "${volume_name}"
 	fi
+	ltfs_reformat "${volume_name}"
+	if test $? -gt 0; then
+	    return 1
 	fi
 	;;
     ltfs-umount)
-	ltfs_umount
-	if test $? -gt 0; then
-	    exit 1
-	else
 	if ! $quiet; then
 	    printf "${MAGENTA}LTFS Tape unmount: ${GREEN}true${NO_COLOR}\n"
 	fi
+	ltfs_umount
+	if test $? -gt 0; then
+	    exit 1
 	fi
 	;;
     media-change)
+	if ! $quiet; then
+	    if  [ ${#volume_name} -ge 1 ]; then
+		printf "${MAGENTA}Media change in pool: ${GREEN}%s${NO_COLOR} (Volume-Name: ${GREEN}%s${NO_COLOR})\n" \
+		       "${mediapool_name}" "${volume_name}"
+	    else
+		printf "${MAGENTA}Media change in pool: ${GREEN}%s${NO_COLOR}\n" \
+		       "${mediapool_name}"
+	    fi
+	fi
 	media_change $mediapool_name $volume_name
 	if test $? -gt 0; then
 	    return 1
-	else
-	    if ! $quiet; then
-		if  [ ${#volume_name} -ge 1 ]; then
-		    printf "${MAGENTA}Media changed in pool: ${GREEN}%s${NO_COLOR} (Volume-Name: ${GREEN}%s${NO_COLOR})\n" \
-			   "${mediapool_name}" "${volume_name}"
-		else
-		    printf "${MAGENTA}Media changed in pool: ${GREEN}%s${NO_COLOR}\n" \
-			   "${mediapool_name}"
-		fi
-	    fi
 	fi
 	;;
     mount)
-        mount_tape "${mediapool_name}" "${volume_name}"
-	if test $? -gt 0; then
-	    exit 1
-	else
 	if ! $quiet; then
 	    if test ${#volume_name} -ge 1; then
-		    printf "${MAGENTA}Tape ${GREEN}'%s'${MAGENTA} for ${GREEN}'%s'${MAGENTA} mounted${NO_COLOR}\n" \
+		printf "${MAGENTA}Mount tape ${GREEN}'%s'${MAGENTA} for ${GREEN}'%s'${NO_COLOR}\n" \
 		       "${volume_name}" "${mediapool_name}"
 	    else
-		    printf "${MAGENTA}Next tape for mediapool ${GREEN}'%s'${MAGENTA} mounted${NO_COLOR}\n" \
+		printf "${MAGENTA}Mount next tape for mediapool ${GREEN}'%s'${NO_COLOR}\n" \
 		       "${mediapool_name}"
 	    fi
 	fi
+	mount_tape "${mediapool_name}" "${volume_name}"
+	if test $? -gt 0; then
+	    exit 1
 	fi
 	;;
     mtx-exchange)

lang/german/README.md

@@ -3,6 +3,13 @@
 
 # dsnap-sync
 
+<p align="center">
+  <span>English</span> |
+  <a href="../..">Englisch</a>
+  <!-- a href="../spanish">Spanisch</a> | -->
+  <a href="lang/german">Deutsch</a>
+</p>
+
 ## Über
 
 `dsnap-sync` ist konzipiert, um Backups für btrfs formatierte Dateisysteme
@@ -357,4 +364,4 @@ Diese Arbeit ist unter der [Creative Common License 4.0][License-CC_BY] lizensie
 ![Creative Common Logo][Logo-CC_BY]
 
 © 2016, 2017  James W. Barnett;
-© 2017 - 2018 Ralf Zerres
+© 2017 - 2019 Ralf Zerres