upgrad to v0.6.10

* update README

README.md

@@ -1,5 +1,5 @@
 <!-- dsnap-sync README.md -->
-<!-- version: 0.6.6 -->
+<!-- version: 0.6.10 -->
 
 # dsnap-sync
 
@@ -184,6 +184,7 @@ Please use your host software package manager.
 		  --label-running <desc>  snapper description tagging active jobs. Default: "dsnap-sync in progress"
 		  --label-synced <desc>   snapper description tagging last synced jobs.
 								  Default: "dsnap-sync last incremental"
+          --calculate-btrfs-size  Enable calculation of sync-size for given snapshots
 		  --color                 Enable colored output messages
 	  -c, --config <config>       Specify the snapper configuration to use. Otherwise will perform for each snapper
 								  configuration. You can select multiple configurations
@@ -191,6 +192,7 @@ Please use your host software package manager.
 		  --config-postfix <name> Specify a postfix that will be appended to the destination snapper config name.
 		  --dry-run               perform a trial run (no changes are written).
 		  --mediapool             Specify the name of the tape MediaPool
+          --no-btrfs-quota        don't consume btrfs-quota to estimate snapshot size
 	  -n, --noconfirm             Do not ask for confirmation for each configuration. Will still prompt for backup
 		  --nonotify              Disable graphical notification (via dbus)
 		  --nopv                  Disable graphical progress output (disable pv)
@@ -387,4 +389,4 @@ This work is licensed under a [Creative Common License 4.0][License-CC_BY]
 ![Creative Common Logo][Logo-CC_BY]
 
 © 2016, 2017  James W. Barnett;
-© 2017 - 2021 Ralf Zerres
+© 2017 - 2025 Ralf Zerres

TODO.md

@@ -4,7 +4,7 @@
 
 - dsnap-sync: restore btrfs-streams from archive backups
   * find last full
-  * iterate oval available incremental snapshots
+  * iterate over available incremental snapshots
 - dsnap-sync: restore btrfs snapshot from snapshot backups
   * $ssh btrfs send `<snapshot_path>/<snapshot-id>/snapshot_ro` | btrfs receive `/<btrfs-restore-dir>/`
 	(`recieved_uuid` attribte of `<btrfs-restore-dir>/snapshot_ro` will be imported from `<snapshot_path>/<snapshot-id>/snapshot_ro`)

bin/dsnap-sync

@@ -39,7 +39,7 @@
 # pair of systemd service and timer-units.
 
 progname="${0##*/}"
-version="0.6.8"
+version="0.6.10"
 
 # global variables
 args=
@@ -49,7 +49,7 @@ batch=0
 btrfs_quota=0
 btrfs_quota_tmp=1
 btrfs_verbose_flag=
-calculate_btrfs_size=0
+calculate_btrfs_size=1
 color=0
 donotify=0
 dryrun=0
@@ -168,8 +168,10 @@ check_transfer_size () {
                            "$source_snapshot"
                 fi
             fi
+
+            # btrfs quotas are expensive and should not be activated if you handle bigger numbers of snapshots
             if [ $btrfs_quota -eq 1 ]; then
-                # qgroup for given path, exclude ancestrals
+                # use qgroup for given path, exclude ancestrals
                 # qgroup identifiers conform to level/id where level 0 is reserved to the qgroups associated with subvolumes
                 transfer_size=$(btrfs qgroup show -f --raw "$source_snapshot" 2>/dev/null \
                                                    | awk 'FNR>2 {print $2}')
@@ -188,7 +190,7 @@ check_transfer_size () {
                 fi
 
                 if [ "$verbose" -ge 3 ]; then
-                    printf "${MAGENTA}BTRFS qgroup show result: ${GREEN}'%s'\b${NO_COLOR}'%s'\n" \
+                    printf "${MAGENTA}BTRFS Quoata-Group for snapshot show result: ${GREEN}'%s'\b${NO_COLOR}'%s'\n" \
                            "$?" "$transfer_size"
                 fi
 
@@ -205,6 +207,8 @@ check_transfer_size () {
                 # should we disable quota usage again?
                 if [ "$btrfs_quota_tmp" -eq 1 ]; then btrfs quota disable "$source_snapshot"; fi
             else
+                # no quota: get an aproximated value for the transfer_size
+                # - not accurate, but inexpensive
                 if [ ${#clone_snapshot} -gt 0 ]; then
                     # WIP: dry run with btrfs send
                     # need to substitue btrfs 'x.yyGiB' suffix, since pv will need 'xG'
@@ -212,12 +216,9 @@ check_transfer_size () {
                                         | pv -f 2>&1 >/dev/null \
                                         | awk -F ' ' '{ gsub(/.[0-9][0-9]MiB/,"M"); gsub(/.[0-9][0-9]GiB/,"G"); print $1 }' )
                 else
-                    # filesystem size
-                    transfer_size=$(du --one-file-system --summarize "$snapper_source_snapshot" 2>/dev/null \
-                                        | awk -F ' ' '{print $1}')
-                    if [ "$transfer_size" -ge 1048576 ]; then
-                        transfer_size=$(($transfer_size / 1024 / 1024))G
-                    fi
+                    # btrfs calculated filesystem size (--si: use 1000 as a base (kB, MB, GB, TB)
+                    transfer_size=$(btrfs filesystem df --si --gbytes "$snapper_source_snapshot" 2>/dev/null \
+                                    | awk -F '=' 'NR==1 {gsub(/.[0-9][0-9]kB/,"K"); gsub(/.[0-9][0-9]MB/,"M"); gsub(/.[0-9][0-9]GB/,"G"); gsub(/.[0-9][0-9]TB/,"T"); print $3} ')
                 fi
                 if [ "$verbose" -ge 2 ]; then
                     printf "${MAGENTA}BTRFS transfer size for ${GREEN}source snapshot${MAGENTA}: size=${GREEN}'%s'${NO_COLOR} ...\n" \
@@ -231,7 +232,6 @@ check_transfer_size () {
                    "$snapper_source_id"
         fi
     fi
-
 }
 
 cleanup_snapper_failed_ids () {
@@ -261,15 +261,15 @@ cleanup_snapper_failed_ids () {
         if [ "$answer" = "yes" ]; then
             failed_id_first=$(snapper --config "$selected_config" list --type single \
                                   | awk -F '|' ' /'"$snap_description_running"'/' \
-                                  | awk ' NR==1 {print $1} ')
+                                  | awk ' NR==1 {print $1} ')q
             failed_id_last=$(snapper --config "$selected_config" list --type single \
                                  | awk -F '|' ' /'"$snap_description_running"'/' \
                                  | awk ' END {print $1} ')
             cmd="snapper --config $selected_config delete"
             if [ "$failed_id_first" -lt "$failed_id_last" ]; then
-                $(eval "$cmd" "$failed_id_first"-"$failed_id_last")
+                eval "$cmd" "$failed_id_first"-"$failed_id_last"
             else
-                $(eval "$cmd" "$failed_id_first")
+                eval "$cmd" "$failed_id_first"
             fi
         fi
     fi
@@ -594,7 +594,7 @@ get_disk_infos () {
             disk_subvolid_match="$i"
             disk_subvolid_match_count=$((disk_subvolid_match_count+1))
         fi
-        $(eval "fs_options_$i='$fs_option'")
+        eval "fs_options_$i='$fs_option'"
         i=$((i+1))
     done
 }
@@ -778,24 +778,31 @@ get_snapper_last_sync_id () {
     fi
 
     snapper_sync_id=0
-    cmd="stat --format %i $snapper_config_dir/$snapper_config 2>/dev/null"
     if [ ${#remote_host} -ge 1 ]; then
         run_ssh="$ssh"
     fi
 
-    # only process, if config does exist
-    if  [ $(eval "$run_ssh" "$cmd") -eq 1 ]; then
+    # only process, if snapper config file does exist
+    # return value for 'cmd':
+    #   0 -> filepath is valid
+    #   1 -> filepath does not exist
+    cmd="stat --format %n $snapper_config_dir/$snapper_config 2>/dev/null"
+    #printf "${MAGENTA}Check for given snapper config ${GREEN}'%s'${MAGENTA} (remote host: ${GREEN}'%s'${MAGENTA}).${NO_COLOR}\n" \
+    #       "$cmd" "$remote_host"
+    if  [ ! "$(eval "$run_ssh" "$cmd")" ]; then
         if [ "$verbose" -ge 3 ]; then
             if [ "${#remote_host}" -ge 1 ]; then
                 printf "${MAGENTA}snapper config ${GREEN}'%s'${MAGENTA} on remote ${GREEN}'%s'${MAGENTA} does not exist yet.${NO_COLOR}\n" \
                        "$snapper_config" "$remote_host"
             else
-                printf "${MAGENTA}snapper config ${GREEN}'%s'${MAGENTA} does not exist yet.${NO_COLOR}\n" \
+                printf "${MAGENTA}snapper config ${GREEN}'%s'${MAGENTA} on source host does not exist.${NO_COLOR}\n" \
                        "$snapper_config"
             fi
         fi
         return 1
     fi
+
+    # filter sync process id
     if [ "${#snapper_subvolid}" -ge 1 ] && [ "${#snapper_uuid}" -ge 1 ]; then
         cmd="snapper --config $snapper_config list --type single \
              | awk '/$snapper_description/' \
@@ -1114,8 +1121,8 @@ parse_params () {
                 backuptype_cmdline="$2"
                 shift 2
                 ;;
-            --calculate-btrfs_size)
-                calulate_size=1
+            --calculate-btrfs-size)
+                calculate_btrfs_size=1
                 shift 1
                 ;;
             -c|--config)
@@ -1262,7 +1269,7 @@ parse_params () {
                 snapper_config_postfix="${1#*=}"
                 shift
                 ;;
-            --calculate-btrfs_size=*)
+            --calculate-btrfs-size=*)
                 case ${1#*=} in
                     yes | Yes | True | true)
                         calculate_btrfs_size=1;
@@ -1408,7 +1415,7 @@ parse_params () {
             nc -w 3 -z "$remote" "$port" > /dev/null || \
                 die "Can't connect to remote host."
         else
-            "$ssh" command -pv sh >/dev/null 2>&1 || \
+            eval "$ssh" command -pv sh >/dev/null 2>&1 || \
             { printf "'remote shell' is not working!\n \
                   Please correct your public authentication and try again.\n" && exit 1; }
         fi
@@ -1528,7 +1535,7 @@ run_config_preparation () {
             fi
 
             # get latest successfully finished snapshot on source
-            # WIP: metadata from last snapshot!
+            # WIP: parse metadata from last snapshot!
             case "$snapper_backup_type" in
             btrfs-snapshot)
                 get_snapper_last_sync_id "snapper_config=${selected_config}" "snapper_description=${snap_description_synced}" \
@@ -1566,7 +1573,7 @@ run_config_preparation () {
             snapper_target_sync_snapshot='none'
             backup_root="$selected_target/$backupdir/$snapper_target_config"
         else
-            # Set snapshot-path for source
+            # Set snapper snapshot-path for source
             get_snapper_config_value "remote=" \
                                      "snapper_config=$snapper_config_dir/$selected_config" \
                                      "config_key=SUBVOLUME"
@@ -1582,7 +1589,7 @@ run_config_preparation () {
                        "$selected_config" "$snapper_sync_id"
             fi
 
-            # verfiy target
+            # get latest successfully finished snapshot on target
             case "$snapper_backup_type" in
                 btrfs-archive)
                     # set snapper_target_sync_id
@@ -1604,9 +1611,12 @@ run_config_preparation () {
                 snapper_target_sync_id=0
             fi
 
-            # check for corresponding source and target sync id's
-            snapper_common_sync_id=0
+            # if source and target sync id's do not match:
+	    # check for last common sync id's
+
             if [ "$snapper_target_sync_id" -ne "$snapper_source_sync_id" ]; then
+                snapper_common_sync_id=0
+
                 # select commen sync id
                 get_snapper_sync_id "snapper_config=${selected_config}" "remote="
                 if [ $? -eq 0 ]; then
@@ -1626,6 +1636,8 @@ run_config_preparation () {
                     # no commen sync id found
                     snapper_target_sync_id=0
                 fi
+            else
+                snapper_common_sync_id=$snapper_source_sync_id
 	    fi
 
             if [ "$snapper_target_sync_id" -eq 0 ]; then
@@ -1876,14 +1888,19 @@ run_backup () {
                    || [ "$snapper_target_sync_id" -eq 0 ] \
                    || [ "$backup_mode" = "full" ] ; then
 
-                # get size of stream that needs to be transfered
+                # send full snapshot to target
                 if [ $calculate_btrfs_size -eq 1 ]; then
+                    # get size of stream that needs to be transfered
                     check_transfer_size "source_snapshot=$snapper_source_snapshot"
                 fi
 
                 # prepare send pipe command
                 create_pv_cmd
-                case "selected_fstype" in
+                if [ "$verbose" -ge 3 ]; then
+                    printf "${MAGENTA}Selected Filesystem tye: ${GREEN}'%s'${NO_COLOR}\n" \
+                           "$selected_fstype"
+                fi
+                case "$selected_fstype" in
                     btrfs)
                         cmd="btrfs send $btrfs_verbose_flag $snapper_source_snapshot 2>$BTRFS_PIPE \
                             | $cmd_pv \
@@ -1914,7 +1931,6 @@ run_backup () {
                         ;;
                 esac
 
-                # send full snapshot to target
                 if [ "$verbose" -ge 2 ]; then
                     if [ ${#transfer_size} -gt 0 ]; then
                         printf "${MAGENTA}Sending ${GREEN}snapshot${NO_COLOR} for snapper config ${GREEN}'%s' ${MAGENTA}(id='${GREEN}%s${MAGENTA}', size='${GREEN}%s${MAGENTA}')${NO_COLOR} ...\n" \
@@ -1924,13 +1940,17 @@ run_backup () {
                                "$selected_config"
                     fi
                 fi
+
                 # the actual data sync to the target
                 # this may take a while, depending on datasize and line-speed
                 if [ "$verbose" -ge 3 ]; then
                     printf "cmd: '%s'\n" "$cmd"
                 fi
-                $(eval "$cmd")
-                if [ "$?" -gt 0 ]; then
+                ret=$(eval "$cmd")
+                if [ "$verbose" -ge 3 ]; then
+                    printf "cmd returned: '%s' \n" "$ret"
+                fi
+                if [ "$ret" -gt 0 ]; then
                     printf "${RED}BTRFS_PIPE: %s${NO_COLOR}\n" "$(cat <"$BTRFS_PIPE")"
                     error_count=$((error_count+1))
                     # go for next configuration
@@ -1938,7 +1958,7 @@ run_backup () {
                     continue
                 fi
             else
-                # source holds synced snapshots
+                # send incremental snapshot to target, since source holds synced snapshots
                 if [ "$verbose" -ge 3 ]; then
                     printf "New ${GREEN}source${NO_COLOR} snapshot id: ${GREEN}'%s'${NO_COLOR} (path: ${GREEN}'%s'${NO_COLOR})\n" \
                            "$snapper_source_id" "$snapper_source_snapshot"
@@ -1957,10 +1977,10 @@ run_backup () {
                 fi
 
                 # verify that we have a matching source and target snapshot-id
-                if [ "$snapper_common_sync_id" -eq 0 ]; then
+                #if [ "$snapper_common_sync_id" -eq 0 ]; then
                     if [ "$snapper_source_id" -eq "$snapper_target_sync_id" ]; then
                         # nothing to do, snapshot already in sync
-                        if [ "$verbose" -ge 3 ]; then
+                        if [ "$verbose" -ge 2 ]; then
                             printf "${MAGENTA}Nothing to do! Source and target snapshot (id: ${GREEN}'%s'${MAGENTA}) are in sync.${NO_COLOR}\n" \
                                    "$snapper_target_sync_id"
                         fi
@@ -1973,7 +1993,7 @@ run_backup () {
                             get_snapper_sync_id "snapper_config=${snapper_source_config}" "remote="
                             if [ $? -eq 0 ]; then
                                 snapper_source_sync_id="$snapper_sync_id"
-                                snapper_source_sync_snapshot="$SUBVOLUME"/.snapshots/"$snapper_sync_id"/"$snapper_snapshot_name"
+                                snapper_source_sync_snapshot="$SUBVOLUME/.snapshots/$snapper_sync_id/$snapper_snapshot_name"
                             else
                                 printf "${RED}Error: ${MAGENTA}No common sync id found. Aborting backup for config ${GREEN}'%s'${NO_COLOR}\n"
                                 error_count=$((error_count+1))
@@ -1990,18 +2010,20 @@ run_backup () {
                         snapper_common_sync_id="$snapper_source_id"
                         snapper_common_sync_snapshot="$snapper_source_sync_snapshot"
                     fi
-                else
-                    # we have a common sync_id
-                    if [ "$snapper_source_sync_id" != "$snapper_target_sync_id" ]; then
-                        # btrfs send: use common sync_id as a valid parent
-                        snapper_source_sync_id="$snapper_common_sync_id"
-                        snapper_source_sync_snapshot="$SUBVOLUME"/.snapshots/"$snapper_source_sync_id"/"$snapper_snapshot_name"
-                    fi
-                fi
+                #else
+                #    # we have a common sync_id
+                #    if [ "$snapper_source_sync_id" != "$snapper_target_sync_id" ]; then
+                #        # btrfs send: use common sync_id as a valid parent
+                #        snapper_source_sync_id="$snapper_common_sync_id"
+                #        snapper_source_sync_snapshot="$SUBVOLUME/.snapshots/$snapper_source_sync_id/$snapper_snapshot_name"
+                #    fi
+                #fi
 
+                # return value for 'cmd'
+		# 256 -> valid btrfs snapshot
+		# ''  -> snapshot does not exits
 	       	cmd="stat --format %i $snapper_source_snapshot 2>/dev/null"
-                ret=$(eval "$cmd")
-                if [ $? -eq 0 ]; then
+		if [ "$(eval "$cmd")" -eq 256 ]; then
                     # get size of stream that needs to be transfered
                     if [ "$calculate_btrfs_size" -eq 1 ]; then
 			    check_transfer_size "source_snapshot=$snapper_source_snapshot" "clone_snapshot=$snapper_common_sync_snapshot"
@@ -2045,13 +2067,12 @@ run_backup () {
 
                     # TODO: handle error if snapshot already exists
                     # cmd="$ssh stat --format %i $snapper_target_snapshot 2>/dev/null"
-                    # ret=$(eval "$cmd")
-                    # if [ $? -eq 1 ]; then
+		    # if [ "$(eval "$cmd")" -eq 256 ]; then
                     #   printf "${RED}Error: ${MAGENTA}target snapshot ${GREEN}'%i'${MAGENTA} already exists.${NO_COLOR}\n"
                     #   printf "${GREEN}btrfs command:${NO_COLOR} '%s'\n" "$cmd"
                     # fi
 
-                    $(eval "$cmd")
+                    eval "$cmd"
                     ret=$?
                     case "$ret" in
                         0)
@@ -2112,7 +2133,7 @@ run_backup () {
                     printf "${MAGENTA}cmd: ${GREEN}'%s'${NO_COLOR}\n" \
                            "$cmd"
                 fi
-                $(eval "$cmd") 1>/dev/null
+                eval "$cmd" 1>/dev/null
             fi
         else
             printf "${MAGENTA}dryrun${NO_COLOR}: Would copy info metadata '%s' to target.\n"  \
@@ -2239,7 +2260,7 @@ run_finalize () {
             printf "${MAGENTA}Kill running ${GREEN}snapperd${MAGENTA} on target id: ${GREEN}'%s'${NO_COLOR} ...\n" \
                    "$snapperd_pid"
         fi
-        $(eval "$ssh" killall -SIGTERM snapperd 2>/dev/null)
+        eval "$ssh" killall -SIGTERM snapperd 2>/dev/null
 
         if [ "$verbose" -ge 3 ]; then
             printf "${MAGENTA}Identify snapper id ${GREEN}'%s'${MAGENTA} on target for configuration ${GREEN}'%s'${NO_COLOR} ...\n" \
@@ -2264,7 +2285,7 @@ run_finalize () {
                     fi
                     ret=$(eval "$cmd")
                     if [ "$verbose" -ge 3 ]; then
-                        printf "return: '%s'\n" "$?"
+                        printf "return: '%s'\n" "$ret"
                     fi
                     sync
 
@@ -2882,7 +2903,7 @@ Options:
      --label-running <desc>  snapper description tagging active jobs. Default: "dsnap-sync in progress"
      --label-synced <desc>   snapper description tagging last synced jobs
                              Default: "dsnap-sync last incremental"
-     --calculate-btrfs_size  Enable calculation of sync-size for given btrfs snapshots
+     --calculate-btrfs-size  Enable calculation of sync-size for given btrfs snapshots
      --color                 Enable colored output messages
  -c, --config <config>       Specify <multiple> snapper configurations.
                              Default: Perform snapshots for each available snapper configuration.
@@ -3262,7 +3283,8 @@ verify_snapper_structure () {
                            "$backup_root/$snapper_snapshots"
                 fi
                 cmd="$ssh btrfs subvolume create $backup_root/$snapper_snapshots 2>/dev/null"
-                if ! $(eval "$cmd"); then
+                ret=$(eval "$cmd")
+                if [ -z "$ret" ]; then
                     printf "${RED}Error: ${MAGENTA}Creation of snapper subvolume ${GREEN}%s${MAGENTA} failed${NO_COLOR}\n" \
                            "$backup_root/$snapper_snapshots"
                     return 1

lang/german/README.md

@@ -1,5 +1,5 @@
 <!-- dsnap-sync README.md -->
-<!-- version: 0.6.6 -->
+<!-- version: 0.6.9 -->
 
 # dsnap-sync
 
@@ -190,6 +190,7 @@ Software Paket Manager.
 		  --label-running <desc>  snapper description tagging active jobs. Default: "dsnap-sync in progress"
 		  --label-synced <desc>   snapper description tagging last synced jobs.
 								  Default: "dsnap-sync last incremental"
+          --calculate-btrfs-size  Enable calculation of sync-size for given snapshots
 		  --color                 Enable colored output messages
 	  -c, --config <config>       Specify the snapper configuration to use. Otherwise will perform for each snapper
 								  configuration. You can select multiple configurations
@@ -423,4 +424,4 @@ Diese Arbeit ist unter der [Creative Common License 4.0][License-CC_BY] lizensie
 ![Creative Common Logo][Logo-CC_BY]
 
 © 2016, 2017  James W. Barnett;
-© 2017 - 2021 Ralf Zerres
+© 2017 - 2023 Ralf Zerres

usr/share/doc/dsnap-sync/Examples.md

@@ -1,5 +1,25 @@
 # Examples
 
+## Snapper configs cache
+
+To integrate already existing snapshop subfolders located on a given
+filesystem, you need to add the snapper config name to the config
+file. Then restart the snapper service to reread the config list.
+
+```
+[root@dwsbackup dsnap-sync]# cat /etc/conf.d/snapper
+## Path: System/Snapper
+
+## Type:        string
+## Default:     ""
+# List of snapper configurations.
+
+#SNAPPER_CONFIGS="root data.dwssrv1 home.dwssrv1 machines.dwssrv1 root.dwssrv1 libvirt.dwssrv1 data2.dwssrv1 lxd.dwssrv1 archiv2.dwssrv1 archiv3.dwssrv1 root.dwssrv2"
+SNAPPER_CONFIGS="root data.dwssrv1"
+
+[root@dwsbackup dsnap-snyc]# systemctl restart snapperd
+```
+
 ## Automounter
 
 `dsnap-sync` will take advantage of systemd automount units to incorporate external,