Rollback feature for sync

script/host/zfs/composition-sync/sync-send.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
+source /cis/core/base.module.sh
+
+
 
 function printNewestOrdinarySnapshot() {
     local _ZFS _RECEIVERHOST
@@ -84,21 +87,39 @@ function sendResume() {
     _RESUME_TOKEN="${1:?"sendResume(): Missing first parameter RESUME_TOKEN"}"
     readonly _RESUME_TOKEN
 
-    zfs send -t "${RESUME_TOKEN}" \
+    zfs send -t "${_RESUME_TOKEN:?"Missing RESUME_TOKEN"}" \
         && return 0
 
     return 1
 }
 
-function isValid() {
+function send() {
-    # printf '%s'
+    local _COMPOSITION _RECEIVERHOST _RECEIVERS_SNAPSHOT _NOW _ZFS _NEW_SNAPSHOT
-    #  - always treats the contents of ${1} as pure plain text.
+    _COMPOSITION="${1:?"send(): Missing first parameter COMPOSITION"}"
-    # grep -qE: checks RegExp, but quiet
+    _RECEIVERHOST="${2:?"send(): Missing second parameter RECEIVERHOST"}"
-    printf '%s' "${1}" | grep -qE "${2:?"isValid(): Missing REGEXP"}"
+    _RECEIVERS_SNAPSHOT="${3}"
-}
+    _NOW=$(date -u "+%Y-%m-%d_%H:%M:%SZ")
+    _ZFS="zpool1/persistent/${_COMPOSITION:?"Missing COMPOSITION"}"
+    _NEW_SNAPSHOT="${_ZFS:?"Missing ZFS"}@SYNC_${_RECEIVERHOST:?"Missing RECEIVERHOST"}_${_NOW:?"Missing NOW"}"
+    readonly _COMPOSITION _RECEIVERHOST _RECEIVERS_SNAPSHOT _NOW _ZFS _NEW_SNAPSHOT
 
-function isValidOptional() {
+    # This common snapshot is the starting-point, if available.
-    [ -z "${1}" ] || isValid "${1}" "${2}"
+    ! _COMMON_SNAPSHOT=$(printFoundCommonSnapshot "${_ZFS}" "${_RECEIVERHOST}" "${_RECEIVERS_SNAPSHOT}") \
+        && echo "Failure in sync-send.sh: abort" >&2 \
+        && return 1
+
+    [ "${_COMMON_SNAPSHOT}" == "" ] \
+        && zfs snapshot "${_NEW_SNAPSHOT}" \
+        && zfs send -c -R "${_NEW_SNAPSHOT}" \
+        && return 0
+
+    [ "${_COMMON_SNAPSHOT}" != "" ] \
+        && removeReceiverhostsSyncSnapshotsExeptTheCommonOne "${_ZFS}" "${_RECEIVERHOST}" "${_COMMON_SNAPSHOT}" \
+        && zfs snapshot "${_NEW_SNAPSHOT}" \
+        && zfs send -c -R -I "${_COMMON_SNAPSHOT}" "${_NEW_SNAPSHOT}" \
+        && return 0
+
+    return 1
 }
 
 
@@ -107,48 +128,21 @@ function isValidOptional() {
 # Parameter 2: Only alphanumeric characters allowed and [.-]   if not leading (due to: -oProxyCommand=...).
 # Parameter 3: Only alphanumeric characters allowed and [._:-] if not leading (due to: -oProxyCommand=...), but can be empty.
 # Parameter 4: Only alphanumeric characters allowed and [._:-] if not leading (due to: -oProxyCommand=...), but can be empty.
-if isValid "${1:?"RECEIVERHOST missing"}" '^[a-zA-Z0-9][a-zA-Z0-9._-]*$' \
+base.set RECEIVERHOST "${1}" '^[a-zA-Z0-9][a-zA-Z0-9._-]*$' || exit 1
-    && isValid "${2:?"COMPOSITION missing"}" '^[a-zA-Z0-9][a-zA-Z0-9.-]*$' \
+base.set COMPOSITION "${2}" '^[a-zA-Z0-9][a-zA-Z0-9.-]*$' || exit 1
-    && isValidOptional "${3}" '^[a-zA-Z0-9][a-zA-Z0-9._:-]*$' \
+base.set RECEIVERS_SNAPSHOT "${3}" '(^[a-zA-Z0-9][a-zA-Z0-9._:-]*$)?' || exit 1
-    && isValidOptional "${4}" '^[a-zA-Z0-9][a-zA-Z0-9._:-]*$'
+base.set RESUME_TOKEN "${4}" '(^[a-zA-Z0-9][a-zA-Z0-9._:-]*$)?' || exit 1
-then
-    _RECEIVERHOST="${1}"
-    _COMPOSITION="${2}"
-    _RECEIVERS_SNAPSHOT="${3}"
-    _RESUME_TOKEN="${4}"
 
-    _NOW=$(date -u "+%Y-%m-%d_%H:%M:%S")
+# Resume mode
-    _ZFS="zpool1/persistent/${_COMPOSITION:?"COMPOSITION missing"}"
+if [ "${RECEIVERS_SNAPSHOT}" == "RESUME" ]; then
-    _NEW_SNAPSHOT="${_ZFS:?"ZFS missing"}@SYNC_${_RECEIVERHOST:?"RECEIVERHOST missing"}_${_NOW:?"NOW missing"}"
+    sendResume "${RESUME_TOKEN}"
 
-    # Resume mode
+    # Exit preserving the code
-    if [ "${_RECEIVERS_SNAPSHOT}" == "RESUME" ]; then
+    exit $?
-        sendResume "${_RESUME_TOKEN}"
-
-        # Exit preserving the code 
-        exit $?
-    fi
-
-    # This common snapshot is the starting-point, if available.
-    ! _COMMON_SNAPSHOT=$(printFoundCommonSnapshot "${_ZFS}" "${_RECEIVERHOST}" "${_RECEIVERS_SNAPSHOT}") \
-        && echo "Failure in sync-send.sh: abort" >&2 \
-        && exit 1
-
-    [ "${_COMMON_SNAPSHOT}" == "" ] \
-        && zfs snapshot "${_NEW_SNAPSHOT}" \
-        && zfs send -c -R "${_NEW_SNAPSHOT}" \
-        && exit 0
-
-    [ "${_COMMON_SNAPSHOT}" != "" ] \
-        && removeReceiverhostsSyncSnapshotsExeptTheCommonOne "${_ZFS}" "${_RECEIVERHOST}" "${_COMMON_SNAPSHOT}" \
-        && zfs snapshot "${_NEW_SNAPSHOT}" \
-        && zfs send -c -R -I "${_COMMON_SNAPSHOT}" "${_NEW_SNAPSHOT}" \
-        && exit 0
-
-else
-    echo "Failure in sync-send.sh: At least one parameter is invalid." >&2
-    exit 1
 fi
 
+send "${COMPOSITION}" "${RECEIVERHOST}" "${RECEIVERS_SNAPSHOT}" \
+    && exit 0
+
 echo "Failure in sync-send.sh: Something unexpected happend." >&2
 exit 1

script/host/zfs/composition-sync/sync.sh

@@ -1,17 +1,11 @@
 #!/bin/bash
 
-_MODE=$(echo "${1:?"MODE missing [--all, --once, --loop]"}" | sed -E 's|[^a-zA-Z0-9_-]*||g')
-_COMPOSITION=$(echo "${2}" | sed -E 's|[^a-zA-Z0-9_-]*||g')
-_SSH_PORT=$(echo "${3:-22}" | sed -E 's/[^0-9]//g')
-
 # Folders always ends with an tailing '/'
 _SCRIPT="$(readlink -f "${0}" 2> /dev/null)"
 _CIS_ROOT="${_SCRIPT%/script/host/zfs/composition-sync/sync.sh}/"  #Removes shortest matching pattern '/script/host/zfs/composition-sync/sync.sh' from the end
-_SEND_SCRIPT="${_CIS_ROOT:?"Missing CIS_ROOT"}script/host/zfs/composition-sync/sync-send.sh"
 _DOMAIN="$("${_CIS_ROOT:?"Missing CIS_ROOT"}core/printOwnDomain.sh")"
 _DEFINITIONS="${_CIS_ROOT:?"Missing CIS_ROOT"}definitions/${_DOMAIN:?"Missing DOMAIN"}/"
 
-_RECEIVERHOST=$(hostname -b)
 
 function stopObsoleteScreenSession() {
     local _RECEIVERHOST _SYNCHOSTS_FILE _SCREEN_SESSION _COMPOSITION _PID
@@ -126,10 +120,11 @@ function removeOutdatedSyncSnapshots() {
 }
 
 function receive() {
-    local _RECEIVERHOST _COMPOSITION
+    local _RECEIVERHOST _COMPOSITION _SEND_SCRIPT
     _RECEIVERHOST="${1:?"receive(): Missing first parameter RECEIVERHOST"}"
     _COMPOSITION="${2:?"receive(): Missing second parameter COMPOSITION"}"
-    readonly _RECEIVERHOST _COMPOSITION
+    _SEND_SCRIPT="${_CIS_ROOT:?"Missing CIS_ROOT"}script/host/zfs/composition-sync/sync-send.sh"
+    readonly _RECEIVERHOST _COMPOSITION _SEND_SCRIPT
 
     (
         flock -n 9 || exit 1
@@ -155,7 +150,8 @@ function receive() {
         # Add "-s" for resumable streams in the next line at zfs receive. Not done yet because of: cannot receive resume stream: kernel modules must be upgraded to receive this stream.
         ${_SSH_COMMAND} "sudo ${_SEND_SCRIPT:?"Missing SEND_SCRIPT"} \"${_RECEIVERHOST}\" \"${_COMPOSITION}\" \"${_COMMON_SNAPSHOT#${_ZFS}@}\" \"${_RESUME_TOKEN}\"" | zfs receive -v "${_ZFS}"
         if [ $? -ne 0 ]; then
-            echo "Unable to receive stream unsing these settings:"
+            tryRollbackToRepair "${_RECEIVERHOST}" "${_ZFS}" && return 0
+            echo "Unable to receive stream using these settings:"
             echo "  - Sending host:     ${_SOURCEHOST}:${_SSH_PORT}"
             echo "  - Receiving host:   ${_RECEIVERHOST}"
             echo "  - Composition:      ${_COMPOSITION}"
@@ -177,19 +173,44 @@ function receive() {
     return 1
 }
 
+function tryRollbackToRepair() {
+    local _RECEIVERHOST _ZFS _ROLLBACK_DAY _ROLLBACK_SNAPSHOT
+    _RECEIVERHOST="${1:?"tryRollbackToRepair(): Missing first parameter RECEIVERHOST"}"
+    _ZFS="${2:?"tryRollbackToRepair(): Missing second parameter ZFS"}"
+    _ROLLBACK_DAY=$(head -n 1 "${_DEFINITIONS:?"Missing DEFINITIONS"}/compositions/${_COMPOSITION:?"Missing COMPOSITION"}/rollback")
+    _ROLLBACK_SNAPSHOT=$(zfs list -t snapshot -H -o name -S creation "${_ZFS}" | head -n 1 | grep -F -- "@SYNC_${_RECEIVERHOST}_${_ROLLBACK_DAY}_")
+    readonly _RECEIVERHOST _ZFS _ROLLBACK_DAY _ROLLBACK_SNAPSHOT
+
+    # Nothing to do
+    [ -z "${_ROLLBACK_SNAPSHOT}" ] && return 0
+
+    # Remove at most the two newest sync snapshots, if the day matches with the rollback file
+    echo "Try to fix by removing: '${_ROLLBACK_SNAPSHOT}'" \
+        && zfs destroy "${_ROLLBACK_SNAPSHOT:?"tryRollbackToRepair(): Missing _ROLLBACK_SNAPSHOT"}" \
+        && return 0
+
+    return 1
+
+}
 
 
-[ "${_MODE}" == "--all" ] \
+
-    && cleanSessions "${_RECEIVERHOST}" composition-sync-hosts \
+MODE=$(echo "${1:?"MODE missing [--all, --once, --loop]"}" | sed -E 's|[^a-zA-Z0-9_-]*||g')
-    && addSessions "${_RECEIVERHOST}" composition-sync-hosts \
+COMPOSITION=$(echo "${2}" | sed -E 's|[^a-zA-Z0-9_-]*||g')
+RECEIVERHOST=$(hostname -b)
+_SSH_PORT=$(echo "${3:-22}" | sed -E 's/[^0-9]//g')
+
+[ "${MODE}" == "--all" ] \
+    && cleanSessions "${RECEIVERHOST}" composition-sync-hosts \
+    && addSessions "${RECEIVERHOST}" composition-sync-hosts \
     && exit 0
 
-[ "${_MODE}" == "--once" ] \
+[ "${MODE}" == "--once" ] \
-    && receive "${_RECEIVERHOST}" "${_COMPOSITION}" \
+    && receive "${RECEIVERHOST}" "${COMPOSITION}" \
     && exit 0
 
-[ "${_MODE}" == "--loop" ] && while true; do
+[ "${MODE}" == "--loop" ] && while true; do
-    receive "${_RECEIVERHOST}" "${_COMPOSITION}" \
+    receive "${RECEIVERHOST}" "${COMPOSITION}" \
         && echo "Sleep for 5s" \
         && sleep 5 \
         && echo \