From 1a9acbb9e5c8022dd48167eb76ea5e5ac2629249 Mon Sep 17 00:00:00 2001 From: m8in Date: Sat, 25 Apr 2026 17:16:12 +0200 Subject: [PATCH] Rollback feature for sync --- script/host/zfs/composition-sync/sync-send.sh | 90 +++++++++---------- script/host/zfs/composition-sync/sync.sh | 55 ++++++++---- 2 files changed, 80 insertions(+), 65 deletions(-) diff --git a/script/host/zfs/composition-sync/sync-send.sh b/script/host/zfs/composition-sync/sync-send.sh index 8708d26..8787710 100755 --- a/script/host/zfs/composition-sync/sync-send.sh +++ b/script/host/zfs/composition-sync/sync-send.sh @@ -1,4 +1,7 @@ #!/bin/bash +source /cis/core/base.module.sh + + function printNewestOrdinarySnapshot() { local _ZFS _RECEIVERHOST @@ -84,21 +87,39 @@ function sendResume() { _RESUME_TOKEN="${1:?"sendResume(): Missing first parameter RESUME_TOKEN"}" readonly _RESUME_TOKEN - zfs send -t "${RESUME_TOKEN}" \ + zfs send -t "${_RESUME_TOKEN:?"Missing RESUME_TOKEN"}" \ && return 0 return 1 } -function isValid() { - # printf '%s' - # - always treats the contents of ${1} as pure plain text. - # grep -qE: checks RegExp, but quiet - printf '%s' "${1}" | grep -qE "${2:?"isValid(): Missing REGEXP"}" -} +function send() { + local _COMPOSITION _RECEIVERHOST _RECEIVERS_SNAPSHOT _NOW _ZFS _NEW_SNAPSHOT + _COMPOSITION="${1:?"send(): Missing first parameter COMPOSITION"}" + _RECEIVERHOST="${2:?"send(): Missing second parameter RECEIVERHOST"}" + _RECEIVERS_SNAPSHOT="${3}" + _NOW=$(date -u "+%Y-%m-%d_%H:%M:%SZ") + _ZFS="zpool1/persistent/${_COMPOSITION:?"Missing COMPOSITION"}" + _NEW_SNAPSHOT="${_ZFS:?"Missing ZFS"}@SYNC_${_RECEIVERHOST:?"Missing RECEIVERHOST"}_${_NOW:?"Missing NOW"}" + readonly _COMPOSITION _RECEIVERHOST _RECEIVERS_SNAPSHOT _NOW _ZFS _NEW_SNAPSHOT -function isValidOptional() { - [ -z "${1}" ] || isValid "${1}" "${2}" + # This common snapshot is the starting-point, if available. + ! _COMMON_SNAPSHOT=$(printFoundCommonSnapshot "${_ZFS}" "${_RECEIVERHOST}" "${_RECEIVERS_SNAPSHOT}") \ + && echo "Failure in sync-send.sh: abort" >&2 \ + && return 1 + + [ "${_COMMON_SNAPSHOT}" == "" ] \ + && zfs snapshot "${_NEW_SNAPSHOT}" \ + && zfs send -c -R "${_NEW_SNAPSHOT}" \ + && return 0 + + [ "${_COMMON_SNAPSHOT}" != "" ] \ + && removeReceiverhostsSyncSnapshotsExeptTheCommonOne "${_ZFS}" "${_RECEIVERHOST}" "${_COMMON_SNAPSHOT}" \ + && zfs snapshot "${_NEW_SNAPSHOT}" \ + && zfs send -c -R -I "${_COMMON_SNAPSHOT}" "${_NEW_SNAPSHOT}" \ + && return 0 + + return 1 } @@ -107,48 +128,21 @@ function isValidOptional() { # Parameter 2: Only alphanumeric characters allowed and [.-] if not leading (due to: -oProxyCommand=...). # Parameter 3: Only alphanumeric characters allowed and [._:-] if not leading (due to: -oProxyCommand=...), but can be empty. # Parameter 4: Only alphanumeric characters allowed and [._:-] if not leading (due to: -oProxyCommand=...), but can be empty. -if isValid "${1:?"RECEIVERHOST missing"}" '^[a-zA-Z0-9][a-zA-Z0-9._-]*$' \ - && isValid "${2:?"COMPOSITION missing"}" '^[a-zA-Z0-9][a-zA-Z0-9.-]*$' \ - && isValidOptional "${3}" '^[a-zA-Z0-9][a-zA-Z0-9._:-]*$' \ - && isValidOptional "${4}" '^[a-zA-Z0-9][a-zA-Z0-9._:-]*$' -then - _RECEIVERHOST="${1}" - _COMPOSITION="${2}" - _RECEIVERS_SNAPSHOT="${3}" - _RESUME_TOKEN="${4}" +base.set RECEIVERHOST "${1}" '^[a-zA-Z0-9][a-zA-Z0-9._-]*$' || exit 1 +base.set COMPOSITION "${2}" '^[a-zA-Z0-9][a-zA-Z0-9.-]*$' || exit 1 +base.set RECEIVERS_SNAPSHOT "${3}" '(^[a-zA-Z0-9][a-zA-Z0-9._:-]*$)?' || exit 1 +base.set RESUME_TOKEN "${4}" '(^[a-zA-Z0-9][a-zA-Z0-9._:-]*$)?' || exit 1 - _NOW=$(date -u "+%Y-%m-%d_%H:%M:%S") - _ZFS="zpool1/persistent/${_COMPOSITION:?"COMPOSITION missing"}" - _NEW_SNAPSHOT="${_ZFS:?"ZFS missing"}@SYNC_${_RECEIVERHOST:?"RECEIVERHOST missing"}_${_NOW:?"NOW missing"}" +# Resume mode +if [ "${RECEIVERS_SNAPSHOT}" == "RESUME" ]; then + sendResume "${RESUME_TOKEN}" - # Resume mode - if [ "${_RECEIVERS_SNAPSHOT}" == "RESUME" ]; then - sendResume "${_RESUME_TOKEN}" - - # Exit preserving the code - exit $? - fi - - # This common snapshot is the starting-point, if available. - ! _COMMON_SNAPSHOT=$(printFoundCommonSnapshot "${_ZFS}" "${_RECEIVERHOST}" "${_RECEIVERS_SNAPSHOT}") \ - && echo "Failure in sync-send.sh: abort" >&2 \ - && exit 1 - - [ "${_COMMON_SNAPSHOT}" == "" ] \ - && zfs snapshot "${_NEW_SNAPSHOT}" \ - && zfs send -c -R "${_NEW_SNAPSHOT}" \ - && exit 0 - - [ "${_COMMON_SNAPSHOT}" != "" ] \ - && removeReceiverhostsSyncSnapshotsExeptTheCommonOne "${_ZFS}" "${_RECEIVERHOST}" "${_COMMON_SNAPSHOT}" \ - && zfs snapshot "${_NEW_SNAPSHOT}" \ - && zfs send -c -R -I "${_COMMON_SNAPSHOT}" "${_NEW_SNAPSHOT}" \ - && exit 0 - -else - echo "Failure in sync-send.sh: At least one parameter is invalid." >&2 - exit 1 + # Exit preserving the code + exit $? fi +send "${COMPOSITION}" "${RECEIVERHOST}" "${RECEIVERS_SNAPSHOT}" \ + && exit 0 + echo "Failure in sync-send.sh: Something unexpected happend." >&2 exit 1 diff --git a/script/host/zfs/composition-sync/sync.sh b/script/host/zfs/composition-sync/sync.sh index afae362..87d9444 100755 --- a/script/host/zfs/composition-sync/sync.sh +++ b/script/host/zfs/composition-sync/sync.sh @@ -1,17 +1,11 @@ #!/bin/bash -_MODE=$(echo "${1:?"MODE missing [--all, --once, --loop]"}" | sed -E 's|[^a-zA-Z0-9_-]*||g') -_COMPOSITION=$(echo "${2}" | sed -E 's|[^a-zA-Z0-9_-]*||g') -_SSH_PORT=$(echo "${3:-22}" | sed -E 's/[^0-9]//g') - # Folders always ends with an tailing '/' _SCRIPT="$(readlink -f "${0}" 2> /dev/null)" _CIS_ROOT="${_SCRIPT%/script/host/zfs/composition-sync/sync.sh}/" #Removes shortest matching pattern '/script/host/zfs/composition-sync/sync.sh' from the end -_SEND_SCRIPT="${_CIS_ROOT:?"Missing CIS_ROOT"}script/host/zfs/composition-sync/sync-send.sh" _DOMAIN="$("${_CIS_ROOT:?"Missing CIS_ROOT"}core/printOwnDomain.sh")" _DEFINITIONS="${_CIS_ROOT:?"Missing CIS_ROOT"}definitions/${_DOMAIN:?"Missing DOMAIN"}/" -_RECEIVERHOST=$(hostname -b) function stopObsoleteScreenSession() { local _RECEIVERHOST _SYNCHOSTS_FILE _SCREEN_SESSION _COMPOSITION _PID @@ -126,10 +120,11 @@ function removeOutdatedSyncSnapshots() { } function receive() { - local _RECEIVERHOST _COMPOSITION + local _RECEIVERHOST _COMPOSITION _SEND_SCRIPT _RECEIVERHOST="${1:?"receive(): Missing first parameter RECEIVERHOST"}" _COMPOSITION="${2:?"receive(): Missing second parameter COMPOSITION"}" - readonly _RECEIVERHOST _COMPOSITION + _SEND_SCRIPT="${_CIS_ROOT:?"Missing CIS_ROOT"}script/host/zfs/composition-sync/sync-send.sh" + readonly _RECEIVERHOST _COMPOSITION _SEND_SCRIPT ( flock -n 9 || exit 1 @@ -155,7 +150,8 @@ function receive() { # Add "-s" for resumable streams in the next line at zfs receive. Not done yet because of: cannot receive resume stream: kernel modules must be upgraded to receive this stream. ${_SSH_COMMAND} "sudo ${_SEND_SCRIPT:?"Missing SEND_SCRIPT"} \"${_RECEIVERHOST}\" \"${_COMPOSITION}\" \"${_COMMON_SNAPSHOT#${_ZFS}@}\" \"${_RESUME_TOKEN}\"" | zfs receive -v "${_ZFS}" if [ $? -ne 0 ]; then - echo "Unable to receive stream unsing these settings:" + tryRollbackToRepair "${_RECEIVERHOST}" "${_ZFS}" && return 0 + echo "Unable to receive stream using these settings:" echo " - Sending host: ${_SOURCEHOST}:${_SSH_PORT}" echo " - Receiving host: ${_RECEIVERHOST}" echo " - Composition: ${_COMPOSITION}" @@ -164,7 +160,7 @@ function receive() { echo "Current state of snapshots:" zfs list -t snapshot "${_ZFS}" 2> /dev/null | tail return 1 - fi + fi protectZFS "${_ZFS}" removeForeignSyncSnapshots "${_RECEIVERHOST}" "${_ZFS}" @@ -177,19 +173,44 @@ function receive() { return 1 } +function tryRollbackToRepair() { + local _RECEIVERHOST _ZFS _ROLLBACK_DAY _ROLLBACK_SNAPSHOT + _RECEIVERHOST="${1:?"tryRollbackToRepair(): Missing first parameter RECEIVERHOST"}" + _ZFS="${2:?"tryRollbackToRepair(): Missing second parameter ZFS"}" + _ROLLBACK_DAY=$(head -n 1 "${_DEFINITIONS:?"Missing DEFINITIONS"}/compositions/${_COMPOSITION:?"Missing COMPOSITION"}/rollback") + _ROLLBACK_SNAPSHOT=$(zfs list -t snapshot -H -o name -S creation "${_ZFS}" | head -n 1 | grep -F -- "@SYNC_${_RECEIVERHOST}_${_ROLLBACK_DAY}_") + readonly _RECEIVERHOST _ZFS _ROLLBACK_DAY _ROLLBACK_SNAPSHOT + + # Nothing to do + [ -z "${_ROLLBACK_SNAPSHOT}" ] && return 0 + + # Remove at most the two newest sync snapshots, if the day matches with the rollback file + echo "Try to fix by removing: '${_ROLLBACK_SNAPSHOT}'" \ + && zfs destroy "${_ROLLBACK_SNAPSHOT:?"tryRollbackToRepair(): Missing _ROLLBACK_SNAPSHOT"}" \ + && return 0 + + return 1 + +} -[ "${_MODE}" == "--all" ] \ - && cleanSessions "${_RECEIVERHOST}" composition-sync-hosts \ - && addSessions "${_RECEIVERHOST}" composition-sync-hosts \ + +MODE=$(echo "${1:?"MODE missing [--all, --once, --loop]"}" | sed -E 's|[^a-zA-Z0-9_-]*||g') +COMPOSITION=$(echo "${2}" | sed -E 's|[^a-zA-Z0-9_-]*||g') +RECEIVERHOST=$(hostname -b) +_SSH_PORT=$(echo "${3:-22}" | sed -E 's/[^0-9]//g') + +[ "${MODE}" == "--all" ] \ + && cleanSessions "${RECEIVERHOST}" composition-sync-hosts \ + && addSessions "${RECEIVERHOST}" composition-sync-hosts \ && exit 0 -[ "${_MODE}" == "--once" ] \ - && receive "${_RECEIVERHOST}" "${_COMPOSITION}" \ +[ "${MODE}" == "--once" ] \ + && receive "${RECEIVERHOST}" "${COMPOSITION}" \ && exit 0 -[ "${_MODE}" == "--loop" ] && while true; do - receive "${_RECEIVERHOST}" "${_COMPOSITION}" \ +[ "${MODE}" == "--loop" ] && while true; do + receive "${RECEIVERHOST}" "${COMPOSITION}" \ && echo "Sleep for 5s" \ && sleep 5 \ && echo \