From cb0a32ac0e80ce26e46ed38b2bca90f4a2b2bc41 Mon Sep 17 00:00:00 2001 From: m8in Date: Sat, 27 Jun 2026 01:36:14 +0200 Subject: [PATCH] Further improvments around compositions --- .../etc/sudoers.d/allow-composition-sync-send | 7 +-- module/composition.module.sh | 56 +++++++++++++++---- .../zfs/composition-sync/setupSyncHost.sh | 8 +-- script/host/zfs/composition-sync/sync-send.sh | 33 +++++------ script/host/zfs/composition-sync/sync.sh | 32 +++++------ script/host/zfs/snapshot.sh | 4 +- 6 files changed, 85 insertions(+), 55 deletions(-) diff --git a/definitions/default/hosts/all/etc/sudoers.d/allow-composition-sync-send b/definitions/default/hosts/all/etc/sudoers.d/allow-composition-sync-send index 3fd6f22..fbbb554 100644 --- a/definitions/default/hosts/all/etc/sudoers.d/allow-composition-sync-send +++ b/definitions/default/hosts/all/etc/sudoers.d/allow-composition-sync-send @@ -3,9 +3,8 @@ # ( ... )? -> defines the group as optional Cmnd_Alias C_COMPOSITION_SYNC = \ /cis/script/host/zfs/composition-sync/sync-send.sh \ - ^[a-zA-Z0-9][a-zA-Z0-9.-]* \ - [a-zA-Z][a-zA-Z0-9/_-]*[a-zA-Z0-9] \ - [a-zA-Z0-9][a-zA-Z0-9_-]* \ - (@[a-zA-Z0-9][a-zA-Z0-9._:-]*)? \ + ^[a-zA-Z][a-zA-Z0-9_-]* \ + [a-zA-Z][a-zA-Z0-9.-]* \ + (@[a-zA-Z0-9._:-]*)? \ ([a-zA-Z0-9][a-zA-Z0-9._:-]*)?$ composition-sync ALL = (root) NOPASSWD: C_COMPOSITION_SYNC diff --git a/module/composition.module.sh b/module/composition.module.sh index 69921ff..af15869 100755 --- a/module/composition.module.sh +++ b/module/composition.module.sh @@ -77,17 +77,15 @@ function composition.printAllSyncingHosts() { return 1 } -# composition.printZFS +# composition.printZfsVerified # - COMPOSITION mandatory: "uptime-kuma-prod" # -# This function prints zhe matching ZFS of zje given composition which stores the data or is synced. -function composition.printZFS() { - local _COMPOSITION _ZFS_BRANCH _ZFS _ZFS_VERIFIED - _COMPOSITION="${1:?"composition.printZfsOfComposition(): Missing first parameter COMPOSITION"}" - _ZFS_BRANCH="$(cat "${CIS[COMPOSITIONS]}${_COMPOSITION}/zfs-branch" 2> /dev/null)" - _ZFS_BRANCH="${_ZFS_BRANCH%/}" #Remove tailing '/' if exists - _ZFS="${_ZFS_BRANCH:-"zpool1/persistent"}/${_COMPOSITION}" - readonly _COMPOSITION _ZFS_BRANCH _ZFS +# This function prints the matching ZFS of the given composition which stores the data or is synced. +function composition.printZfsVerified() { + local _COMPOSITION _ZFS _ZFS_VERIFIED + _COMPOSITION="${1:?"composition.printZfsVerified(): Missing first parameter COMPOSITION"}" + _ZFS="$(composition.printZfs "${_COMPOSITION}")" + readonly _COMPOSITION _ZFS if composition.shouldRunOnThisHost "${_COMPOSITION}"; then _ZFS_VERIFIED="$(zfs list -H -o name "${_ZFS}" 2> /dev/null)" @@ -99,7 +97,45 @@ function composition.printZFS() { && echo "${_ZFS_VERIFIED}" \ && return 0 - print.failure "ZFS not found:" "${_ZFS}" + print.failure "ZFS not found:" "Given value was: ${_ZFS}" + return 1 +} + +# composition.printZfsBackup +# - COMPOSITION mandatory: "uptime-kuma-prod" +# +# This function prints the ZFS branch where the given composition is stored in. +function composition.printZfsBackup() { + local _COMPOSITION + _COMPOSITION="${1:?"composition.printZfsBackup(): Missing first parameter COMPOSITION"}" + readonly _COMPOSITION + + echo "$(composition.printZfs "${_COMPOSITION}")-BACKUP" \ + && return 0 + + print.failure "No valid ZFS backup branch found." + return 1 +} + +# composition.printZfs +# - COMPOSITION mandatory: "uptime-kuma-prod" +# +# This function prints the ZFS branch where the given composition is stored in. +function composition.printZfs() { + local _COMPOSITION _ZFS_FILE + _COMPOSITION="${1:?"composition.printZfs(): Missing first parameter COMPOSITION"}" + _ZFS_FILE="${CIS[COMPOSITIONS]}${_COMPOSITION}/zfs" + readonly _COMPOSITION _ZFS_FILE + + ! [ -f "${_ZFS_FILE}" ] \ + && echo "zpool1/persistent/${_COMPOSITION}" \ + && return 0 + + base.set _RESULT "$(head -n 1 "${_ZFS_FILE}" 2> /dev/null)" "${REGEX[ZFS]}" \ + && echo "${_RESULT}" \ + && return 0 + + print.failure "No valid ZFS branch found in file: '${_ZFS_FILE}'" return 1 } diff --git a/script/host/zfs/composition-sync/setupSyncHost.sh b/script/host/zfs/composition-sync/setupSyncHost.sh index 3257b92..39e14d7 100755 --- a/script/host/zfs/composition-sync/setupSyncHost.sh +++ b/script/host/zfs/composition-sync/setupSyncHost.sh @@ -38,10 +38,10 @@ function setup() { && echo echo "Optionally you can create following file:" - echo " - rollback (e.g.: date +%F > rollback) : allows the removal of the newest @SYNC snapshots of this day," - echo " as long as no normal snapshot is reached." - echo " - ssh-port (e.g.: echo 22 > ssh-port) : allows to use a custom port for the SSH connection." - echo ' - zfs-branch (e.g.: echo zpool1/persistent > zfs-branch) : allows to use a custom zfs prefix like: ${zfs-branch}/${composition}.' + echo " - rollback (e.g.: date +%F > rollback) : allows the removal of the newest @SYNC snapshots of this day," + echo " as long as no normal snapshot is reached." + echo " - ssh-port (e.g.: echo 22 > ssh-port) : allows to use a custom port for the SSH connection." + echo ' - zfs (e.g.: echo zpool1/persistent/${composition} > zfs) : allows to use a custom zfs prefix like: ${zfs}.' return 0 } diff --git a/script/host/zfs/composition-sync/sync-send.sh b/script/host/zfs/composition-sync/sync-send.sh index 81754e3..f9384f4 100755 --- a/script/host/zfs/composition-sync/sync-send.sh +++ b/script/host/zfs/composition-sync/sync-send.sh @@ -96,15 +96,14 @@ function sendResume() { } function send() { - local _ZFS_BRANCH _COMPOSITION _RECEIVERHOST _RECEIVERS_SNAPSHOT _NOW _ZFS _NEW_SNAPSHOT - _ZFS_BRANCH="${1:?"send(): Missing first parameter ZFS_BRANCH"}" - _COMPOSITION="${2:?"send(): Missing first parameter COMPOSITION"}" - _RECEIVERHOST="${3:?"send(): Missing second parameter RECEIVERHOST"}" - _RECEIVERS_SNAPSHOT="${4}" + local _COMPOSITION _RECEIVERHOST _RECEIVERS_SNAPSHOT _NOW _ZFS _NEW_SNAPSHOT + _COMPOSITION="${1:?"send(): Missing first parameter COMPOSITION"}" + _RECEIVERHOST="${2:?"send(): Missing second parameter RECEIVERHOST"}" + _RECEIVERS_SNAPSHOT="${3}" _NOW=$(date -u "+%Y-%m-%d_%H:%M:%SZ") - _ZFS="${_ZFS_BRANCH:?"Missing ZFS_BRANCH"}/${_COMPOSITION:?"Missing COMPOSITION"}" + _ZFS=$(composition.printZfsVerified "${_COMPOSITION}") _NEW_SNAPSHOT="${_ZFS:?"Missing ZFS"}@SYNC_${_RECEIVERHOST:?"Missing RECEIVERHOST"}_${_NOW:?"Missing NOW"}" - readonly _ZFS_BRANCH _COMPOSITION _RECEIVERHOST _RECEIVERS_SNAPSHOT _NOW _ZFS _NEW_SNAPSHOT + readonly _COMPOSITION _RECEIVERHOST _RECEIVERS_SNAPSHOT _NOW _NEW_SNAPSHOT # This common snapshot is the starting-point, if available. ! _COMMON_SNAPSHOT=$(printFoundCommonSnapshot "${_ZFS}" "${_RECEIVERHOST}" "${_RECEIVERS_SNAPSHOT}") \ @@ -127,30 +126,28 @@ function send() { -# Parameter 1: Only alphanumeric characters allowed and [._-] if not leading (due to: -oProxyCommand=...). -# Parameter 2: Only alphanumeric characters allowed and [/_-] if not leading (due to: -oProxyCommand=...). -# Parameter 3: Only alphanumeric characters allowed and [.-] if not leading (due to: -oProxyCommand=...). +# Parameter 1: Only alphanumeric characters allowed and [.-] if not leading (due to: -oProxyCommand=...). +# Parameter 2: Only alphanumeric characters allowed and [._-] if not leading (due to: -oProxyCommand=...). +# Parameter 3: Only alphanumeric characters allowed and [._:-] if not leading (due to: -oProxyCommand=...), but can be empty. # Parameter 4: Only alphanumeric characters allowed and [._:-] if not leading (due to: -oProxyCommand=...), but can be empty. -# Parameter 5: Only alphanumeric characters allowed and [._:-] if not leading (due to: -oProxyCommand=...), but can be empty. -base.set RECEIVERHOST "${1}" "${REGEX[HOST]}" -base.set ZFS_BRANCH "${2}" "${REGEX[ZFS]}" -base.set COMPOSITION "${3}" "${REGEX[COMPOSITION]}" -base.set RECEIVERS_SNAPSHOT "${4}" "${REGEX[SNAPSHOT]}" optional -base.set RESUME_TOKEN "${5}" '^[a-zA-Z0-9][a-zA-Z0-9._:-]*$' optional +base.set COMPOSITION "${1}" "${REGEX[COMPOSITION]}" +base.set RECEIVERHOST "${2}" "${REGEX[HOST]}" +base.set RECEIVERS_SNAPSHOT "${3}" "${REGEX[SNAPSHOT]}" optional +base.set RESUME_TOKEN "${4}" '^[a-zA-Z0-9][a-zA-Z0-9._:-]*$' optional ! composition.shouldBeSyncedByGivenHost "${COMPOSITION}" "${RECEIVERHOST}" \ && base.abort "no sync-host available" \ "Host '${RECEIVERHOST}' is no sync-host for composition: '${COMPOSITION}'" # Resume mode -if [ "${RECEIVERS_SNAPSHOT}" == "@RESUME" ] && [ -n "${RESUME_TOKEN}" ]; then +if [ "${RECEIVERS_SNAPSHOT}" == '@RESUME' ] && [ -n "${RESUME_TOKEN}" ]; then sendResume "${RESUME_TOKEN}" # Exit preserving the code exit $? fi -send "${ZFS_BRANCH}" "${COMPOSITION}" "${RECEIVERHOST}" "${RECEIVERS_SNAPSHOT}" \ +send "${COMPOSITION}" "${RECEIVERHOST}" "${RECEIVERS_SNAPSHOT}" \ && exit 0 print.failure "Something unexpected happend." diff --git a/script/host/zfs/composition-sync/sync.sh b/script/host/zfs/composition-sync/sync.sh index 5248723..a0cfe20 100755 --- a/script/host/zfs/composition-sync/sync.sh +++ b/script/host/zfs/composition-sync/sync.sh @@ -89,7 +89,7 @@ function tryRollbackToRepair() { } function receive() { - local _RECEIVERHOST _COMPOSITION _SOURCEHOST _SSH_PORT _SSH_COMMAND _SEND_SCRIPT _ZFS_BRANCH _ZFS + local _RECEIVERHOST _COMPOSITION _SOURCEHOST _SSH_PORT _SSH_COMMAND _SEND_SCRIPT _ZFS_BACKUP _COMPOSITION="${1:?"receive(): Missing first parameter COMPOSITION"}" _RECEIVERHOST="${CIS[HOST]:?"Missing CIS_HOST"}" base.set _SOURCEHOST "$(composition.printRunningHost "${_COMPOSITION}")" '^[a-zA-Z0-9][a-zA-Z0-9.-]*$' @@ -97,47 +97,45 @@ function receive() { base.set _SSH_PORT "${_SSH_PORT:-22}" '^[1-9][0-9]{1,4}$' _SSH_COMMAND="ssh -p ${_SSH_PORT} -o ConnectTimeout=20 -o ServerAliveInterval=15 -C composition-sync@${_SOURCEHOST}" _SEND_SCRIPT="${CIS[SCRIPTSROOT]:?"Missing CIS_SCRIPTSROOT"}host/zfs/composition-sync/sync-send.sh" - _ZFS_BRANCH=$(head -n 1 "${CIS[DOMAINDEFINITIONS]:?"Missing CIS_DOMAINDEFINITIONS"}compositions/${_COMPOSITION}/zfs-branch" 2> /dev/null) - base.set _ZFS_BRANCH "${_ZFS_BRANCH:-zpool1/persistent}" '^[a-zA-Z][a-zA-Z0-9/_-]*[a-zA-Z0-9]$' - _ZFS="${_ZFS_BRANCH%/}/${_COMPOSITION}-BACKUP" - readonly _RECEIVERHOST _COMPOSITION _SSH_COMMAND _SEND_SCRIPT _ZFS_BRANCH _ZFS + _ZFS_BACKUP="$(composition.printZfs "${_COMPOSITION}")-BACKUP" + readonly _RECEIVERHOST _COMPOSITION _SSH_COMMAND _SEND_SCRIPT _ZFS_BACKUP ( flock -n 9 || exit 1 _COMMON_SNAPSHOT="" - _RESUME_TOKEN=$(zfs get -H -o value receive_resume_token "${_ZFS}" 2> /dev/null) + _RESUME_TOKEN=$(zfs get -H -o value receive_resume_token "${_ZFS_BACKUP}" 2> /dev/null) if [ -n "${_RESUME_TOKEN}" ] && [ "${_RESUME_TOKEN}" != "-" ]; then print.important "Resume token present trying to resume at ${_RESUME_TOKEN}\n" _COMMON_SNAPSHOT="@RESUME" else _RESUME_TOKEN="" - _COMMON_SNAPSHOT=$(zfs list -H -o name -S creation -t snapshot "${_ZFS}" 2> /dev/null | head -n 1) - ! [ -z "${_COMMON_SNAPSHOT}" ] \ + _COMMON_SNAPSHOT=$(zfs list -H -o name -S creation -t snapshot "${_ZFS_BACKUP}" 2> /dev/null | head -n 1) + [ -n "${_COMMON_SNAPSHOT}" ] \ && print.data "Rolling back to newest snapshot: ${_COMMON_SNAPSHOT} ... " \ && zfs rollback -r "${_COMMON_SNAPSHOT}" \ + && _COMMON_SNAPSHOT="@${_COMMON_SNAPSHOT#*@}" \ && print.good "done\n" fi # Add "-s" for resumable streams in the next line at zfs receive. Not done yet because of: cannot receive resume stream: kernel modules must be upgraded to receive this stream. - ${_SSH_COMMAND} "sudo ${_SEND_SCRIPT:?"Missing SEND_SCRIPT"} \"${_RECEIVERHOST}\" \"${_ZFS_BRANCH}\" \"${_COMPOSITION}\" \"@${_COMMON_SNAPSHOT#*@}\" \"${_RESUME_TOKEN}\"" | zfs receive -v "${_ZFS}" + ${_SSH_COMMAND} "sudo ${_SEND_SCRIPT:?"Missing SEND_SCRIPT"} \"${_COMPOSITION}\" \"${_RECEIVERHOST}\" \"${_COMMON_SNAPSHOT}\" \"${_RESUME_TOKEN}\"" | zfs receive -v "${_ZFS_BACKUP}" if [ $? -ne 0 ]; then print.failure "Unable to receive stream" \ "These settings were used" \ "- Sending host: ${_SOURCEHOST}:${_SSH_PORT}" \ - "- Receiving host: ${_RECEIVERHOST}" \ - "- ZFS Branch: ${_ZFS_BRANCH}" \ "- Composition: ${_COMPOSITION}" \ - "- Offered snapshot: @${_COMMON_SNAPSHOT#*@}" \ + "- Receiving host: ${_RECEIVERHOST}" \ + "- Offered snapshot: ${_COMMON_SNAPSHOT}" \ "- Resume token: ${_RESUME_TOKEN}" print.highlight "Current state of snapshots:\n" - zfs list -t snapshot "${_ZFS}" 2> /dev/null | tail - tryRollbackToRepair "${_COMPOSITION}" "${_ZFS}" && return 0 + zfs list -t snapshot "${_ZFS_BACKUP}" 2> /dev/null | tail + tryRollbackToRepair "${_COMPOSITION}" "${_ZFS_BACKUP}" && return 0 return 1 fi - protectZFS "${_ZFS}" - removeForeignSyncSnapshots "${_ZFS}" - removeOutdatedSyncSnapshots "${_ZFS}" + protectZFS "${_ZFS_BACKUP}" + removeForeignSyncSnapshots "${_ZFS_BACKUP}" + removeOutdatedSyncSnapshots "${_ZFS_BACKUP}" ) 9>>/tmp/synccomposition.${_COMPOSITION}.lock diff --git a/script/host/zfs/snapshot.sh b/script/host/zfs/snapshot.sh index f8d99f2..2a05479 100755 --- a/script/host/zfs/snapshot.sh +++ b/script/host/zfs/snapshot.sh @@ -16,7 +16,7 @@ function cleanup() { local _COMPOSITION _ZFS composition.printAll | while read -r _COMPOSITION; do - _ZFS="$(composition.printZFS "${_COMPOSITION}")" + _ZFS="$(composition.printZfsVerified "${_COMPOSITION}")" [ -n "${_ZFS}" ] \ && printf -- "Cleaning snapshots of: '%b'\n" "${_ZFS}" \ && local _LIST=( $(zfs list -t snap -H -o name -S creation "${_ZFS}" | grep -F '@SNAP' ) ) @@ -87,7 +87,7 @@ function snapshot() { _MODE="${1:-"$(cat "${CIS[COMPOSITIONS]}${_COMPOSITION}/snapshot-mode" 2> /dev/null)"}" _MODE="${_MODE:-"HOURLY"}" - _ZFS="$(composition.printZFS "${_COMPOSITION}")" + _ZFS="$(composition.printZfsVerified "${_COMPOSITION}")" if [ -n "${_ZFS}" ]; then ( flock -n 9 || return 1