Further improvments around compositions

definitions/default/hosts/all/etc/sudoers.d/allow-composition-sync-send

@@ -3,9 +3,8 @@
 # ( ... )?            -> defines the group as optional
 Cmnd_Alias C_COMPOSITION_SYNC = \
     /cis/script/host/zfs/composition-sync/sync-send.sh \
-        ^[a-zA-Z0-9][a-zA-Z0-9.-]* \
+        ^[a-zA-Z][a-zA-Z0-9_-]* \
-         [a-zA-Z][a-zA-Z0-9/_-]*[a-zA-Z0-9] \
+         [a-zA-Z][a-zA-Z0-9.-]* \
-         [a-zA-Z0-9][a-zA-Z0-9_-]* \
+        (@[a-zA-Z0-9._:-]*)? \
-        (@[a-zA-Z0-9][a-zA-Z0-9._:-]*)? \
         ([a-zA-Z0-9][a-zA-Z0-9._:-]*)?$
 composition-sync ALL = (root) NOPASSWD: C_COMPOSITION_SYNC

module/composition.module.sh

@@ -77,17 +77,15 @@ function composition.printAllSyncingHosts() {
     return 1
 }
 
-# composition.printZFS
+# composition.printZfsVerified
 #  - COMPOSITION mandatory: "uptime-kuma-prod"
 #
-# This function prints zhe matching ZFS of zje given composition which stores the data or is synced.
+# This function prints the matching ZFS of the given composition which stores the data or is synced.
-function composition.printZFS() {
+function composition.printZfsVerified() {
-    local _COMPOSITION _ZFS_BRANCH _ZFS _ZFS_VERIFIED
+    local _COMPOSITION _ZFS _ZFS_VERIFIED
-    _COMPOSITION="${1:?"composition.printZfsOfComposition(): Missing first parameter COMPOSITION"}"
+    _COMPOSITION="${1:?"composition.printZfsVerified(): Missing first parameter COMPOSITION"}"
-    _ZFS_BRANCH="$(cat "${CIS[COMPOSITIONS]}${_COMPOSITION}/zfs-branch" 2> /dev/null)"
+    _ZFS="$(composition.printZfs "${_COMPOSITION}")"
-    _ZFS_BRANCH="${_ZFS_BRANCH%/}"        #Remove tailing '/' if exists
+    readonly _COMPOSITION _ZFS
-    _ZFS="${_ZFS_BRANCH:-"zpool1/persistent"}/${_COMPOSITION}"
-    readonly _COMPOSITION _ZFS_BRANCH _ZFS
 
     if composition.shouldRunOnThisHost "${_COMPOSITION}"; then
         _ZFS_VERIFIED="$(zfs list -H -o name "${_ZFS}" 2> /dev/null)"
@@ -99,7 +97,45 @@ function composition.printZFS() {
         && echo "${_ZFS_VERIFIED}" \
         && return 0
 
-    print.failure "ZFS not found:" "${_ZFS}"
+    print.failure "ZFS not found:" "Given value was: ${_ZFS}"
+    return 1
+}
+
+# composition.printZfsBackup
+#  - COMPOSITION mandatory: "uptime-kuma-prod"
+#
+# This function prints the ZFS branch where the given composition is stored in.
+function composition.printZfsBackup() {
+    local _COMPOSITION
+    _COMPOSITION="${1:?"composition.printZfsBackup(): Missing first parameter COMPOSITION"}"
+    readonly _COMPOSITION
+
+    echo "$(composition.printZfs "${_COMPOSITION}")-BACKUP" \
+        && return 0
+
+    print.failure "No valid ZFS backup branch found."
+    return 1
+}
+
+# composition.printZfs
+#  - COMPOSITION mandatory: "uptime-kuma-prod"
+#
+# This function prints the ZFS branch where the given composition is stored in.
+function composition.printZfs() {
+    local _COMPOSITION _ZFS_FILE
+    _COMPOSITION="${1:?"composition.printZfs(): Missing first parameter COMPOSITION"}"
+    _ZFS_FILE="${CIS[COMPOSITIONS]}${_COMPOSITION}/zfs"
+    readonly _COMPOSITION _ZFS_FILE
+
+    ! [ -f "${_ZFS_FILE}" ] \
+        && echo "zpool1/persistent/${_COMPOSITION}" \
+        && return 0
+
+    base.set _RESULT "$(head -n 1 "${_ZFS_FILE}" 2> /dev/null)" "${REGEX[ZFS]}" \
+        && echo "${_RESULT}" \
+        && return 0
+
+    print.failure "No valid ZFS branch found in file: '${_ZFS_FILE}'"
     return 1
 }
 

script/host/zfs/composition-sync/setupSyncHost.sh

@@ -41,7 +41,7 @@ function setup() {
     echo "  - rollback (e.g.: date +%F > rollback)                         : allows the removal of the newest @SYNC snapshots of this day,"
     echo "                                                                       as long as no normal snapshot is reached."
     echo "  - ssh-port (e.g.: echo 22 > ssh-port)                          : allows to use a custom port for the SSH connection."
-    echo '  - zfs-branch (e.g.: echo zpool1/persistent > zfs-branch) : allows to use a custom zfs prefix like: ${zfs-branch}/${composition}.'
+    echo '  - zfs      (e.g.: echo zpool1/persistent/${composition} > zfs) : allows to use a custom zfs prefix like: ${zfs}.'
     return 0
 }
 

script/host/zfs/composition-sync/sync-send.sh

@@ -96,15 +96,14 @@ function sendResume() {
 }
 
 function send() {
-    local _ZFS_BRANCH _COMPOSITION _RECEIVERHOST _RECEIVERS_SNAPSHOT _NOW _ZFS _NEW_SNAPSHOT
+    local _COMPOSITION _RECEIVERHOST _RECEIVERS_SNAPSHOT _NOW _ZFS _NEW_SNAPSHOT
-    _ZFS_BRANCH="${1:?"send(): Missing first parameter ZFS_BRANCH"}"
+    _COMPOSITION="${1:?"send(): Missing first parameter COMPOSITION"}"
-    _COMPOSITION="${2:?"send(): Missing first parameter COMPOSITION"}"
+    _RECEIVERHOST="${2:?"send(): Missing second parameter RECEIVERHOST"}"
-    _RECEIVERHOST="${3:?"send(): Missing second parameter RECEIVERHOST"}"
+    _RECEIVERS_SNAPSHOT="${3}"
-    _RECEIVERS_SNAPSHOT="${4}"
     _NOW=$(date -u "+%Y-%m-%d_%H:%M:%SZ")
-    _ZFS="${_ZFS_BRANCH:?"Missing ZFS_BRANCH"}/${_COMPOSITION:?"Missing COMPOSITION"}"
+    _ZFS=$(composition.printZfsVerified "${_COMPOSITION}")
     _NEW_SNAPSHOT="${_ZFS:?"Missing ZFS"}@SYNC_${_RECEIVERHOST:?"Missing RECEIVERHOST"}_${_NOW:?"Missing NOW"}"
-    readonly _ZFS_BRANCH _COMPOSITION _RECEIVERHOST _RECEIVERS_SNAPSHOT _NOW _ZFS _NEW_SNAPSHOT
+    readonly _COMPOSITION _RECEIVERHOST _RECEIVERS_SNAPSHOT _NOW _NEW_SNAPSHOT
 
     # This common snapshot is the starting-point, if available.
     ! _COMMON_SNAPSHOT=$(printFoundCommonSnapshot "${_ZFS}" "${_RECEIVERHOST}" "${_RECEIVERS_SNAPSHOT}") \
@@ -127,30 +126,28 @@ function send() {
 
 
 
-# Parameter 1: Only alphanumeric characters allowed and [._-]  if not leading (due to: -oProxyCommand=...).
+# Parameter 1: Only alphanumeric characters allowed and [.-]   if not leading (due to: -oProxyCommand=...).
-# Parameter 2: Only alphanumeric characters allowed and [/_-]  if not leading (due to: -oProxyCommand=...).
+# Parameter 2: Only alphanumeric characters allowed and [._-]  if not leading (due to: -oProxyCommand=...).
-# Parameter 3: Only alphanumeric characters allowed and [.-]   if not leading (due to: -oProxyCommand=...).
+# Parameter 3: Only alphanumeric characters allowed and [._:-] if not leading (due to: -oProxyCommand=...), but can be empty.
 # Parameter 4: Only alphanumeric characters allowed and [._:-] if not leading (due to: -oProxyCommand=...), but can be empty.
-# Parameter 5: Only alphanumeric characters allowed and [._:-] if not leading (due to: -oProxyCommand=...), but can be empty.
+base.set COMPOSITION "${1}" "${REGEX[COMPOSITION]}"
-base.set RECEIVERHOST "${1}" "${REGEX[HOST]}"
+base.set RECEIVERHOST "${2}" "${REGEX[HOST]}"
-base.set ZFS_BRANCH "${2}" "${REGEX[ZFS]}"
+base.set RECEIVERS_SNAPSHOT "${3}" "${REGEX[SNAPSHOT]}" optional
-base.set COMPOSITION "${3}" "${REGEX[COMPOSITION]}"
+base.set RESUME_TOKEN "${4}" '^[a-zA-Z0-9][a-zA-Z0-9._:-]*$' optional
-base.set RECEIVERS_SNAPSHOT "${4}" "${REGEX[SNAPSHOT]}" optional
-base.set RESUME_TOKEN "${5}" '^[a-zA-Z0-9][a-zA-Z0-9._:-]*$' optional
 
 ! composition.shouldBeSyncedByGivenHost "${COMPOSITION}" "${RECEIVERHOST}" \
     && base.abort "no sync-host available" \
         "Host '${RECEIVERHOST}' is no sync-host for composition: '${COMPOSITION}'"
 
 # Resume mode
-if [ "${RECEIVERS_SNAPSHOT}" == "@RESUME" ] && [ -n "${RESUME_TOKEN}" ]; then
+if [ "${RECEIVERS_SNAPSHOT}" == '@RESUME' ] && [ -n "${RESUME_TOKEN}" ]; then
     sendResume "${RESUME_TOKEN}"
 
     # Exit preserving the code
     exit $?
 fi
 
-send "${ZFS_BRANCH}" "${COMPOSITION}" "${RECEIVERHOST}" "${RECEIVERS_SNAPSHOT}" \
+send "${COMPOSITION}" "${RECEIVERHOST}" "${RECEIVERS_SNAPSHOT}" \
     && exit 0
 
 print.failure "Something unexpected happend."

script/host/zfs/composition-sync/sync.sh

@@ -89,7 +89,7 @@ function tryRollbackToRepair() {
 }
 
 function receive() {
-    local _RECEIVERHOST _COMPOSITION _SOURCEHOST _SSH_PORT _SSH_COMMAND _SEND_SCRIPT _ZFS_BRANCH _ZFS
+    local _RECEIVERHOST _COMPOSITION _SOURCEHOST _SSH_PORT _SSH_COMMAND _SEND_SCRIPT _ZFS_BACKUP
     _COMPOSITION="${1:?"receive(): Missing first parameter COMPOSITION"}"
     _RECEIVERHOST="${CIS[HOST]:?"Missing CIS_HOST"}"
     base.set _SOURCEHOST "$(composition.printRunningHost "${_COMPOSITION}")" '^[a-zA-Z0-9][a-zA-Z0-9.-]*$'
@@ -97,47 +97,45 @@ function receive() {
     base.set _SSH_PORT "${_SSH_PORT:-22}" '^[1-9][0-9]{1,4}$'
     _SSH_COMMAND="ssh -p ${_SSH_PORT} -o ConnectTimeout=20 -o ServerAliveInterval=15 -C composition-sync@${_SOURCEHOST}"
     _SEND_SCRIPT="${CIS[SCRIPTSROOT]:?"Missing CIS_SCRIPTSROOT"}host/zfs/composition-sync/sync-send.sh"
-    _ZFS_BRANCH=$(head -n 1 "${CIS[DOMAINDEFINITIONS]:?"Missing CIS_DOMAINDEFINITIONS"}compositions/${_COMPOSITION}/zfs-branch" 2> /dev/null)
+    _ZFS_BACKUP="$(composition.printZfs "${_COMPOSITION}")-BACKUP"
-    base.set _ZFS_BRANCH "${_ZFS_BRANCH:-zpool1/persistent}" '^[a-zA-Z][a-zA-Z0-9/_-]*[a-zA-Z0-9]$'
+    readonly _RECEIVERHOST _COMPOSITION _SSH_COMMAND _SEND_SCRIPT _ZFS_BACKUP
-    _ZFS="${_ZFS_BRANCH%/}/${_COMPOSITION}-BACKUP"
-    readonly _RECEIVERHOST _COMPOSITION _SSH_COMMAND _SEND_SCRIPT _ZFS_BRANCH _ZFS
     (
         flock -n 9 || exit 1
 
         _COMMON_SNAPSHOT=""
-        _RESUME_TOKEN=$(zfs get -H -o value receive_resume_token "${_ZFS}" 2> /dev/null)
+        _RESUME_TOKEN=$(zfs get -H -o value receive_resume_token "${_ZFS_BACKUP}" 2> /dev/null)
         if [ -n "${_RESUME_TOKEN}" ] && [ "${_RESUME_TOKEN}" != "-" ]; then
             print.important "Resume token present trying to resume at ${_RESUME_TOKEN}\n"
             _COMMON_SNAPSHOT="@RESUME"
         else
             _RESUME_TOKEN=""
-            _COMMON_SNAPSHOT=$(zfs list -H -o name -S creation -t snapshot "${_ZFS}" 2> /dev/null | head -n 1)
+            _COMMON_SNAPSHOT=$(zfs list -H -o name -S creation -t snapshot "${_ZFS_BACKUP}" 2> /dev/null | head -n 1)
-            ! [ -z "${_COMMON_SNAPSHOT}" ] \
+            [ -n "${_COMMON_SNAPSHOT}" ] \
                 && print.data "Rolling back to newest snapshot: ${_COMMON_SNAPSHOT} ... " \
                 && zfs rollback -r "${_COMMON_SNAPSHOT}" \
+                && _COMMON_SNAPSHOT="@${_COMMON_SNAPSHOT#*@}" \
                 && print.good "done\n"
         fi
 
         # Add "-s" for resumable streams in the next line at zfs receive. Not done yet because of: cannot receive resume stream: kernel modules must be upgraded to receive this stream.
-        ${_SSH_COMMAND} "sudo ${_SEND_SCRIPT:?"Missing SEND_SCRIPT"} \"${_RECEIVERHOST}\" \"${_ZFS_BRANCH}\" \"${_COMPOSITION}\" \"@${_COMMON_SNAPSHOT#*@}\" \"${_RESUME_TOKEN}\"" | zfs receive -v "${_ZFS}"
+        ${_SSH_COMMAND} "sudo ${_SEND_SCRIPT:?"Missing SEND_SCRIPT"} \"${_COMPOSITION}\" \"${_RECEIVERHOST}\" \"${_COMMON_SNAPSHOT}\" \"${_RESUME_TOKEN}\"" | zfs receive -v "${_ZFS_BACKUP}"
         if [ $? -ne 0 ]; then
             print.failure "Unable to receive stream" \
             "These settings were used" \
             "- Sending host:     ${_SOURCEHOST}:${_SSH_PORT}" \
-            "- Receiving host:   ${_RECEIVERHOST}" \
-            "- ZFS Branch:       ${_ZFS_BRANCH}" \
             "- Composition:      ${_COMPOSITION}" \
-            "- Offered snapshot: @${_COMMON_SNAPSHOT#*@}" \
+            "- Receiving host:   ${_RECEIVERHOST}" \
+            "- Offered snapshot: ${_COMMON_SNAPSHOT}" \
             "- Resume token:     ${_RESUME_TOKEN}"
             print.highlight "Current state of snapshots:\n"
-            zfs list -t snapshot "${_ZFS}" 2> /dev/null | tail
+            zfs list -t snapshot "${_ZFS_BACKUP}" 2> /dev/null | tail
-            tryRollbackToRepair "${_COMPOSITION}" "${_ZFS}" && return 0
+            tryRollbackToRepair "${_COMPOSITION}" "${_ZFS_BACKUP}" && return 0
             return 1
         fi
 
-        protectZFS "${_ZFS}"
+        protectZFS "${_ZFS_BACKUP}"
-        removeForeignSyncSnapshots "${_ZFS}"
+        removeForeignSyncSnapshots "${_ZFS_BACKUP}"
-        removeOutdatedSyncSnapshots "${_ZFS}"
+        removeOutdatedSyncSnapshots "${_ZFS_BACKUP}"
 
     ) 9>>/tmp/synccomposition.${_COMPOSITION}.lock
 

script/host/zfs/snapshot.sh

@@ -16,7 +16,7 @@ function cleanup() {
     local _COMPOSITION _ZFS
     composition.printAll | while read -r _COMPOSITION; do
 
-        _ZFS="$(composition.printZFS "${_COMPOSITION}")"
+        _ZFS="$(composition.printZfsVerified "${_COMPOSITION}")"
         [ -n "${_ZFS}" ] \
             && printf -- "Cleaning snapshots of: '%b'\n" "${_ZFS}" \
             && local _LIST=( $(zfs list -t snap -H -o name -S creation "${_ZFS}" | grep -F '@SNAP' ) )
@@ -87,7 +87,7 @@ function snapshot() {
         _MODE="${1:-"$(cat "${CIS[COMPOSITIONS]}${_COMPOSITION}/snapshot-mode" 2> /dev/null)"}"
         _MODE="${_MODE:-"HOURLY"}"
 
-        _ZFS="$(composition.printZFS "${_COMPOSITION}")"
+        _ZFS="$(composition.printZfsVerified "${_COMPOSITION}")"
         if [ -n "${_ZFS}" ]; then
             (
                 flock -n 9 || return 1