Everbrent/cis
8
0
Fork 0
mirror of https://github.com/m8tin/cis.git synced 2026-08-01 11:47:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

Further improvments around compositions

Browse Source
This commit is contained in:
m8in
2026-06-27 01:36:14 +02:00
parent 8a4426961c
commit cb0a32ac0e
6 changed files with 85 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
definitions/default/hosts/all/etc/sudoers.d/allow-composition-sync-send
+3 -4
View File
@@ -3,9 +3,8 @@
# ( ... )? -> defines the group as optional
Cmnd_Alias C_COMPOSITION_SYNC = \
/cis/script/host/zfs/composition-sync/sync-send.sh \
^[a-zA-Z0-9][a-zA-Z0-9.-]* \
[a-zA-Z][a-zA-Z0-9/_-]*[a-zA-Z0-9] \
[a-zA-Z0-9][a-zA-Z0-9_-]* \
(@[a-zA-Z0-9][a-zA-Z0-9._:-]*)? \
^[a-zA-Z][a-zA-Z0-9_-]* \
[a-zA-Z][a-zA-Z0-9.-]* \
(@[a-zA-Z0-9._:-]*)? \
([a-zA-Z0-9][a-zA-Z0-9._:-]*)?$
composition-sync ALL = (root) NOPASSWD: C_COMPOSITION_SYNC
module/composition.module.sh
+46 -10
View File
@@ -77,17 +77,15 @@ function composition.printAllSyncingHosts() {
return 1
}
# composition.printZFS
# composition.printZfsVerified
# - COMPOSITION mandatory: "uptime-kuma-prod"
#
# This function prints zhe matching ZFS of zje given composition which stores the data or is synced.
function composition.printZFS() {
local _COMPOSITION _ZFS_BRANCH _ZFS _ZFS_VERIFIED
_COMPOSITION="${1:?"composition.printZfsOfComposition(): Missing first parameter COMPOSITION"}"
_ZFS_BRANCH="$(cat "${CIS[COMPOSITIONS]}${_COMPOSITION}/zfs-branch" 2> /dev/null)"
_ZFS_BRANCH="${_ZFS_BRANCH%/}" #Remove tailing '/' if exists
_ZFS="${_ZFS_BRANCH:-"zpool1/persistent"}/${_COMPOSITION}"
readonly _COMPOSITION _ZFS_BRANCH _ZFS
# This function prints the matching ZFS of the given composition which stores the data or is synced.
function composition.printZfsVerified() {
local _COMPOSITION _ZFS _ZFS_VERIFIED
_COMPOSITION="${1:?"composition.printZfsVerified(): Missing first parameter COMPOSITION"}"
_ZFS="$(composition.printZfs "${_COMPOSITION}")"
readonly _COMPOSITION _ZFS
if composition.shouldRunOnThisHost "${_COMPOSITION}"; then
_ZFS_VERIFIED="$(zfs list -H -o name "${_ZFS}" 2> /dev/null)"
@@ -99,7 +97,45 @@ function composition.printZFS() {
&& echo "${_ZFS_VERIFIED}" \
&& return 0
print.failure "ZFS not found:" "${_ZFS}"
print.failure "ZFS not found:" "Given value was: ${_ZFS}"
return 1
}
# composition.printZfsBackup
# - COMPOSITION mandatory: "uptime-kuma-prod"
#
# This function prints the ZFS branch where the given composition is stored in.
function composition.printZfsBackup() {
local _COMPOSITION
_COMPOSITION="${1:?"composition.printZfsBackup(): Missing first parameter COMPOSITION"}"
readonly _COMPOSITION
echo "$(composition.printZfs "${_COMPOSITION}")-BACKUP" \
&& return 0
print.failure "No valid ZFS backup branch found."
return 1
}
# composition.printZfs
# - COMPOSITION mandatory: "uptime-kuma-prod"
#
# This function prints the ZFS branch where the given composition is stored in.
function composition.printZfs() {
local _COMPOSITION _ZFS_FILE
_COMPOSITION="${1:?"composition.printZfs(): Missing first parameter COMPOSITION"}"
_ZFS_FILE="${CIS[COMPOSITIONS]}${_COMPOSITION}/zfs"
readonly _COMPOSITION _ZFS_FILE
! [ -f "${_ZFS_FILE}" ] \
&& echo "zpool1/persistent/${_COMPOSITION}" \
&& return 0
base.set _RESULT "$(head -n 1 "${_ZFS_FILE}" 2> /dev/null)" "${REGEX[ZFS]}" \
&& echo "${_RESULT}" \
&& return 0
print.failure "No valid ZFS branch found in file: '${_ZFS_FILE}'"
return 1
}
script/host/zfs/composition-sync/setupSyncHost.sh
+4 -4
View File
@@ -38,10 +38,10 @@ function setup() {
&& echo
echo "Optionally you can create following file:"
echo " - rollback (e.g.: date +%F > rollback) : allows the removal of the newest @SYNC snapshots of this day,"
echo " as long as no normal snapshot is reached."
echo " - ssh-port (e.g.: echo 22 > ssh-port) : allows to use a custom port for the SSH connection."
echo ' - zfs-branch (e.g.: echo zpool1/persistent > zfs-branch) : allows to use a custom zfs prefix like: ${zfs-branch}/${composition}.'
echo " - rollback (e.g.: date +%F > rollback) : allows the removal of the newest @SYNC snapshots of this day,"
echo " as long as no normal snapshot is reached."
echo " - ssh-port (e.g.: echo 22 > ssh-port) : allows to use a custom port for the SSH connection."
echo ' - zfs (e.g.: echo zpool1/persistent/${composition} > zfs) : allows to use a custom zfs prefix like: ${zfs}.'
return 0
}
script/host/zfs/composition-sync/sync-send.sh
+15 -18
View File
@@ -96,15 +96,14 @@ function sendResume() {
}
function send() {
local _ZFS_BRANCH _COMPOSITION _RECEIVERHOST _RECEIVERS_SNAPSHOT _NOW _ZFS _NEW_SNAPSHOT
_ZFS_BRANCH="${1:?"send(): Missing first parameter ZFS_BRANCH"}"
_COMPOSITION="${2:?"send(): Missing first parameter COMPOSITION"}"
_RECEIVERHOST="${3:?"send(): Missing second parameter RECEIVERHOST"}"
_RECEIVERS_SNAPSHOT="${4}"
local _COMPOSITION _RECEIVERHOST _RECEIVERS_SNAPSHOT _NOW _ZFS _NEW_SNAPSHOT
_COMPOSITION="${1:?"send(): Missing first parameter COMPOSITION"}"
_RECEIVERHOST="${2:?"send(): Missing second parameter RECEIVERHOST"}"
_RECEIVERS_SNAPSHOT="${3}"
_NOW=$(date -u "+%Y-%m-%d_%H:%M:%SZ")
_ZFS="${_ZFS_BRANCH:?"Missing ZFS_BRANCH"}/${_COMPOSITION:?"Missing COMPOSITION"}"
_ZFS=$(composition.printZfsVerified "${_COMPOSITION}")
_NEW_SNAPSHOT="${_ZFS:?"Missing ZFS"}@SYNC_${_RECEIVERHOST:?"Missing RECEIVERHOST"}_${_NOW:?"Missing NOW"}"
readonly _ZFS_BRANCH _COMPOSITION _RECEIVERHOST _RECEIVERS_SNAPSHOT _NOW _ZFS _NEW_SNAPSHOT
readonly _COMPOSITION _RECEIVERHOST _RECEIVERS_SNAPSHOT _NOW _NEW_SNAPSHOT
# This common snapshot is the starting-point, if available.
! _COMMON_SNAPSHOT=$(printFoundCommonSnapshot "${_ZFS}" "${_RECEIVERHOST}" "${_RECEIVERS_SNAPSHOT}") \
@@ -127,30 +126,28 @@ function send() {
# Parameter 1: Only alphanumeric characters allowed and [._-] if not leading (due to: -oProxyCommand=...).
# Parameter 2: Only alphanumeric characters allowed and [/_-] if not leading (due to: -oProxyCommand=...).
# Parameter 3: Only alphanumeric characters allowed and [.-] if not leading (due to: -oProxyCommand=...).
# Parameter 1: Only alphanumeric characters allowed and [.-] if not leading (due to: -oProxyCommand=...).
# Parameter 2: Only alphanumeric characters allowed and [._-] if not leading (due to: -oProxyCommand=...).
# Parameter 3: Only alphanumeric characters allowed and [._:-] if not leading (due to: -oProxyCommand=...), but can be empty.
# Parameter 4: Only alphanumeric characters allowed and [._:-] if not leading (due to: -oProxyCommand=...), but can be empty.
# Parameter 5: Only alphanumeric characters allowed and [._:-] if not leading (due to: -oProxyCommand=...), but can be empty.
base.set RECEIVERHOST "${1}" "${REGEX[HOST]}"
base.set ZFS_BRANCH "${2}" "${REGEX[ZFS]}"
base.set COMPOSITION "${3}" "${REGEX[COMPOSITION]}"
base.set RECEIVERS_SNAPSHOT "${4}" "${REGEX[SNAPSHOT]}" optional
base.set RESUME_TOKEN "${5}" '^[a-zA-Z0-9][a-zA-Z0-9._:-]*$' optional
base.set COMPOSITION "${1}" "${REGEX[COMPOSITION]}"
base.set RECEIVERHOST "${2}" "${REGEX[HOST]}"
base.set RECEIVERS_SNAPSHOT "${3}" "${REGEX[SNAPSHOT]}" optional
base.set RESUME_TOKEN "${4}" '^[a-zA-Z0-9][a-zA-Z0-9._:-]*$' optional
! composition.shouldBeSyncedByGivenHost "${COMPOSITION}" "${RECEIVERHOST}" \
&& base.abort "no sync-host available" \
"Host '${RECEIVERHOST}' is no sync-host for composition: '${COMPOSITION}'"
# Resume mode
if [ "${RECEIVERS_SNAPSHOT}" == "@RESUME" ] && [ -n "${RESUME_TOKEN}" ]; then
if [ "${RECEIVERS_SNAPSHOT}" == '@RESUME' ] && [ -n "${RESUME_TOKEN}" ]; then
sendResume "${RESUME_TOKEN}"
# Exit preserving the code
exit $?
fi
send "${ZFS_BRANCH}" "${COMPOSITION}" "${RECEIVERHOST}" "${RECEIVERS_SNAPSHOT}" \
send "${COMPOSITION}" "${RECEIVERHOST}" "${RECEIVERS_SNAPSHOT}" \
&& exit 0
print.failure "Something unexpected happend."
script/host/zfs/composition-sync/sync.sh
+15 -17
View File
@@ -89,7 +89,7 @@ function tryRollbackToRepair() {
}
function receive() {
local _RECEIVERHOST _COMPOSITION _SOURCEHOST _SSH_PORT _SSH_COMMAND _SEND_SCRIPT _ZFS_BRANCH _ZFS
local _RECEIVERHOST _COMPOSITION _SOURCEHOST _SSH_PORT _SSH_COMMAND _SEND_SCRIPT _ZFS_BACKUP
_COMPOSITION="${1:?"receive(): Missing first parameter COMPOSITION"}"
_RECEIVERHOST="${CIS[HOST]:?"Missing CIS_HOST"}"
base.set _SOURCEHOST "$(composition.printRunningHost "${_COMPOSITION}")" '^[a-zA-Z0-9][a-zA-Z0-9.-]*$'
@@ -97,47 +97,45 @@ function receive() {
base.set _SSH_PORT "${_SSH_PORT:-22}" '^[1-9][0-9]{1,4}$'
_SSH_COMMAND="ssh -p ${_SSH_PORT} -o ConnectTimeout=20 -o ServerAliveInterval=15 -C composition-sync@${_SOURCEHOST}"
_SEND_SCRIPT="${CIS[SCRIPTSROOT]:?"Missing CIS_SCRIPTSROOT"}host/zfs/composition-sync/sync-send.sh"
_ZFS_BRANCH=$(head -n 1 "${CIS[DOMAINDEFINITIONS]:?"Missing CIS_DOMAINDEFINITIONS"}compositions/${_COMPOSITION}/zfs-branch" 2> /dev/null)
base.set _ZFS_BRANCH "${_ZFS_BRANCH:-zpool1/persistent}" '^[a-zA-Z][a-zA-Z0-9/_-]*[a-zA-Z0-9]$'
_ZFS="${_ZFS_BRANCH%/}/${_COMPOSITION}-BACKUP"
readonly _RECEIVERHOST _COMPOSITION _SSH_COMMAND _SEND_SCRIPT _ZFS_BRANCH _ZFS
_ZFS_BACKUP="$(composition.printZfs "${_COMPOSITION}")-BACKUP"
readonly _RECEIVERHOST _COMPOSITION _SSH_COMMAND _SEND_SCRIPT _ZFS_BACKUP
(
flock -n 9 || exit 1
_COMMON_SNAPSHOT=""
_RESUME_TOKEN=$(zfs get -H -o value receive_resume_token "${_ZFS}" 2> /dev/null)
_RESUME_TOKEN=$(zfs get -H -o value receive_resume_token "${_ZFS_BACKUP}" 2> /dev/null)
if [ -n "${_RESUME_TOKEN}" ] && [ "${_RESUME_TOKEN}" != "-" ]; then
print.important "Resume token present trying to resume at ${_RESUME_TOKEN}\n"
_COMMON_SNAPSHOT="@RESUME"
else
_RESUME_TOKEN=""
_COMMON_SNAPSHOT=$(zfs list -H -o name -S creation -t snapshot "${_ZFS}" 2> /dev/null | head -n 1)
! [ -z "${_COMMON_SNAPSHOT}" ] \
_COMMON_SNAPSHOT=$(zfs list -H -o name -S creation -t snapshot "${_ZFS_BACKUP}" 2> /dev/null | head -n 1)
[ -n "${_COMMON_SNAPSHOT}" ] \
&& print.data "Rolling back to newest snapshot: ${_COMMON_SNAPSHOT} ... " \
&& zfs rollback -r "${_COMMON_SNAPSHOT}" \
&& _COMMON_SNAPSHOT="@${_COMMON_SNAPSHOT#*@}" \
&& print.good "done\n"
fi
# Add "-s" for resumable streams in the next line at zfs receive. Not done yet because of: cannot receive resume stream: kernel modules must be upgraded to receive this stream.
${_SSH_COMMAND} "sudo ${_SEND_SCRIPT:?"Missing SEND_SCRIPT"} \"${_RECEIVERHOST}\" \"${_ZFS_BRANCH}\" \"${_COMPOSITION}\" \"@${_COMMON_SNAPSHOT#*@}\" \"${_RESUME_TOKEN}\"" | zfs receive -v "${_ZFS}"
${_SSH_COMMAND} "sudo ${_SEND_SCRIPT:?"Missing SEND_SCRIPT"} \"${_COMPOSITION}\" \"${_RECEIVERHOST}\" \"${_COMMON_SNAPSHOT}\" \"${_RESUME_TOKEN}\"" | zfs receive -v "${_ZFS_BACKUP}"
if [ $? -ne 0 ]; then
print.failure "Unable to receive stream" \
"These settings were used" \
"- Sending host: ${_SOURCEHOST}:${_SSH_PORT}" \
"- Receiving host: ${_RECEIVERHOST}" \
"- ZFS Branch: ${_ZFS_BRANCH}" \
"- Composition: ${_COMPOSITION}" \
"- Offered snapshot: @${_COMMON_SNAPSHOT#*@}" \
"- Receiving host: ${_RECEIVERHOST}" \
"- Offered snapshot: ${_COMMON_SNAPSHOT}" \
"- Resume token: ${_RESUME_TOKEN}"
print.highlight "Current state of snapshots:\n"
zfs list -t snapshot "${_ZFS}" 2> /dev/null | tail
tryRollbackToRepair "${_COMPOSITION}" "${_ZFS}" && return 0
zfs list -t snapshot "${_ZFS_BACKUP}" 2> /dev/null | tail
tryRollbackToRepair "${_COMPOSITION}" "${_ZFS_BACKUP}" && return 0
return 1
fi
protectZFS "${_ZFS}"
removeForeignSyncSnapshots "${_ZFS}"
removeOutdatedSyncSnapshots "${_ZFS}"
protectZFS "${_ZFS_BACKUP}"
removeForeignSyncSnapshots "${_ZFS_BACKUP}"
removeOutdatedSyncSnapshots "${_ZFS_BACKUP}"
) 9>>/tmp/synccomposition.${_COMPOSITION}.lock
script/host/zfs/snapshot.sh
+2 -2
View File
@@ -16,7 +16,7 @@ function cleanup() {
local _COMPOSITION _ZFS
composition.printAll | while read -r _COMPOSITION; do
_ZFS="$(composition.printZFS "${_COMPOSITION}")"
_ZFS="$(composition.printZfsVerified "${_COMPOSITION}")"
[ -n "${_ZFS}" ] \
&& printf -- "Cleaning snapshots of: '%b'\n" "${_ZFS}" \
&& local _LIST=( $(zfs list -t snap -H -o name -S creation "${_ZFS}" | grep -F '@SNAP' ) )
@@ -87,7 +87,7 @@ function snapshot() {
_MODE="${1:-"$(cat "${CIS[COMPOSITIONS]}${_COMPOSITION}/snapshot-mode" 2> /dev/null)"}"
_MODE="${_MODE:-"HOURLY"}"
_ZFS="$(composition.printZFS "${_COMPOSITION}")"
_ZFS="$(composition.printZfsVerified "${_COMPOSITION}")"
if [ -n "${_ZFS}" ]; then
(
flock -n 9 || return 1