First zfs scripts

This commit is contained in:
m8in
2026-04-18 00:30:13 +02:00
parent 72bbf86921
commit 74f381cc1c
3 changed files with 441 additions and 0 deletions
+122
View File
@@ -0,0 +1,122 @@
#!/bin/bash
function printFoundCommonSnapshot() {
local _ZFS _COMMON_SNAPSHOT_CANDIDATE
_ZFS="${1:?"printFoundCommonSnapshot(): Missing first parameter ZFS"}"
_COMMON_SNAPSHOT_CANDIDATE="${2:?"printFoundCommonSnapshot(): Missing second parameter COMMON_SNAPSHOT_CANDIDATE"}"
readonly _ZFS _COMMON_SNAPSHOT_CANDIDATE
local _FOUND_COMMON_SNAPSHOT
_FOUND_COMMON_SNAPSHOT=""
while read -r _ROW
do
if [ "${_ROW}" == "${_ZFS}@${_COMMON_SNAPSHOT_CANDIDATE}" ]; then
_FOUND_COMMON_SNAPSHOT="${_ROW}"
break
fi
done < <(zfs list -H -o name -S creation -t snapshot "${_ZFS}")
[ $? -eq 0 ] \
&& echo "${_FOUND_COMMON_SNAPSHOT}" \
&& return 0
return 1
}
function removeAllSyncSnapshotsExeptTheCommonOne() {
local _ZFS _RECEIVERHOST _COMMON_SNAPSHOT
_ZFS="${1:?"removeAllSyncSnapshotsExeptTheCommonOne(): Missing first parameter ZFS"}"
_RECEIVERHOST="${2:?"removeAllSyncSnapshotsExeptTheCommonOne(): Missing second parameter RECEIVERHOST"}"
_COMMON_SNAPSHOT="${3:?"removeAllSyncSnapshotsExeptTheCommonOne(): Missing third parameter COMMON_SNAPSHOT"}"
readonly _ZFS _RECEIVERHOST _COMMON_SNAPSHOT
while read -r _ROW
do
# Skip the common snapshot to keep it.
# If the common snapshot is not a sync-snapshot all sync-snapshots will be removed.
if [ "${_ROW}" == "${_COMMON_SNAPSHOT}" ]; then
continue
fi
# Destroy all remaining sync-snapshots of the receiving host
zfs destroy "${_ROW}"
done < <(zfs list -H -o name -S creation -t snapshot "${_ZFS}" | grep -E "^${_ZFS}@SYNC_${_RECEIVERHOST}_")
}
function sendResume() {
local _RESUME_TOKEN
_RESUME_TOKEN="${1:?"sendResume(): Missing first parameter RESUME_TOKEN"}"
readonly _RESUME_TOKEN
zfs send -t "${RESUME_TOKEN}" \
&& return 0
return 1
}
function isValid() {
# printf '%s'
# - always treats the contents of ${1} as pure plain text.
# grep -qE: checks RegExp, but quiet
printf '%s' "${1}" | grep -qE "${2:?"isValid(): Missing REGEXP"}"
}
function isValidOptional() {
[ -z "${1}" ] || isValid "${1}" "${2}"
}
# Parameter 1: Only alphanumeric characters allowed and [._-] if not leading (due to: -oProxyCommand=...).
# Parameter 2: Only alphanumeric characters allowed and [.-] if not leading (due to: -oProxyCommand=...).
# Parameter 3: Only alphanumeric characters allowed and [._:-] if not leading (due to: -oProxyCommand=...), but can be empty.
# Parameter 4: Only alphanumeric characters allowed and [._:-] if not leading (due to: -oProxyCommand=...), but can be empty.
if isValid "${1:?"RECEIVERHOST missing"}" '^[a-zA-Z0-9][a-zA-Z0-9._-]*$' \
&& isValid "${2:?"COMPOSITION missing"}" '^[a-zA-Z0-9][a-zA-Z0-9.-]*$' \
&& isValidOptional "${3}" '^[a-zA-Z0-9][a-zA-Z0-9._:-]*$' \
&& isValidOptional "${4}" '^[a-zA-Z0-9][a-zA-Z0-9._:-]*$'
then
_RECEIVERHOST="${1}"
_COMPOSITION="${2}"
_RECEIVERS_SNAPSHOT="${3}"
_RESUME_TOKEN="${4}"
_ZFS="zpool1/persistent/${_COMPOSITION:?"COMPOSITION missing"}"
# Resume mode
if [ "${_RECEIVERS_SNAPSHOT}" == "RESUME" ]; then
sendResume "${_RESUME_TOKEN}"
exit $?
fi
# This common snapshot is the starting-point, if available.
_COMMON_SNAPSHOT="$(printFoundCommonSnapshot ${_ZFS} ${_RECEIVERS_SNAPSHOT})"
[ "${_COMMON_SNAPSHOT}" == "" ] \
&& [ "${_RECEIVERS_SNAPSHOT}" != "" ] \
&& echo "Requested snapshot '${_RECEIVERS_SNAPSHOT}' not available" \
&& exit 1
[ "${_COMMON_SNAPSHOT}" != "" ] \
&& removeAllSyncSnapshotsExeptTheCommonOne "${_ZFS}" "${_RECEIVERHOST}" "${_COMMON_SNAPSHOT}"
# Now create the first or a further sync-snapshot as end-point.
_NEW_SNAPSHOT="${_ZFS}@SYNC_${_RECEIVERHOST:?"RECEIVERHOST missing"}_$(date -u "+%Y-%m-%d_%H:%M:%S")"
[ "${_COMMON_SNAPSHOT}" == "" ] \
&& zfs snapshot "${_NEW_SNAPSHOT}" \
&& zfs send -c -R "${_NEW_SNAPSHOT}" \
&& exit 0
[ "${_COMMON_SNAPSHOT}" != "" ] \
&& removeAllSyncSnapshotsExeptTheCommonOne "${_ZFS}" "${_RECEIVERHOST}" "${_COMMON_SNAPSHOT}" \
&& zfs snapshot "${_NEW_SNAPSHOT}" \
&& zfs send -c -R -I "${_COMMON_SNAPSHOT}" "${_NEW_SNAPSHOT}" \
&& exit 0
else
echo "Failure: At least one parameter is invalid" >&2
exit 1
fi
exit 1
+201
View File
@@ -0,0 +1,201 @@
#!/bin/bash
_MODE=$(echo "${1:?"MODE missing [--all, --once, --loop]"}" | sed -E 's|[^a-zA-Z0-9_-]*||g')
_COMPOSITION=$(echo "${2}" | sed -E 's|[^a-zA-Z0-9_-]*||g')
_SSH_PORT=$(echo "${3:-22}" | sed -E 's/[^0-9]//g')
# Folders always ends with an tailing '/'
_SCRIPT="$(readlink -f "${0}" 2> /dev/null)"
_CIS_ROOT="${_SCRIPT%/script/host/zfs/composition-sync/sync.sh}/" #Removes shortest matching pattern '/script/host/zfs/composition-sync/sync.sh' from the end
_SEND_SCRIPT="${_CIS_ROOT:?"Missing CIS_ROOT"}script/host/zfs/composition-sync/sync-send.sh"
_DOMAIN="$("${_CIS_ROOT:?"Missing CIS_ROOT"}core/printOwnDomain.sh")"
_DEFINITIONS="${_CIS_ROOT:?"Missing CIS_ROOT"}definitions/${_DOMAIN:?"Missing DOMAIN"}/"
_RECEIVERHOST=$(hostname -b)
function stopObsoleteScreenSession() {
local _RECEIVERHOST _SYNCHOSTS_FILE _SCREEN_SESSION _COMPOSITION _PID
_RECEIVERHOST="${1:?"stopObsoleteScreenSession(): Missing first parameter RECEIVERHOST"}"
_SYNCHOSTS_FILE="${2:?"stopObsoleteScreenSession(): Missing second parameter SYNCHOSTS_FILE"}"
_SCREEN_SESSION="${3:?"stopObsoleteScreenSession(): Missing third parameter SCREEN_SESSION"}"
_COMPOSITION=$(echo "$_SCREEN_SESSION" | grep -oE "[^.]+$")
_PID=$(echo "$_SCREEN_SESSION" | grep -oE "^[0-9]+")
readonly _RECEIVERHOST _SYNCHOSTS_FILE _SCREEN_SESSION _COMPOSITION _PID
! grep -qiE "^${_RECEIVERHOST}$" "${_DEFINITIONS}compositions/${_COMPOSITION}/${_SYNCHOSTS_FILE}" \
&& echo "Stopping screen session of composition-sync: ${_COMPOSITION}" \
&& screen -XS "${_PID}" quit
}
function cleanSessions() {
local _RECEIVERHOST _SYNCHOSTS_FILE
_RECEIVERHOST="${1:?"cleanSessions(): Missing first parameter RECEIVERHOST"}"
_SYNCHOSTS_FILE="${2:?"cleanSessions(): Missing second parameter SYNCHOSTS_FILE"}"
readonly _RECEIVERHOST _SYNCHOSTS_FILE
screen -ls | grep -oE "[0-9]+\.compositionsync\.[a-zA-Z0-9_-]+" | while read -r _SCREEN_SESSION; do
stopObsoleteScreenSession "${_RECEIVERHOST}" "${_SYNCHOSTS_FILE}" "${_SCREEN_SESSION}"
done
}
function startMissingScreenSession() {
local _COMPOSITION _SSH_PORT
_COMPOSITION="${1:?"startMissingScreenSession(): Missing first parameter COMPOSITION"}"
_SSH_PORT="${2:-22}"
readonly _COMPOSITION _SSH_PORT
! screen -ls | grep -qoE "[0-9]+\.compositionsync\.${_COMPOSITION}" \
&& echo "Starting screen session of composition-sync: ${_COMPOSITION}" \
&& screen -dmS "compositionsync.${_COMPOSITION}" "${_SCRIPT}" --loop "${_COMPOSITION}" "${_SSH_PORT}"
}
function addSessions() {
local _RECEIVERHOST _SYNCHOSTS_FILE
_RECEIVERHOST="${1:?"addSessions(): Missing first parameter RECEIVERHOST"}"
_SYNCHOSTS_FILE="${2:?"addSessions(): Missing second parameter SYNCHOSTS_FILE"}"
readonly _RECEIVERHOST _SYNCHOSTS_FILE
local _COMPOSITION
grep -lrE "^${_RECEIVERHOST}" ${_DEFINITIONS}compositions/*/${_SYNCHOSTS_FILE} | while read -r _CURRENT_SYNCHOSTS_FILE; do
_SSH_PORT=$(grep -E "^${_RECEIVERHOST} usePort [0-9]*.*$" "${_CURRENT_SYNCHOSTS_FILE}" | cut -d' ' -f3 | xargs)
_COMPOSITION=$(basename $(dirname "${_CURRENT_SYNCHOSTS_FILE}"))
startMissingScreenSession "${_COMPOSITION}" "${_SSH_PORT}"
done
}
function destroySyncSnapshot() {
local _ZFS _SNAPSHOT
_ZFS="${1:?"destroySyncSnapshot(): Missing first parameter ZFS"}"
_SNAPSHOT="${2}"
readonly _ZFS _SNAPSHOT
# Nothing to do
[ -z "${_SNAPSHOT}" ] && return 0
echo "${_SNAPSHOT}" | grep -qF "${_ZFS:?"destroySyncSnapshot(): Missing ZFS"}@SYNC" \
&& zfs destroy "${_SNAPSHOT}" \
&& return 0
return 1
}
function protectZFS() {
local _ZFS
_ZFS="${1:?"protectZFS(): Missing first parameter ZFS"}"
readonly _ZFS
zfs set readonly=on "${_ZFS}"
zfs set mountpoint=none "${_ZFS}"
return 0
}
function removeForeignSyncSnapshots() {
local _RECEIVERHOST _ZFS
_RECEIVERHOST="${1:?"removeForeignSyncSnapshots(): Missing first parameter RECEIVERHOST"}"
_ZFS="${2:?"removeForeignSyncSnapshots(): Missing second parameter ZFS"}"
readonly _RECEIVERHOST _ZFS
zfs list -t snapshot -H -o name "${_ZFS}" | grep -- "${_ZFS}@SYNC" | grep -v -i "@SYNC_${_RECEIVERHOST}_" | while read _SNAP; do
echo -n "Removing foreign snapshot: ${_SNAP} ... " \
&& destroySyncSnapshot "${_ZFS}" "${_SNAP}" \
&& echo "done"
done
return 0
}
function removeOutdatedSyncSnapshots() {
local _RECEIVERHOST _ZFS _NEWEST_SNAPSHOT
_RECEIVERHOST="${1:?"removeOutdatedSyncSnapshots(): Missing first parameter RECEIVERHOST"}"
_ZFS="${2:?"removeOutdatedSyncSnapshots(): Missing second parameter ZFS"}"
_NEWEST_SNAPSHOT=$(zfs list -H -o name -S name -t snapshot "${_ZFS}" | grep -E "^${_ZFS}@SYNC_${_RECEIVERHOST}_" | head -n 1)
readonly _RECEIVERHOST _ZFS _NEWEST_SNAPSHOT
# Nothing to do, because if there is no newest snapshot then there cannot be anyone
[ -z "${_NEWEST_SNAPSHOT}" ] && return 0
# Remove all but the newest snapshot, which is the common snapshot in the next run
zfs list -t snapshot -H -o name "${_ZFS}" | grep -- "${_ZFS}@SYNC_${_RECEIVERHOST}_" | grep -v -i "${_NEWEST_SNAPSHOT}" | while read _SNAP; do
echo -n "Removing outdated snapshot: ${_SNAP} ... " \
&& destroySyncSnapshot "${_ZFS}" "${_SNAP}" \
&& echo "done"
done
return 0
}
function receive() {
local _RECEIVERHOST _COMPOSITION
_RECEIVERHOST="${1:?"receive(): Missing first parameter RECEIVERHOST"}"
_COMPOSITION="${2:?"receive(): Missing second parameter COMPOSITION"}"
readonly _RECEIVERHOST _COMPOSITION
(
flock -n 9 || exit 1
_SOURCEHOST=$(cat ${_DEFINITIONS}compositions/${_COMPOSITION}/current-host)
_ZFS="zpool1/persistent/${_COMPOSITION}-BACKUP"
_SSH_COMMAND="ssh -p ${_SSH_PORT} -o ConnectTimeout=20 -o ServerAliveInterval=15 -C composition-sync@${_SOURCEHOST}"
_COMMON_SNAPSHOT=""
_RESUME_TOKEN=$(zfs get -H -o value receive_resume_token "${_ZFS}" 2> /dev/null)
if [ -n "${_RESUME_TOKEN}" ] && [ "${_RESUME_TOKEN}" != "-" ]; then
echo "Resume token present trying to resume at ${_RESUME_TOKEN}"
_COMMON_SNAPSHOT="RESUME"
else
_RESUME_TOKEN=""
_COMMON_SNAPSHOT=$(zfs list -H -o name -S creation -t snapshot "${_ZFS}" 2> /dev/null | head -n 1)
! [ -z "${_COMMON_SNAPSHOT}" ] \
&& echo "Rolling back to newest snapshot: ${_COMMON_SNAPSHOT}" \
&& zfs rollback -r "${_COMMON_SNAPSHOT}"
fi
# Add "-s" for resumable streams in the next line at zfs receive. Not done yet because of: cannot receive resume stream: kernel modules must be upgraded to receive this stream.
${_SSH_COMMAND} "sudo ${_SEND_SCRIPT:?"Missing SEND_SCRIPT"} \"${_RECEIVERHOST}\" \"${_COMPOSITION}\" \"${_COMMON_SNAPSHOT#${_ZFS}@}\" \"${_RESUME_TOKEN}\"" | zfs receive -v "${_ZFS}"
[ $? -ne 0 ] \
&& echo "Unable to receive stream unsing these settings:" \
&& echo " - Sending host: ${_SOURCEHOST}:${_SSH_PORT}" \
&& echo " - Receiving host: ${_RECEIVERHOST}" \
&& echo " - Composition: ${_COMPOSITION}" \
&& echo " - Offered snapshot: ${_COMMON_SNAPSHOT}" \
&& echo " - Resume token: ${_RESUME_TOKEN}" \
&& return 1
protectZFS "${_ZFS}"
removeForeignSyncSnapshots "${_RECEIVERHOST}" "${_ZFS}"
removeOutdatedSyncSnapshots "${_RECEIVERHOST}" "${_ZFS}"
) 9>>/tmp/synccomposition.${_COMPOSITION}.lock
[ $? -eq 0 ] && return 0
return 1
}
[ "${_MODE}" == "--all" ] \
&& cleanSessions "${_RECEIVERHOST}" composition-sync-hosts \
&& addSessions "${_RECEIVERHOST}" composition-sync-hosts \
&& exit 0
[ "${_MODE}" == "--once" ] \
&& receive "${_RECEIVERHOST}" "${_COMPOSITION}" \
&& exit 0
[ "${_MODE}" == "--loop" ] && while true; do
receive "${_RECEIVERHOST}" "${_COMPOSITION}" \
&& echo "Sleep for 5s" \
&& sleep 5 \
&& echo \
&& continue
echo
echo "Waiting 5min then ABORT!"
sleep 300
break
done
exit 1
+118
View File
@@ -0,0 +1,118 @@
#!/bin/bash
if [ "$#" -lt 2 ]; then
echo "Compares the difference of two snapshots of the same dataset."
echo "If a file was modified, but has the same content, it will be skipped."
echo "Remaining files can be analysed more deeply using --singlefile mode."
echo
echo "Usage:"
echo " - $0 <dataset>@<snap1> <dataset>@<snap2> [M+-] # scans files, default is modified only [M]"
echo " - $0 <dataset>@<snap1> <dataset>@<snap2> --singlefile [path] # deeper look at one file's differences"
exit 1
fi
SNAP1_FULL=$1
SNAP2_FULL=$2
TYPES_FILTER=${3:-M}
TARGET_PATH=$4
DATASET=${SNAP1_FULL%@*}
SNAP1_NAME=${SNAP1_FULL#*@}
SNAP2_NAME=${SNAP2_FULL#*@}
MOUNTPOINT=$(zfs get -H -o value mountpoint "$DATASET")
if [ "$MOUNTPOINT" == "none" ] || [ ! -d "$MOUNTPOINT" ]; then
echo "Failure: dataset is not accessable via its mountpoint."
exit 1
fi
[ "${#TYPES_FILTER}" -gt 3 ] && [ "$TYPES_FILTER" != "--singlefile" ] \
&& [ -n "$TARGET_PATH"] \
&& echo "Failure: Mode ${TYPES_FILTER} unknown." \
&& exit 1
[ "$TYPES_FILTER" == "--singlefile" ] \
&& [ -z "$TARGET_PATH"] \
&& echo "Failure: Mode --singlefile requires path." \
&& exit 1
if [ "$TYPES_FILTER" == "--singlefile" ] && [ -n "$TARGET_PATH" ]; then
rel_path=${TARGET_PATH#$MOUNTPOINT/}
file1="$MOUNTPOINT/.zfs/snapshot/$SNAP1_NAME/$rel_path"
file2="$MOUNTPOINT/.zfs/snapshot/$SNAP2_NAME/$rel_path"
echo "Vergleiche: $rel_path"
echo "Snapshot 1: $file1"
echo "Snapshot 2: $file2"
echo "--------------------------------------------------------"
if [ ! -f "$file1" ] && [ ! -f "$file2" ]; then
echo "Fehler: Datei existiert in beiden Snapshots nicht."
exit 1
fi
# Standard Diff (meldet 'Binary files differ' bei Binärdateien)
diff -u "$file1" "$file2"
# Alternativ: vimdiff (einfach die obere Zeile auskommentieren und hier das # entfernen)
# vimdiff "$file1" "$file2"
exit 0
fi
echo -e "Diff(Bytes)\tPfad"
echo -e "--------------------------------------------------------"
zfs diff -H "$SNAP1_FULL" "$SNAP2_FULL" | while IFS=$'\t' read -r type path; do
if [[ ! "$TYPES_FILTER" == *"$type"* ]]; then
continue
fi
rel_path=${path#$MOUNTPOINT/}
file1="$MOUNTPOINT/.zfs/snapshot/$SNAP1_NAME/$rel_path"
file2="$MOUNTPOINT/.zfs/snapshot/$SNAP2_NAME/$rel_path"
case "$type" in
"M")
if [ -f "$file1" ] && [ -f "$file2" ]; then
read -r size1 mtime1 < <(stat -c "%s %Y" "$file1")
read -r size2 mtime2 < <(stat -c "%s %Y" "$file2")
diff_val=$((size2 - size1))
if [ "$diff_val" -eq 0 ]; then
[ "$mtime1" == "$mtime2" ] && continue
hash_line1=$(sha1sum "$file1")
sha1_1=${hash_line1%% *}
hash_line2=$(sha1sum "$file2")
sha1_2=${hash_line2%% *}
[ "$sha1_1" == "$sha1_2" ] && continue
echo -e "0\t\t${path}"
else
echo -e "${diff_val}\t\t${path}"
fi
fi
;;
"+")
if [ -f "$file2" ]; then
size2=$(stat -c%s "$file2")
echo -e "+${size2}\t\t${path}"
fi
;;
"-")
if [ -f "$file1" ]; then
size1=$(stat -c%s "$file1")
echo -e "-${size1}\t\t${path}"
fi
;;
esac
done
echo -e "--------------------------------------------------------"
echo -e "Use the following command for a deeper look at one file:"
echo -e "$0 $1 $2 --singlefile [Pfad]"