zfs sync

script/zfs/sync/synccontainer-all.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+HOSTOWNER=$(cat /invra/hostowner)
+BACKUPHOST=$(hostname)
+STATE_DIR=/invra/state/${HOSTOWNER}/containers/;
+
+screen -ls | grep -oE "[0-9]+\.synccontainer\.[a-zA-Z0-9_-]+" | while read -r SCREEN_SESSION; do
+	CONTAINER=$(echo "$SCREEN_SESSION" | grep -oE "[^.]+$")
+	PID=$(echo "$SCREEN_SESSION" | grep -oE "^[0-9]+")
+	grep -iE "^${BACKUPHOST}$" ${STATE_DIR}/${CONTAINER}/standby-hosts > /dev/null
+	if [ $? -ne 0 ]; then
+		echo "quit screen session ${SCREEN_SESSION}"
+		screen -XS "$PID" quit
+	fi
+done
+
+grep -lrE "^${BACKUPHOST}$" /invra/state/${HOSTOWNER}/containers/*/standby-hosts > /dev/null
+if [ $? -eq 0 ]; then
+	grep -lrE "^${BACKUPHOST}$" /invra/state/${HOSTOWNER}/containers/*/standby-hosts | while read -r STANDBY_FILE; do
+		CONTAINER=$(basename $(dirname ${STANDBY_FILE}))
+		screen -ls | grep -oE "[0-9]+\.synccontainer\.$CONTAINER" > /dev/null
+		if [ $? -ne 0 ]; then
+			echo "starte container sync"
+			screen -dmS "synccontainer.$CONTAINER" /invra/scripts/hosts/zfs/synccontainer.sh "$CONTAINER"
+		fi
+	done
+fi
+

script/zfs/sync/synccontainer-check.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+TMP="$(mktemp)"
+(
+	HOSTNAME="$(hostname)"
+	HOSTOWNER="$(cat /invra/hostowner)"
+	MAX_BEHIND=0
+	CURRENT_UNIXTIME=$(date -u +%s)
+	echo "OK#Checks running" 
+	for CONTAINER_PATH in /invra/state/${HOSTOWNER}/containers/*; do 
+		grep -E "^${HOSTNAME}$" "${CONTAINER_PATH}/standby-hosts" &> /dev/null || continue;
+		CONTAINER_NAME="$(basename "$CONTAINER_PATH")";
+		TS=$(zfs list -o name -r -t snapshot "zpool1/persistent/${CONTAINER_NAME}-BACKUP" | grep "@SYNC_${HOSTNAME}" | head -n1 | grep -oP "\\d{4}-\\d{2}-\\d{2}_\\d{2}:\\d{2}:\\d{2}")
+		LAST_SNAPSHOT_TIME="$(echo "${TS}" | sed "s/_/ /g")"
+		LAST_SNAPSHOT_UNIXTIME=$(date -u --date="TZ=\"UTC\" ${LAST_SNAPSHOT_TIME}" +%s)
+		SECONDS_BEHIND=$[ $CURRENT_UNIXTIME - $LAST_SNAPSHOT_UNIXTIME ]
+		if [ "$SECONDS_BEHIND" -gt "$MAX_BEHIND" ]; then
+			MAX_BEHIND="$SECONDS_BEHIND"
+		fi
+		if [ "$SECONDS_BEHIND" -gt 30 ]; then
+			echo "LAGGING_SYNC_${CONTAINER_NAME}_${HOSTNAME}?FAIL#${SECONDS_BEHIND} behind"
+			
+		fi
+	done
+	echo $CURRENT_UNIXTIME
+) > "$TMP"
+chmod 655 "$TMP"
+mkdir -p /var/www/html/monitoring &>/dev/null
+mv "$TMP" /var/www/html/monitoring/synccontainer.check.txt
+
+

script/zfs/sync/synccontainer-receiver.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+CONTAINER=${1:?"CONTAINER missing"}
+CONTAINER=$(echo $1 | sed -E 's|[^a-zA-Z0-9_-]*||g')
+(
+	flock -n 9 || exit 1
+
+	BACKUPHOST=$(hostname)
+	HOSTOWNER=$(cat /invra/hostowner)
+	SOURCEHOST=$(cat /invra/state/${HOSTOWNER}/containers/${CONTAINER}/current-host)
+
+	MOUNTPOINT="none"
+	DATASET="zpool1/persistent/${CONTAINER}-BACKUP"
+	SNAPSHOT_PREFIX="${DATASET}@SYNC_${BACKUPHOST}_"
+
+	LAST_SNAPSHOT_NAME=""
+	RESUME_TOKEN=""
+	zfs list -Hr -o name -s name "${DATASET}" | grep -E "^${DATASET}$" > /dev/null
+	if [ $? -eq 0 ]; then
+		LAST_SNAPSHOT_NAME=$(zfs list -H -o name -S name -t snapshot -r "${DATASET}" | grep -E "^${SNAPSHOT_PREFIX}" | head -n 1)
+		RESUME_TOKEN="$(zfs get -o value -H receive_resume_token "${DATASET}")"
+	fi
+
+	if [[ "x$RESUME_TOKEN" != "x" && "x$RESUME_TOKEN" != "x-" ]]; then
+		echo "Resume token present trying to resume at $RESUME_TOKEN"
+		LAST_SNAPSHOT_NAME="RESUME"
+	fi
+	
+	if [[ "x${LAST_SNAPSHOT_NAME}" != "x" && "${LAST_SNAPSHOT_NAME}" != "RESUME" ]]; then
+		zfs rollback -r "${LAST_SNAPSHOT_NAME}"
+	fi
+
+	# Beiim zfs receive in der nächsten Zeile fehlt noch das "-s" für resumable streams. Der tzrlxsrv kann das aber momentan nicht. Fehlermeldung: cannot receive resume stream: kernel modules must be upgraded to receive this stream.
+	(while sleep 1; do echo; done) | ssh -o ConnectTimeout=20 -C invencom@${SOURCEHOST} "sudo /invra/scripts/hosts/zfs/synccontainer-sender.sh \"${BACKUPHOST}\" \"${CONTAINER}\" \"${LAST_SNAPSHOT_NAME#$SNAPSHOT_PREFIX}\"" \"${RESUME_TOKEN}\" | zfs receive -v "${DATASET}"
+	if [ $? -ne 0 ]; then
+		exit 1
+	fi
+
+	# Dataset gegen Veränderungen sichern
+	zfs set readonly=on "${DATASET}"
+	zfs set "mountpoint=${MOUNTPOINT}" "${DATASET}"
+
+	# Aufsetzpunkte fremder Synchronisierer wegräumen
+	zfs list -t snapshot -o name -r "${DATASET}" | grep -- "${DATASET}@SYNC" | grep -v -i "_${BACKUPHOST}_" | while read SNAP; do
+		echo "Destroying $SNAP"
+		zfs destroy $SNAP
+	done
+
+	# Alte Snapshots wegräumen
+	while read -r ZEILE
+	do
+		if [ "$ZEILE" = "" ]; then
+			break
+		fi
+		if [[ "$ZEILE" > "$LAST_SNAPSHOT_NAME"  ]]; then
+			break
+		fi
+		zfs destroy "$ZEILE"
+	done < <(zfs list -Hr -o name -s name -t snapshot "${DATASET}" | grep -E "^${SNAPSHOT_PREFIX}")
+) 9>>/tmp/synccontainer.${CONTAINER}.lock
+
+if [ $? -ne 0 ]; then
+	exit 1
+fi
+exit 0

script/zfs/sync/synccontainer-sender.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+BACKUPHOST=${1:?"BACKUPHOST missing"}
+CONTAINER=${2:?"CONTAINER missing"}
+BACKUPHOST=$(echo $1 | sed -E 's|[^a-zA-Z0-9._-]*||g')
+CONTAINER=$(echo $2 | sed -E 's|[^a-zA-Z0-9_-]*||g')
+LAST_SNAPSHOT=$(echo $3 | sed -E 's|[^a-zA-Z0-9._:-]*||g')
+NEW_SNAPSHOT=$(date -u "+%Y-%m-%d_%H:%M:%S")
+
+if [[ "${LAST_SNAPSHOT}" == "RESUME" ]]; then
+	RESUME_TOKEN=$(echo $4 | sed -E 's|[^a-zA-Z0-9._:-]*||g')
+        zfs send -t "${RESUME_TOKEN}"
+        exit
+fi
+
+DATASET="zpool1/persistent/$CONTAINER"
+SNAPSHOT_PREFIX="${DATASET}@SYNC_${BACKUPHOST}_"
+LAST_SNAPSHOT_NAME="${SNAPSHOT_PREFIX}${LAST_SNAPSHOT}"
+NEW_SNAPSHOT_NAME="${SNAPSHOT_PREFIX}${NEW_SNAPSHOT}"
+SNAPSHOT_FOUND=""
+
+# Existiert der Snapshot?
+while read -r ZEILE
+do
+	if [[ "$ZEILE" == "$LAST_SNAPSHOT_NAME" ]]; then
+		SNAPSHOT_FOUND="1"
+		continue
+	fi
+done < <(zfs list -H -o name -s name -t snapshot "${DATASET}" | grep -E "^${SNAPSHOT_PREFIX}")
+
+# Falls ja, alle anderen Snapshots wegräumen - eine frühere Version des Skripts hat hier nur die Älteren weggeräumt. Das führt allerdings zum Vollmüllen
+# mit neueren Snapshots, wenn der Sync immer wieder fehlschlägt - im Einzelfall bis zur Unbenutzbarkeit des Senders
+if [[ "${SNAPSHOT_FOUND}x" == "1x" ]]; then
+	while read -r ZEILE
+	do
+	        if [[ "$ZEILE" == "$LAST_SNAPSHOT_NAME" ]]; then
+        	        continue
+	        fi
+        	zfs destroy "$ZEILE"
+	done < <(zfs list -H -o name -s name -t snapshot "${DATASET}" | grep -E "^${SNAPSHOT_PREFIX}")
+fi
+
+zfs snapshot "$NEW_SNAPSHOT_NAME"
+
+if [[ "$LAST_SNAPSHOT" != "" ]]; then
+	if [[ "$SNAPSHOT_FOUND" == "" ]]; then
+		echo "Angeforderter Snapshot '${LAST_SNAPSHOT}' nicht vorhanden"
+		exit 1;
+	fi
+	zfs send -I "${LAST_SNAPSHOT_NAME}" "${NEW_SNAPSHOT_NAME}"
+else
+	zfs send "${NEW_SNAPSHOT_NAME}"
+fi

script/zfs/sync/synccontainer.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+BACKUPHOST=$(hostname)
+CONTAINER=${1:?"Kein Container angegeben"}
+DATASET="zpool1/persistent/$CONTAINER"
+SNAPSHOT_PREFIX="${DATASET}@SYNC_${BACKUPHOST}_"
+
+while true; do
+
+	/invra/scripts/hosts/zfs/synccontainer-receiver.sh "$CONTAINER"
+	sleep 5
+
+#	LAST_SNAPSHOT_NAME=$(zfs list -Hr -o name -S name -t snapshot "${DATASET}" | grep -E "^${SNAPSHOT_PREFIX}" | head -n 1)
+#	LAST_SNAPSHOT_TIME=${LAST_SNAPSHOT_NAME#${SNAPSHOT_PREFIX}}
+#	LAST_SNAPSHOT_TIME="$(echo "${LAST_SNAPSHOT_TIME}" | sed "s/_/ /g")"
+#	LAST_SNAPSHOT_UNIXTIME=$(date -u --date="TZ=\"UTC\" ${LAST_SNAPSHOT_TIME}" +%s)
+#	CURRENT_UNIXTIME=$(date -u +%s)
+#	SECONDS_BEHIND=$[ $CURRENT_UNIXTIME - $LAST_SNAPSHOT_UNIXTIME ]
+#	mkdir -p /var/www/html/monitoring > /dev/null 2>&1
+#	echo $CURRENT_UNIXTIME > "/var/www/html/monitoring/containersync.${CONTAINER}"
+#	echo "OK: $SECONDS_BEHIND seconds behind" >> "/var/www/html/monitoring/containersync.${CONTAINER}"
+
+done
+