better structure

2025-12-06 07:48:26 +01:00 · 2025-06-21 01:04:31 +02:00
parent 5901afdaae
commit ac7f693ec9
9 changed files with 0 additions and 0 deletions
--- a/script/host/zfs/snapshot/snapshot-cleanup.sh
+++ b/script/host/zfs/snapshot/snapshot-cleanup.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+MIN_MIN=$(date --date="- 5 minutes" -u "+%Y%m%d%H%M")
+HOUR_MIN=$(date --date="- 1 days" -u "+%Y%m%d%H")
+DAY_MIN=$(date --date="- 7 days" -u "+%Y%m%d")
+MONTH_MIN=$(date --date="- 3 years" -u "+%Y%m")
+ 
+zfs list -Hr -o name -t snapshot -r "zpool1/persistent" | grep -E "^zpool1/persistent/[a-zA-Z0-9_-]+@(SNAPHOURLY|SNAPDAILY|SNAPMONTHLY|SNAPMINUTLY)_[0-9]{6,12}$" | while read SNAPSHOT; do
+        SNAPSHOT_TIME=$(echo "$SNAPSHOT" | grep -oE "[0-9]+$")
+        if [[ ${#SNAPSHOT_TIME} == 12 && "$SNAPSHOT_TIME" < "${MIN_MIN}" ]]; then
+                zfs destroy "${SNAPSHOT}"
+        fi
+        if [[ ${#SNAPSHOT_TIME} == 10 && "$SNAPSHOT_TIME" < "${HOUR_MIN}" ]]; then
+                zfs destroy "${SNAPSHOT}"
+        fi
+        if [[ ${#SNAPSHOT_TIME} == 8 && "${SNAPSHOT_TIME}" < "${DAY_MIN}" ]]; then
+                zfs destroy "${SNAPSHOT}"
+        fi
+        if [[ ${#SNAPSHOT_TIME} == 6 && "${SNAPSHOT_TIME}" < "${MONTH_MIN}" ]]; then
+                zfs destroy "${SNAPSHOT}"
+        fi
+done
+ 
+ 
+ 
+MONTH_MIN_QA=$(date --date="- 1 month" -u "+%Y%m")
+ 
+zfs list -Hr -o name -t snapshot -r "zpool1/persistent" | grep -E "^zpool1/persistent/[a-zA-Z0-9_-]+-qa@SNAPMONTHLY_[0-9]{6}$" | while read SNAPSHOT_QA; do
+        SNAPSHOT_TIME_QA=$(echo "$SNAPSHOT_QA" | grep -oE "[0-9]+$")
+        if [[ "${SNAPSHOT_TIME_QA}" < "${MONTH_MIN_QA}" ]]; then
+                zfs destroy "${SNAPSHOT_QA}"
+        fi
+done
+
--- a/script/host/zfs/snapshot/snapshot-minutly.sh
+++ b/script/host/zfs/snapshot/snapshot-minutly.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin
+
+_TIMESTAMP="$(date -u "+%Y%m%d%H%M")"
+_ZFS_FILESYSTEM="${1:?"Missing first parameter ZFS_FILESYSTEM."}"
+echo "${_ZFS_FILESYSTEM}" | grep -E '\-prod$' &> /dev/null \
+    && zfs snapshot "${_ZFS_FILESYSTEM}@SNAPMINUTLY_${_TIMESTAMP}" \
+    && exit 0
+
+echo "Snapshot konnte nicht angelegt werden:"
+echo "  - ${_ZFS_FILESYSTEM}@SNAPMINUTLY_${_TIMESTAMP}"
+echo "  (Minuten-Snapshots sollen nur auf 'PROD'-Containeren angelegt werden, sodass diese dann syncronisiert werden)"
+exit 1
--- a/script/host/zfs/snapshot/snapshot.sh
+++ b/script/host/zfs/snapshot/snapshot.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+HOUR=$(date -u "+%Y%m%d%H")
+DAY=${HOUR:0:8}
+MONTH=${HOUR:0:6}
+HOSTOWNER=$(cat /invra/hostowner)
+if [ ! -d /tmp/locks ]; then
+	mkdir /tmp/locks
+fi
+
+zfs list -Hr -o name zpool1/persistent | grep -v -- -BACKUP | tail -n +2 | while read DATASET; do
+	CONTAINER=${DATASET#zpool1/persistent/}
+	(
+		flock -n 9 || exit 1
+
+		MODE_FILE="/invra/state/$HOSTOWNER/containers/$CONTAINER/snapshot-mode"
+		HOURLY=1
+		DAILY=1
+		MONTHLY=1
+
+		if [ -f "$MODE_FILE" ]; then
+			grep -i "NONE" "$MODE_FILE" &> /dev/null
+			if [ $? -eq 0 ]; then
+				exit
+			fi
+			grep -i "HOURLY" "$MODE_FILE" &> /dev/null
+			if [ $? -ne 0 ]; then
+				HOURLY=0
+			fi
+			grep -i "DAILY" "$MODE_FILE" &> /dev/null
+			if [ $? -ne 0 ]; then
+				DAILY=0
+			fi
+			grep -i "MONTHLY" "$MODE_FILE" &> /dev/null
+			if [ $? -ne 0 ]; then
+				MONTHLY=0
+			fi
+		fi
+		SNAPSHOT_HOUR="${DATASET}@SNAPHOURLY_${HOUR}"
+		SNAPSHOT_DAY="${DATASET}@SNAPDAILY_${DAY}"
+		SNAPSHOT_MONTH="${DATASET}@SNAPMONTHLY_${MONTH}"
+
+		zfs list -H -t snapshot -o name -r "$DATASET" | grep -E "^${SNAPSHOT_HOUR}$" > /dev/null
+		if [[ $? -ne 0 && $HOURLY -eq 1 ]]; then
+			zfs snapshot "${SNAPSHOT_HOUR}"
+		fi
+
+		zfs list -H -t snapshot -o name -r "$DATASET" | grep -E "^${SNAPSHOT_DAY}$" > /dev/null
+		if [[ $? -ne 0 && $DAILY -eq 1 ]]; then
+			zfs snapshot "${SNAPSHOT_DAY}"
+		fi
+
+		zfs list -H -t snapshot -o name -r "$DATASET" | grep -E "^${SNAPSHOT_MONTH}$" > /dev/null
+		if [[ $? -ne 0 && $MONTHLY -eq 1 ]]; then
+			zfs snapshot "${SNAPSHOT_MONTH}"
+		fi
+	) 9>>/tmp/locks/snapshot.${CONTAINER}.lock
+done
--- a/script/host/zfs/sync/synccontainer-all.sh
+++ b/script/host/zfs/sync/synccontainer-all.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+HOSTOWNER=$(cat /invra/hostowner)
+BACKUPHOST=$(hostname)
+STATE_DIR=/invra/state/${HOSTOWNER}/containers/;
+
+screen -ls | grep -oE "[0-9]+\.synccontainer\.[a-zA-Z0-9_-]+" | while read -r SCREEN_SESSION; do
+	CONTAINER=$(echo "$SCREEN_SESSION" | grep -oE "[^.]+$")
+	PID=$(echo "$SCREEN_SESSION" | grep -oE "^[0-9]+")
+	grep -iE "^${BACKUPHOST}$" ${STATE_DIR}/${CONTAINER}/standby-hosts > /dev/null
+	if [ $? -ne 0 ]; then
+		echo "quit screen session ${SCREEN_SESSION}"
+		screen -XS "$PID" quit
+	fi
+done
+
+grep -lrE "^${BACKUPHOST}$" /invra/state/${HOSTOWNER}/containers/*/standby-hosts > /dev/null
+if [ $? -eq 0 ]; then
+	grep -lrE "^${BACKUPHOST}$" /invra/state/${HOSTOWNER}/containers/*/standby-hosts | while read -r STANDBY_FILE; do
+		CONTAINER=$(basename $(dirname ${STANDBY_FILE}))
+		screen -ls | grep -oE "[0-9]+\.synccontainer\.$CONTAINER" > /dev/null
+		if [ $? -ne 0 ]; then
+			echo "starte container sync"
+			screen -dmS "synccontainer.$CONTAINER" /invra/scripts/hosts/zfs/synccontainer.sh "$CONTAINER"
+		fi
+	done
+fi
+
--- a/script/host/zfs/sync/synccontainer-check.sh
+++ b/script/host/zfs/sync/synccontainer-check.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+TMP="$(mktemp)"
+(
+	HOSTNAME="$(hostname)"
+	HOSTOWNER="$(cat /invra/hostowner)"
+	MAX_BEHIND=0
+	CURRENT_UNIXTIME=$(date -u +%s)
+	echo "OK#Checks running" 
+	for CONTAINER_PATH in /invra/state/${HOSTOWNER}/containers/*; do 
+		grep -E "^${HOSTNAME}$" "${CONTAINER_PATH}/standby-hosts" &> /dev/null || continue;
+		CONTAINER_NAME="$(basename "$CONTAINER_PATH")";
+		TS=$(zfs list -o name -r -t snapshot "zpool1/persistent/${CONTAINER_NAME}-BACKUP" | grep "@SYNC_${HOSTNAME}" | head -n1 | grep -oP "\\d{4}-\\d{2}-\\d{2}_\\d{2}:\\d{2}:\\d{2}")
+		LAST_SNAPSHOT_TIME="$(echo "${TS}" | sed "s/_/ /g")"
+		LAST_SNAPSHOT_UNIXTIME=$(date -u --date="TZ=\"UTC\" ${LAST_SNAPSHOT_TIME}" +%s)
+		SECONDS_BEHIND=$[ $CURRENT_UNIXTIME - $LAST_SNAPSHOT_UNIXTIME ]
+		if [ "$SECONDS_BEHIND" -gt "$MAX_BEHIND" ]; then
+			MAX_BEHIND="$SECONDS_BEHIND"
+		fi
+		if [ "$SECONDS_BEHIND" -gt 30 ]; then
+			echo "LAGGING_SYNC_${CONTAINER_NAME}_${HOSTNAME}?FAIL#${SECONDS_BEHIND} behind"
+			
+		fi
+	done
+	echo $CURRENT_UNIXTIME
+) > "$TMP"
+chmod 655 "$TMP"
+mkdir -p /var/www/html/monitoring &>/dev/null
+mv "$TMP" /var/www/html/monitoring/synccontainer.check.txt
+
+
--- a/script/host/zfs/sync/synccontainer-receiver.sh
+++ b/script/host/zfs/sync/synccontainer-receiver.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+CONTAINER=${1:?"CONTAINER missing"}
+CONTAINER=$(echo $1 | sed -E 's|[^a-zA-Z0-9_-]*||g')
+(
+	flock -n 9 || exit 1
+
+	BACKUPHOST=$(hostname)
+	HOSTOWNER=$(cat /invra/hostowner)
+	SOURCEHOST=$(cat /invra/state/${HOSTOWNER}/containers/${CONTAINER}/current-host)
+
+	MOUNTPOINT="none"
+	DATASET="zpool1/persistent/${CONTAINER}-BACKUP"
+	SNAPSHOT_PREFIX="${DATASET}@SYNC_${BACKUPHOST}_"
+
+	LAST_SNAPSHOT_NAME=""
+	RESUME_TOKEN=""
+	zfs list -Hr -o name -s name "${DATASET}" | grep -E "^${DATASET}$" > /dev/null
+	if [ $? -eq 0 ]; then
+		LAST_SNAPSHOT_NAME=$(zfs list -H -o name -S name -t snapshot -r "${DATASET}" | grep -E "^${SNAPSHOT_PREFIX}" | head -n 1)
+		RESUME_TOKEN="$(zfs get -o value -H receive_resume_token "${DATASET}")"
+	fi
+
+	if [[ "x$RESUME_TOKEN" != "x" && "x$RESUME_TOKEN" != "x-" ]]; then
+		echo "Resume token present trying to resume at $RESUME_TOKEN"
+		LAST_SNAPSHOT_NAME="RESUME"
+	fi
+	
+	if [[ "x${LAST_SNAPSHOT_NAME}" != "x" && "${LAST_SNAPSHOT_NAME}" != "RESUME" ]]; then
+		zfs rollback -r "${LAST_SNAPSHOT_NAME}"
+	fi
+
+	# Beiim zfs receive in der nächsten Zeile fehlt noch das "-s" für resumable streams. Der tzrlxsrv kann das aber momentan nicht. Fehlermeldung: cannot receive resume stream: kernel modules must be upgraded to receive this stream.
+	(while sleep 1; do echo; done) | ssh -o ConnectTimeout=20 -C invencom@${SOURCEHOST} "sudo /invra/scripts/hosts/zfs/synccontainer-sender.sh \"${BACKUPHOST}\" \"${CONTAINER}\" \"${LAST_SNAPSHOT_NAME#$SNAPSHOT_PREFIX}\"" \"${RESUME_TOKEN}\" | zfs receive -v "${DATASET}"
+	if [ $? -ne 0 ]; then
+		exit 1
+	fi
+
+	# Dataset gegen Veränderungen sichern
+	zfs set readonly=on "${DATASET}"
+	zfs set "mountpoint=${MOUNTPOINT}" "${DATASET}"
+
+	# Aufsetzpunkte fremder Synchronisierer wegräumen
+	zfs list -t snapshot -o name -r "${DATASET}" | grep -- "${DATASET}@SYNC" | grep -v -i "_${BACKUPHOST}_" | while read SNAP; do
+		echo "Destroying $SNAP"
+		zfs destroy $SNAP
+	done
+
+	# Alte Snapshots wegräumen
+	while read -r ZEILE
+	do
+		if [ "$ZEILE" = "" ]; then
+			break
+		fi
+		if [[ "$ZEILE" > "$LAST_SNAPSHOT_NAME"  ]]; then
+			break
+		fi
+		zfs destroy "$ZEILE"
+	done < <(zfs list -Hr -o name -s name -t snapshot "${DATASET}" | grep -E "^${SNAPSHOT_PREFIX}")
+) 9>>/tmp/synccontainer.${CONTAINER}.lock
+
+if [ $? -ne 0 ]; then
+	exit 1
+fi
+exit 0
--- a/script/host/zfs/sync/synccontainer-sender.sh
+++ b/script/host/zfs/sync/synccontainer-sender.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+BACKUPHOST=${1:?"BACKUPHOST missing"}
+CONTAINER=${2:?"CONTAINER missing"}
+BACKUPHOST=$(echo $1 | sed -E 's|[^a-zA-Z0-9._-]*||g')
+CONTAINER=$(echo $2 | sed -E 's|[^a-zA-Z0-9_-]*||g')
+LAST_SNAPSHOT=$(echo $3 | sed -E 's|[^a-zA-Z0-9._:-]*||g')
+NEW_SNAPSHOT=$(date -u "+%Y-%m-%d_%H:%M:%S")
+
+if [[ "${LAST_SNAPSHOT}" == "RESUME" ]]; then
+	RESUME_TOKEN=$(echo $4 | sed -E 's|[^a-zA-Z0-9._:-]*||g')
+        zfs send -t "${RESUME_TOKEN}"
+        exit
+fi
+
+DATASET="zpool1/persistent/$CONTAINER"
+SNAPSHOT_PREFIX="${DATASET}@SYNC_${BACKUPHOST}_"
+LAST_SNAPSHOT_NAME="${SNAPSHOT_PREFIX}${LAST_SNAPSHOT}"
+NEW_SNAPSHOT_NAME="${SNAPSHOT_PREFIX}${NEW_SNAPSHOT}"
+SNAPSHOT_FOUND=""
+
+# Existiert der Snapshot?
+while read -r ZEILE
+do
+	if [[ "$ZEILE" == "$LAST_SNAPSHOT_NAME" ]]; then
+		SNAPSHOT_FOUND="1"
+		continue
+	fi
+done < <(zfs list -H -o name -s name -t snapshot "${DATASET}" | grep -E "^${SNAPSHOT_PREFIX}")
+
+# Falls ja, alle anderen Snapshots wegräumen - eine frühere Version des Skripts hat hier nur die Älteren weggeräumt. Das führt allerdings zum Vollmüllen
+# mit neueren Snapshots, wenn der Sync immer wieder fehlschlägt - im Einzelfall bis zur Unbenutzbarkeit des Senders
+if [[ "${SNAPSHOT_FOUND}x" == "1x" ]]; then
+	while read -r ZEILE
+	do
+	        if [[ "$ZEILE" == "$LAST_SNAPSHOT_NAME" ]]; then
+        	        continue
+	        fi
+        	zfs destroy "$ZEILE"
+	done < <(zfs list -H -o name -s name -t snapshot "${DATASET}" | grep -E "^${SNAPSHOT_PREFIX}")
+fi
+
+zfs snapshot "$NEW_SNAPSHOT_NAME"
+
+if [[ "$LAST_SNAPSHOT" != "" ]]; then
+	if [[ "$SNAPSHOT_FOUND" == "" ]]; then
+		echo "Angeforderter Snapshot '${LAST_SNAPSHOT}' nicht vorhanden"
+		exit 1;
+	fi
+	zfs send -I "${LAST_SNAPSHOT_NAME}" "${NEW_SNAPSHOT_NAME}"
+else
+	zfs send "${NEW_SNAPSHOT_NAME}"
+fi
--- a/script/host/zfs/sync/synccontainer.sh
+++ b/script/host/zfs/sync/synccontainer.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+BACKUPHOST=$(hostname)
+CONTAINER=${1:?"Kein Container angegeben"}
+DATASET="zpool1/persistent/$CONTAINER"
+SNAPSHOT_PREFIX="${DATASET}@SYNC_${BACKUPHOST}_"
+
+while true; do
+
+	/invra/scripts/hosts/zfs/synccontainer-receiver.sh "$CONTAINER"
+	sleep 5
+
+#	LAST_SNAPSHOT_NAME=$(zfs list -Hr -o name -S name -t snapshot "${DATASET}" | grep -E "^${SNAPSHOT_PREFIX}" | head -n 1)
+#	LAST_SNAPSHOT_TIME=${LAST_SNAPSHOT_NAME#${SNAPSHOT_PREFIX}}
+#	LAST_SNAPSHOT_TIME="$(echo "${LAST_SNAPSHOT_TIME}" | sed "s/_/ /g")"
+#	LAST_SNAPSHOT_UNIXTIME=$(date -u --date="TZ=\"UTC\" ${LAST_SNAPSHOT_TIME}" +%s)
+#	CURRENT_UNIXTIME=$(date -u +%s)
+#	SECONDS_BEHIND=$[ $CURRENT_UNIXTIME - $LAST_SNAPSHOT_UNIXTIME ]
+#	mkdir -p /var/www/html/monitoring > /dev/null 2>&1
+#	echo $CURRENT_UNIXTIME > "/var/www/html/monitoring/containersync.${CONTAINER}"
+#	echo "OK: $SECONDS_BEHIND seconds behind" >> "/var/www/html/monitoring/containersync.${CONTAINER}"
+
+done
+