diff --git a/script/zfs/sync/synccontainer-all.sh b/script/zfs/sync/synccontainer-all.sh new file mode 100644 index 0000000..bb0b758 --- /dev/null +++ b/script/zfs/sync/synccontainer-all.sh @@ -0,0 +1,27 @@ +#!/bin/bash +HOSTOWNER=$(cat /invra/hostowner) +BACKUPHOST=$(hostname) +STATE_DIR=/invra/state/${HOSTOWNER}/containers/; + +screen -ls | grep -oE "[0-9]+\.synccontainer\.[a-zA-Z0-9_-]+" | while read -r SCREEN_SESSION; do + CONTAINER=$(echo "$SCREEN_SESSION" | grep -oE "[^.]+$") + PID=$(echo "$SCREEN_SESSION" | grep -oE "^[0-9]+") + grep -iE "^${BACKUPHOST}$" ${STATE_DIR}/${CONTAINER}/standby-hosts > /dev/null + if [ $? -ne 0 ]; then + echo "quit screen session ${SCREEN_SESSION}" + screen -XS "$PID" quit + fi +done + +grep -lrE "^${BACKUPHOST}$" /invra/state/${HOSTOWNER}/containers/*/standby-hosts > /dev/null +if [ $? -eq 0 ]; then + grep -lrE "^${BACKUPHOST}$" /invra/state/${HOSTOWNER}/containers/*/standby-hosts | while read -r STANDBY_FILE; do + CONTAINER=$(basename $(dirname ${STANDBY_FILE})) + screen -ls | grep -oE "[0-9]+\.synccontainer\.$CONTAINER" > /dev/null + if [ $? -ne 0 ]; then + echo "starte container sync" + screen -dmS "synccontainer.$CONTAINER" /invra/scripts/hosts/zfs/synccontainer.sh "$CONTAINER" + fi + done +fi + diff --git a/script/zfs/sync/synccontainer-check.sh b/script/zfs/sync/synccontainer-check.sh new file mode 100644 index 0000000..2e34a60 --- /dev/null +++ b/script/zfs/sync/synccontainer-check.sh @@ -0,0 +1,30 @@ +#!/bin/bash +TMP="$(mktemp)" +( + HOSTNAME="$(hostname)" + HOSTOWNER="$(cat /invra/hostowner)" + MAX_BEHIND=0 + CURRENT_UNIXTIME=$(date -u +%s) + echo "OK#Checks running" + for CONTAINER_PATH in /invra/state/${HOSTOWNER}/containers/*; do + grep -E "^${HOSTNAME}$" "${CONTAINER_PATH}/standby-hosts" &> /dev/null || continue; + CONTAINER_NAME="$(basename "$CONTAINER_PATH")"; + TS=$(zfs list -o name -r -t snapshot "zpool1/persistent/${CONTAINER_NAME}-BACKUP" | grep "@SYNC_${HOSTNAME}" | head -n1 | grep -oP "\\d{4}-\\d{2}-\\d{2}_\\d{2}:\\d{2}:\\d{2}") + LAST_SNAPSHOT_TIME="$(echo "${TS}" | sed "s/_/ /g")" + LAST_SNAPSHOT_UNIXTIME=$(date -u --date="TZ=\"UTC\" ${LAST_SNAPSHOT_TIME}" +%s) + SECONDS_BEHIND=$[ $CURRENT_UNIXTIME - $LAST_SNAPSHOT_UNIXTIME ] + if [ "$SECONDS_BEHIND" -gt "$MAX_BEHIND" ]; then + MAX_BEHIND="$SECONDS_BEHIND" + fi + if [ "$SECONDS_BEHIND" -gt 30 ]; then + echo "LAGGING_SYNC_${CONTAINER_NAME}_${HOSTNAME}?FAIL#${SECONDS_BEHIND} behind" + + fi + done + echo $CURRENT_UNIXTIME +) > "$TMP" +chmod 655 "$TMP" +mkdir -p /var/www/html/monitoring &>/dev/null +mv "$TMP" /var/www/html/monitoring/synccontainer.check.txt + + diff --git a/script/zfs/sync/synccontainer-receiver.sh b/script/zfs/sync/synccontainer-receiver.sh new file mode 100644 index 0000000..f8ed3e8 --- /dev/null +++ b/script/zfs/sync/synccontainer-receiver.sh @@ -0,0 +1,64 @@ +#!/bin/bash +CONTAINER=${1:?"CONTAINER missing"} +CONTAINER=$(echo $1 | sed -E 's|[^a-zA-Z0-9_-]*||g') +( + flock -n 9 || exit 1 + + BACKUPHOST=$(hostname) + HOSTOWNER=$(cat /invra/hostowner) + SOURCEHOST=$(cat /invra/state/${HOSTOWNER}/containers/${CONTAINER}/current-host) + + MOUNTPOINT="none" + DATASET="zpool1/persistent/${CONTAINER}-BACKUP" + SNAPSHOT_PREFIX="${DATASET}@SYNC_${BACKUPHOST}_" + + LAST_SNAPSHOT_NAME="" + RESUME_TOKEN="" + zfs list -Hr -o name -s name "${DATASET}" | grep -E "^${DATASET}$" > /dev/null + if [ $? -eq 0 ]; then + LAST_SNAPSHOT_NAME=$(zfs list -H -o name -S name -t snapshot -r "${DATASET}" | grep -E "^${SNAPSHOT_PREFIX}" | head -n 1) + RESUME_TOKEN="$(zfs get -o value -H receive_resume_token "${DATASET}")" + fi + + if [[ "x$RESUME_TOKEN" != "x" && "x$RESUME_TOKEN" != "x-" ]]; then + echo "Resume token present trying to resume at $RESUME_TOKEN" + LAST_SNAPSHOT_NAME="RESUME" + fi + + if [[ "x${LAST_SNAPSHOT_NAME}" != "x" && "${LAST_SNAPSHOT_NAME}" != "RESUME" ]]; then + zfs rollback -r "${LAST_SNAPSHOT_NAME}" + fi + + # Beiim zfs receive in der nächsten Zeile fehlt noch das "-s" für resumable streams. Der tzrlxsrv kann das aber momentan nicht. Fehlermeldung: cannot receive resume stream: kernel modules must be upgraded to receive this stream. + (while sleep 1; do echo; done) | ssh -o ConnectTimeout=20 -C invencom@${SOURCEHOST} "sudo /invra/scripts/hosts/zfs/synccontainer-sender.sh \"${BACKUPHOST}\" \"${CONTAINER}\" \"${LAST_SNAPSHOT_NAME#$SNAPSHOT_PREFIX}\"" \"${RESUME_TOKEN}\" | zfs receive -v "${DATASET}" + if [ $? -ne 0 ]; then + exit 1 + fi + + # Dataset gegen Veränderungen sichern + zfs set readonly=on "${DATASET}" + zfs set "mountpoint=${MOUNTPOINT}" "${DATASET}" + + # Aufsetzpunkte fremder Synchronisierer wegräumen + zfs list -t snapshot -o name -r "${DATASET}" | grep -- "${DATASET}@SYNC" | grep -v -i "_${BACKUPHOST}_" | while read SNAP; do + echo "Destroying $SNAP" + zfs destroy $SNAP + done + + # Alte Snapshots wegräumen + while read -r ZEILE + do + if [ "$ZEILE" = "" ]; then + break + fi + if [[ "$ZEILE" > "$LAST_SNAPSHOT_NAME" ]]; then + break + fi + zfs destroy "$ZEILE" + done < <(zfs list -Hr -o name -s name -t snapshot "${DATASET}" | grep -E "^${SNAPSHOT_PREFIX}") +) 9>>/tmp/synccontainer.${CONTAINER}.lock + +if [ $? -ne 0 ]; then + exit 1 +fi +exit 0 diff --git a/script/zfs/sync/synccontainer-sender.sh b/script/zfs/sync/synccontainer-sender.sh new file mode 100644 index 0000000..dae11de --- /dev/null +++ b/script/zfs/sync/synccontainer-sender.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +BACKUPHOST=${1:?"BACKUPHOST missing"} +CONTAINER=${2:?"CONTAINER missing"} +BACKUPHOST=$(echo $1 | sed -E 's|[^a-zA-Z0-9._-]*||g') +CONTAINER=$(echo $2 | sed -E 's|[^a-zA-Z0-9_-]*||g') +LAST_SNAPSHOT=$(echo $3 | sed -E 's|[^a-zA-Z0-9._:-]*||g') +NEW_SNAPSHOT=$(date -u "+%Y-%m-%d_%H:%M:%S") + +if [[ "${LAST_SNAPSHOT}" == "RESUME" ]]; then + RESUME_TOKEN=$(echo $4 | sed -E 's|[^a-zA-Z0-9._:-]*||g') + zfs send -t "${RESUME_TOKEN}" + exit +fi + +DATASET="zpool1/persistent/$CONTAINER" +SNAPSHOT_PREFIX="${DATASET}@SYNC_${BACKUPHOST}_" +LAST_SNAPSHOT_NAME="${SNAPSHOT_PREFIX}${LAST_SNAPSHOT}" +NEW_SNAPSHOT_NAME="${SNAPSHOT_PREFIX}${NEW_SNAPSHOT}" +SNAPSHOT_FOUND="" + +# Existiert der Snapshot? +while read -r ZEILE +do + if [[ "$ZEILE" == "$LAST_SNAPSHOT_NAME" ]]; then + SNAPSHOT_FOUND="1" + continue + fi +done < <(zfs list -H -o name -s name -t snapshot "${DATASET}" | grep -E "^${SNAPSHOT_PREFIX}") + +# Falls ja, alle anderen Snapshots wegräumen - eine frühere Version des Skripts hat hier nur die Älteren weggeräumt. Das führt allerdings zum Vollmüllen +# mit neueren Snapshots, wenn der Sync immer wieder fehlschlägt - im Einzelfall bis zur Unbenutzbarkeit des Senders +if [[ "${SNAPSHOT_FOUND}x" == "1x" ]]; then + while read -r ZEILE + do + if [[ "$ZEILE" == "$LAST_SNAPSHOT_NAME" ]]; then + continue + fi + zfs destroy "$ZEILE" + done < <(zfs list -H -o name -s name -t snapshot "${DATASET}" | grep -E "^${SNAPSHOT_PREFIX}") +fi + +zfs snapshot "$NEW_SNAPSHOT_NAME" + +if [[ "$LAST_SNAPSHOT" != "" ]]; then + if [[ "$SNAPSHOT_FOUND" == "" ]]; then + echo "Angeforderter Snapshot '${LAST_SNAPSHOT}' nicht vorhanden" + exit 1; + fi + zfs send -I "${LAST_SNAPSHOT_NAME}" "${NEW_SNAPSHOT_NAME}" +else + zfs send "${NEW_SNAPSHOT_NAME}" +fi diff --git a/script/zfs/sync/synccontainer.sh b/script/zfs/sync/synccontainer.sh new file mode 100644 index 0000000..556ca68 --- /dev/null +++ b/script/zfs/sync/synccontainer.sh @@ -0,0 +1,23 @@ +#!/bin/bash +BACKUPHOST=$(hostname) +CONTAINER=${1:?"Kein Container angegeben"} +DATASET="zpool1/persistent/$CONTAINER" +SNAPSHOT_PREFIX="${DATASET}@SYNC_${BACKUPHOST}_" + +while true; do + + /invra/scripts/hosts/zfs/synccontainer-receiver.sh "$CONTAINER" + sleep 5 + +# LAST_SNAPSHOT_NAME=$(zfs list -Hr -o name -S name -t snapshot "${DATASET}" | grep -E "^${SNAPSHOT_PREFIX}" | head -n 1) +# LAST_SNAPSHOT_TIME=${LAST_SNAPSHOT_NAME#${SNAPSHOT_PREFIX}} +# LAST_SNAPSHOT_TIME="$(echo "${LAST_SNAPSHOT_TIME}" | sed "s/_/ /g")" +# LAST_SNAPSHOT_UNIXTIME=$(date -u --date="TZ=\"UTC\" ${LAST_SNAPSHOT_TIME}" +%s) +# CURRENT_UNIXTIME=$(date -u +%s) +# SECONDS_BEHIND=$[ $CURRENT_UNIXTIME - $LAST_SNAPSHOT_UNIXTIME ] +# mkdir -p /var/www/html/monitoring > /dev/null 2>&1 +# echo $CURRENT_UNIXTIME > "/var/www/html/monitoring/containersync.${CONTAINER}" +# echo "OK: $SECONDS_BEHIND seconds behind" >> "/var/www/html/monitoring/containersync.${CONTAINER}" + +done +