#!/bin/bash set -euo pipefail # ================================================================ # Backup script for mrt0rtikize.ru Matrix instance (k3s cluster) # ================================================================ # Output: backups/mrt0rtikize-YYYYMMDD-HHMMSS/ # # Run this BEFORE switching DNS. Ensure TTL is already set to 60s # on all mrt0rtikize.ru DNS records (24h before planned cutover). # # Steps: # 1. Stop Synapse + MAS to prevent DB writes # 2. Dump PostgreSQL (built-in PG from ESS chart) # 3. Export generated secrets (CRITICAL: signing key, MAS keys) # 4. Export deployment markers ConfigMap # 5. Export ESS ArgoCD Application (values reference) # 6. Export media file locations (manual restore, noted in README) # ================================================================ readonly K3S_KUBECONFIG="${KUBECONFIG:-/home/mrt0rtikize/infra/k3s/config}" readonly NS="matrix-mrt0rtikize" readonly BACKUP_BASE="$(dirname "$(realpath "$0")")/../backups" readonly TIMESTAMP="$(date +%Y%m%d-%H%M%S)" readonly BACKUP_DIR="${BACKUP_BASE}/${NS}-${TIMESTAMP}" readonly K="${KUBECTL:-kubectl} --kubeconfig ${K3S_KUBECONFIG}" # Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' log() { echo -e "${GREEN}[$(date +%H:%M:%S)]${NC} $*"; } warn() { echo -e "${YELLOW}[$(date +%H:%M:%S)] WARN:${NC} $*"; } err() { echo -e "${RED}[$(date +%H:%M:%S)] ERROR:${NC} $*"; } # ------------------------------------------------------------------- # Prerequisites # ------------------------------------------------------------------- log "=== Checking prerequisites ===" if ! ${K} get ns "${NS}" >/dev/null 2>&1; then err "Namespace ${NS} not found on k3s cluster. Check KUBECONFIG (${K3S_KUBECONFIG})." exit 1 fi mkdir -p "${BACKUP_DIR}" log "Backup directory: ${BACKUP_DIR}" # ------------------------------------------------------------------- # Step 1: Backup Synapse media files (BEFORE stopping Synapse) # ------------------------------------------------------------------- log "=== Step 1: Backing up Synapse media ===" SYNAPSE_POD_NAME=$( ${K} -n "${NS}" get pods -l "app.kubernetes.io/component=matrix-server" -o jsonpath='{.items[?(@.status.phase=="Running")].metadata.name}' 2>/dev/null) if [ -z "${SYNAPSE_POD_NAME}" ]; then warn "No running Synapse pod found. Cannot backup media." else log "Using Synapse pod: ${SYNAPSE_POD_NAME}" ${K} exec -n "${NS}" "${SYNAPSE_POD_NAME}" -- tar czf /tmp/synapse-media.tar.gz -C /media media_store/ ${K} cp "${NS}/${SYNAPSE_POD_NAME}:/tmp/synapse-media.tar.gz" "${BACKUP_DIR}/synapse-media.tar.gz" ${K} exec -n "${NS}" "${SYNAPSE_POD_NAME}" -- rm /tmp/synapse-media.tar.gz log "Synapse media saved: ${BACKUP_DIR}/synapse-media.tar.gz ($(du -h "${BACKUP_DIR}/synapse-media.tar.gz" | cut -f1))" fi # ------------------------------------------------------------------- # Step 2: Stop Synapse + MAS (start downtime window) # ------------------------------------------------------------------- log "=== Step 2: Stopping Synapse + MAS ===" SYNAPSE_READY=$(${K} -n "${NS}" get sts "${NS}-synapse-main" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0") MAS_READY=$(${K} -n "${NS}" get deploy "${NS}-matrix-authentication-service" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0") if [ "${SYNAPSE_READY}" != "0" ]; then log "Scaling Synapse to 0..." ${K} -n "${NS}" scale sts "${NS}-synapse-main" --replicas=0 else log "Synapse already scaled to 0." fi if [ "${MAS_READY}" != "0" ]; then log "Scaling MAS to 0..." ${K} -n "${NS}" scale deploy "${NS}-matrix-authentication-service" --replicas=0 else log "MAS already scaled to 0." fi log "Waiting for Synapse + MAS pods to terminate..." ${K} -n "${NS}" wait --for=delete pod -l "app.kubernetes.io/component=matrix-server" --timeout=120s 2>/dev/null || warn "Some Synapse pods may still be terminating." ${K} -n "${NS}" wait --for=delete pod -l "app.kubernetes.io/component=matrix-authentication" --timeout=120s 2>/dev/null || warn "Some MAS pods may still be terminating." log "Synapse + MAS stopped." # ------------------------------------------------------------------- # Step 3: Dump PostgreSQL # ------------------------------------------------------------------- log "=== Step 3: Dumping PostgreSQL ===" PG_POD=$(${K} -n "${NS}" get pods -l "app.kubernetes.io/name=postgres" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) if [ -z "${PG_POD}" ]; then PG_POD=$(${K} -n "${NS}" get pods -o name 2>/dev/null | grep postgres | head -1 | cut -d/ -f2) fi if [ -z "${PG_POD}" ]; then err "Could not find PostgreSQL pod in namespace ${NS}." err "Available pods:" ${K} -n "${NS}" get pods exit 1 fi log "Using PostgreSQL pod: ${PG_POD}" ${K} exec -n "${NS}" "${PG_POD}" -- pg_dumpall -U postgres > "${BACKUP_DIR}/dump-all.sql" log "PostgreSQL dump saved: ${BACKUP_DIR}/dump-all.sql ($(wc -c < "${BACKUP_DIR}/dump-all.sql") bytes)" ${K} exec -n "${NS}" "${PG_POD}" -- pg_dump -U postgres -d synapse > "${BACKUP_DIR}/dump-synapse.sql" 2>/dev/null || \ warn "Could not dump synapse DB individually (will use dump-all.sql for restore)." ${K} exec -n "${NS}" "${PG_POD}" -- pg_dump -U postgres -d matrixauthenticationservice > "${BACKUP_DIR}/dump-mas.sql" 2>/dev/null || \ warn "Could not dump MAS DB individually (will use dump-all.sql for restore)." # ------------------------------------------------------------------- # Step 4: Export generated secrets (CRITICAL) # ------------------------------------------------------------------- log "=== Step 4: Exporting generated secrets ===" if ${K} -n "${NS}" get secret "${NS}-generated" >/dev/null 2>&1; then ${K} -n "${NS}" get secret "${NS}-generated" -o yaml > "${BACKUP_DIR}/secret-generated.yaml" log "Generated secret saved: ${BACKUP_DIR}/secret-generated.yaml" else err "CRITICAL: ${NS}-generated secret NOT FOUND!" err "This contains SYNAPSE_SIGNING_KEY, MAS keys, and MACAROON." err "Without it, federation identity is lost and all rooms break." err "Available secrets:" ${K} -n "${NS}" get secrets exit 1 fi log "Secret contents (verify these exist):" ${K} -n "${NS}" get secret "${NS}-generated" -o jsonpath='{.data}' | python3 -c " import json, sys keys = ['SYNAPSE_SIGNING_KEY', 'MAS_ENCRYPTION_SECRET', 'MAS_RSA_PRIVATE_KEY', 'SYNAPSE_MACAROON', 'MAS_SYNAPSE_SHARED_SECRET', 'POSTGRES_SYNAPSE_PASSWORD', 'POSTGRES_MATRIX_AUTHENTICATION_SERVICE_PASSWORD'] d = json.load(sys.stdin) for k in keys: present = 'OK' if k in d else 'MISSING!' print(f' {k}: {present}') " 2>/dev/null || warn "Could not verify secret keys." # ------------------------------------------------------------------- # Step 5: Export deployment markers # ------------------------------------------------------------------- log "=== Step 5: Exporting deployment markers ===" MARKER_CM=$(${K} -n "${NS}" get cm -l "app.kubernetes.io/managed-by=matrix-tools-deployment-markers" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) if [ -n "${MARKER_CM}" ]; then ${K} -n "${NS}" get cm "${MARKER_CM}" -o yaml > "${BACKUP_DIR}/markers.yaml" log "Deployment markers saved: ${BACKUP_DIR}/markers.yaml" else warn "No deployment markers ConfigMap found (non-critical, ESS may regenerate)." fi # ------------------------------------------------------------------- # Step 6: Export ESS values from ArgoCD # ------------------------------------------------------------------- log "=== Step 6: Exporting ESS ArgoCD application ===" if ${K} -n argocd get application "${NS}" >/dev/null 2>&1; then ${K} -n argocd get application "${NS}" -o yaml > "${BACKUP_DIR}/argo-app.yaml" log "ArgoCD Application saved: ${BACKUP_DIR}/argo-app.yaml" else warn "ArgoCD Application '${NS}' not found (running without ArgoCD?)." warn "Save your ESS values manually from helm get values or git." fi # ------------------------------------------------------------------- # Write README # ------------------------------------------------------------------- log "=== Writing README ===" cat > "${BACKUP_DIR}/README.txt" << READEOF Backup for ${NS} — ${TIMESTAMP} ================================== Files: dump-all.sql Full PostgreSQL dump (pg_dumpall) secret-generated.yaml CRITICAL: contains SYNAPSE_SIGNING_KEY, MAS keys, MACAROON markers.yaml Deployment markers ConfigMap (ESS state tracking) argo-app.yaml ESS ArgoCD Application (values reference) synapse-media.tar.gz Synapse media files (local + remote content) dump-synapse.sql Synapse DB only (optional, for easier restore) dump-mas.sql MAS DB only (optional, for easier restore) README.txt This file CRITICAL: Do NOT lose secret-generated.yaml. - SYNAPSE_SIGNING_KEY identifies this server to the Matrix federation. Changing it breaks all existing rooms and federation relationships. - MAS_ENCRYPTION_SECRET encrypts user sessions. Changing it forces all users to re-login. - SYNAPSE_MACAROON is the admin API token. Restore order on new cluster: 1. Create CNPG databases + secrets on new cluster (see PLAN.md) 2. Deploy ESS chart on new cluster (starts with empty DB) 3. Stop Synapse + MAS on new cluster 4. Restore PG dump into CNPG shared-pg 5. Apply this secret-generated.yaml to new cluster's namespace 6. Apply markers.yaml 7. Restore media: kubectl cp synapse-media.tar.gz to new Synapse pod, untar to /media/ 8. Restart Synapse + MAS on new cluster 9. Verify: login, federation tester, Element Call 10. Cut DNS to new NLB IP READEOF log "README saved: ${BACKUP_DIR}/README.txt" # ------------------------------------------------------------------- # Summary # ------------------------------------------------------------------- cat <