245 lines
10 KiB
Bash
Executable File
245 lines
10 KiB
Bash
Executable File
#!/bin/bash
|
|
set -euo pipefail
|
|
|
|
# ================================================================
|
|
# Backup script for mrt0rtikize.ru Matrix instance (k3s cluster)
|
|
# ================================================================
|
|
# Output: backups/mrt0rtikize-YYYYMMDD-HHMMSS/
|
|
#
|
|
# Run this BEFORE switching DNS. Ensure TTL is already set to 60s
|
|
# on all mrt0rtikize.ru DNS records (24h before planned cutover).
|
|
#
|
|
# Steps:
|
|
# 1. Stop Synapse + MAS to prevent DB writes
|
|
# 2. Dump PostgreSQL (built-in PG from ESS chart)
|
|
# 3. Export generated secrets (CRITICAL: signing key, MAS keys)
|
|
# 4. Export deployment markers ConfigMap
|
|
# 5. Export ESS ArgoCD Application (values reference)
|
|
# 6. Export media file locations (manual restore, noted in README)
|
|
# ================================================================
|
|
|
|
readonly K3S_KUBECONFIG="${KUBECONFIG:-/home/mrt0rtikize/infra/k3s/config}"
|
|
readonly NS="matrix-mrt0rtikize"
|
|
readonly BACKUP_BASE="$(dirname "$(realpath "$0")")/../backups"
|
|
readonly TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
|
|
readonly BACKUP_DIR="${BACKUP_BASE}/${NS}-${TIMESTAMP}"
|
|
|
|
readonly K="${KUBECTL:-kubectl} --kubeconfig ${K3S_KUBECONFIG}"
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m'
|
|
|
|
log() { echo -e "${GREEN}[$(date +%H:%M:%S)]${NC} $*"; }
|
|
warn() { echo -e "${YELLOW}[$(date +%H:%M:%S)] WARN:${NC} $*"; }
|
|
err() { echo -e "${RED}[$(date +%H:%M:%S)] ERROR:${NC} $*"; }
|
|
|
|
# -------------------------------------------------------------------
|
|
# Prerequisites
|
|
# -------------------------------------------------------------------
|
|
log "=== Checking prerequisites ==="
|
|
|
|
if ! ${K} get ns "${NS}" >/dev/null 2>&1; then
|
|
err "Namespace ${NS} not found on k3s cluster. Check KUBECONFIG (${K3S_KUBECONFIG})."
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "${BACKUP_DIR}"
|
|
log "Backup directory: ${BACKUP_DIR}"
|
|
|
|
# -------------------------------------------------------------------
|
|
# Step 1: Backup Synapse media files (BEFORE stopping Synapse)
|
|
# -------------------------------------------------------------------
|
|
log "=== Step 1: Backing up Synapse media ==="
|
|
|
|
SYNAPSE_POD_NAME=$( ${K} -n "${NS}" get pods -l "app.kubernetes.io/component=matrix-server" -o jsonpath='{.items[?(@.status.phase=="Running")].metadata.name}' 2>/dev/null)
|
|
|
|
if [ -z "${SYNAPSE_POD_NAME}" ]; then
|
|
warn "No running Synapse pod found. Cannot backup media."
|
|
else
|
|
log "Using Synapse pod: ${SYNAPSE_POD_NAME}"
|
|
${K} exec -n "${NS}" "${SYNAPSE_POD_NAME}" -- tar czf /tmp/synapse-media.tar.gz -C /media media_store/
|
|
${K} cp "${NS}/${SYNAPSE_POD_NAME}:/tmp/synapse-media.tar.gz" "${BACKUP_DIR}/synapse-media.tar.gz"
|
|
${K} exec -n "${NS}" "${SYNAPSE_POD_NAME}" -- rm /tmp/synapse-media.tar.gz
|
|
log "Synapse media saved: ${BACKUP_DIR}/synapse-media.tar.gz ($(du -h "${BACKUP_DIR}/synapse-media.tar.gz" | cut -f1))"
|
|
fi
|
|
|
|
# -------------------------------------------------------------------
|
|
# Step 2: Stop Synapse + MAS (start downtime window)
|
|
# -------------------------------------------------------------------
|
|
log "=== Step 2: Stopping Synapse + MAS ==="
|
|
|
|
SYNAPSE_READY=$(${K} -n "${NS}" get sts "${NS}-synapse-main" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
|
|
MAS_READY=$(${K} -n "${NS}" get deploy "${NS}-matrix-authentication-service" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
|
|
|
|
if [ "${SYNAPSE_READY}" != "0" ]; then
|
|
log "Scaling Synapse to 0..."
|
|
${K} -n "${NS}" scale sts "${NS}-synapse-main" --replicas=0
|
|
else
|
|
log "Synapse already scaled to 0."
|
|
fi
|
|
|
|
if [ "${MAS_READY}" != "0" ]; then
|
|
log "Scaling MAS to 0..."
|
|
${K} -n "${NS}" scale deploy "${NS}-matrix-authentication-service" --replicas=0
|
|
else
|
|
log "MAS already scaled to 0."
|
|
fi
|
|
|
|
log "Waiting for Synapse + MAS pods to terminate..."
|
|
${K} -n "${NS}" wait --for=delete pod -l "app.kubernetes.io/component=matrix-server" --timeout=120s 2>/dev/null || warn "Some Synapse pods may still be terminating."
|
|
${K} -n "${NS}" wait --for=delete pod -l "app.kubernetes.io/component=matrix-authentication" --timeout=120s 2>/dev/null || warn "Some MAS pods may still be terminating."
|
|
|
|
log "Synapse + MAS stopped."
|
|
|
|
# -------------------------------------------------------------------
|
|
# Step 3: Dump PostgreSQL
|
|
# -------------------------------------------------------------------
|
|
log "=== Step 3: Dumping PostgreSQL ==="
|
|
|
|
PG_POD=$(${K} -n "${NS}" get pods -l "app.kubernetes.io/name=postgres" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
|
|
if [ -z "${PG_POD}" ]; then
|
|
PG_POD=$(${K} -n "${NS}" get pods -o name 2>/dev/null | grep postgres | head -1 | cut -d/ -f2)
|
|
fi
|
|
|
|
if [ -z "${PG_POD}" ]; then
|
|
err "Could not find PostgreSQL pod in namespace ${NS}."
|
|
err "Available pods:"
|
|
${K} -n "${NS}" get pods
|
|
exit 1
|
|
fi
|
|
|
|
log "Using PostgreSQL pod: ${PG_POD}"
|
|
|
|
${K} exec -n "${NS}" "${PG_POD}" -- pg_dumpall -U postgres > "${BACKUP_DIR}/dump-all.sql"
|
|
log "PostgreSQL dump saved: ${BACKUP_DIR}/dump-all.sql ($(wc -c < "${BACKUP_DIR}/dump-all.sql") bytes)"
|
|
|
|
${K} exec -n "${NS}" "${PG_POD}" -- pg_dump -U postgres -d synapse > "${BACKUP_DIR}/dump-synapse.sql" 2>/dev/null || \
|
|
warn "Could not dump synapse DB individually (will use dump-all.sql for restore)."
|
|
|
|
${K} exec -n "${NS}" "${PG_POD}" -- pg_dump -U postgres -d matrixauthenticationservice > "${BACKUP_DIR}/dump-mas.sql" 2>/dev/null || \
|
|
warn "Could not dump MAS DB individually (will use dump-all.sql for restore)."
|
|
|
|
# -------------------------------------------------------------------
|
|
# Step 4: Export generated secrets (CRITICAL)
|
|
# -------------------------------------------------------------------
|
|
log "=== Step 4: Exporting generated secrets ==="
|
|
|
|
if ${K} -n "${NS}" get secret "${NS}-generated" >/dev/null 2>&1; then
|
|
${K} -n "${NS}" get secret "${NS}-generated" -o yaml > "${BACKUP_DIR}/secret-generated.yaml"
|
|
log "Generated secret saved: ${BACKUP_DIR}/secret-generated.yaml"
|
|
else
|
|
err "CRITICAL: ${NS}-generated secret NOT FOUND!"
|
|
err "This contains SYNAPSE_SIGNING_KEY, MAS keys, and MACAROON."
|
|
err "Without it, federation identity is lost and all rooms break."
|
|
err "Available secrets:"
|
|
${K} -n "${NS}" get secrets
|
|
exit 1
|
|
fi
|
|
|
|
log "Secret contents (verify these exist):"
|
|
${K} -n "${NS}" get secret "${NS}-generated" -o jsonpath='{.data}' | python3 -c "
|
|
import json, sys
|
|
keys = ['SYNAPSE_SIGNING_KEY', 'MAS_ENCRYPTION_SECRET', 'MAS_RSA_PRIVATE_KEY',
|
|
'SYNAPSE_MACAROON', 'MAS_SYNAPSE_SHARED_SECRET',
|
|
'POSTGRES_SYNAPSE_PASSWORD', 'POSTGRES_MATRIX_AUTHENTICATION_SERVICE_PASSWORD']
|
|
d = json.load(sys.stdin)
|
|
for k in keys:
|
|
present = 'OK' if k in d else 'MISSING!'
|
|
print(f' {k}: {present}')
|
|
" 2>/dev/null || warn "Could not verify secret keys."
|
|
|
|
# -------------------------------------------------------------------
|
|
# Step 5: Export deployment markers
|
|
# -------------------------------------------------------------------
|
|
log "=== Step 5: Exporting deployment markers ==="
|
|
|
|
MARKER_CM=$(${K} -n "${NS}" get cm -l "app.kubernetes.io/managed-by=matrix-tools-deployment-markers" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
|
|
if [ -n "${MARKER_CM}" ]; then
|
|
${K} -n "${NS}" get cm "${MARKER_CM}" -o yaml > "${BACKUP_DIR}/markers.yaml"
|
|
log "Deployment markers saved: ${BACKUP_DIR}/markers.yaml"
|
|
else
|
|
warn "No deployment markers ConfigMap found (non-critical, ESS may regenerate)."
|
|
fi
|
|
|
|
# -------------------------------------------------------------------
|
|
# Step 6: Export ESS values from ArgoCD
|
|
# -------------------------------------------------------------------
|
|
log "=== Step 6: Exporting ESS ArgoCD application ==="
|
|
|
|
if ${K} -n argocd get application "${NS}" >/dev/null 2>&1; then
|
|
${K} -n argocd get application "${NS}" -o yaml > "${BACKUP_DIR}/argo-app.yaml"
|
|
log "ArgoCD Application saved: ${BACKUP_DIR}/argo-app.yaml"
|
|
else
|
|
warn "ArgoCD Application '${NS}' not found (running without ArgoCD?)."
|
|
warn "Save your ESS values manually from helm get values or git."
|
|
fi
|
|
|
|
# -------------------------------------------------------------------
|
|
# Write README
|
|
# -------------------------------------------------------------------
|
|
log "=== Writing README ==="
|
|
|
|
cat > "${BACKUP_DIR}/README.txt" << READEOF
|
|
Backup for ${NS} — ${TIMESTAMP}
|
|
==================================
|
|
|
|
Files:
|
|
dump-all.sql Full PostgreSQL dump (pg_dumpall)
|
|
secret-generated.yaml CRITICAL: contains SYNAPSE_SIGNING_KEY, MAS keys, MACAROON
|
|
markers.yaml Deployment markers ConfigMap (ESS state tracking)
|
|
argo-app.yaml ESS ArgoCD Application (values reference)
|
|
synapse-media.tar.gz Synapse media files (local + remote content)
|
|
dump-synapse.sql Synapse DB only (optional, for easier restore)
|
|
dump-mas.sql MAS DB only (optional, for easier restore)
|
|
README.txt This file
|
|
|
|
CRITICAL: Do NOT lose secret-generated.yaml.
|
|
- SYNAPSE_SIGNING_KEY identifies this server to the Matrix federation.
|
|
Changing it breaks all existing rooms and federation relationships.
|
|
- MAS_ENCRYPTION_SECRET encrypts user sessions.
|
|
Changing it forces all users to re-login.
|
|
- SYNAPSE_MACAROON is the admin API token.
|
|
|
|
Restore order on new cluster:
|
|
1. Create CNPG databases + secrets on new cluster (see PLAN.md)
|
|
2. Deploy ESS chart on new cluster (starts with empty DB)
|
|
3. Stop Synapse + MAS on new cluster
|
|
4. Restore PG dump into CNPG shared-pg
|
|
5. Apply this secret-generated.yaml to new cluster's namespace
|
|
6. Apply markers.yaml
|
|
7. Restore media: kubectl cp synapse-media.tar.gz to new Synapse pod, untar to /media/
|
|
8. Restart Synapse + MAS on new cluster
|
|
9. Verify: login, federation tester, Element Call
|
|
10. Cut DNS to new NLB IP
|
|
READEOF
|
|
|
|
log "README saved: ${BACKUP_DIR}/README.txt"
|
|
|
|
# -------------------------------------------------------------------
|
|
# Summary
|
|
# -------------------------------------------------------------------
|
|
cat <<SUMMARY
|
|
|
|
${GREEN}========================================${NC}
|
|
${GREEN} Backup Complete: ${NS}${NC}
|
|
${GREEN}========================================${NC}
|
|
|
|
Directory: ${BACKUP_DIR}
|
|
Size: $(du -sh "${BACKUP_DIR}" | cut -f1)
|
|
|
|
${YELLOW}Next steps:${NC}
|
|
1. Verify PG dump: head -20 ${BACKUP_DIR}/dump-all.sql
|
|
2. Verify secrets: grep SYNAPSE_SIGNING_KEY ${BACKUP_DIR}/secret-generated.yaml
|
|
3. Copy media files from node path (see media-info.txt)
|
|
4. Proceed with migration on new cluster
|
|
|
|
${RED}WARNING: Synapse + MAS are still SCALED TO 0 on k3s.${NC}
|
|
To restore service on k3s (if migration is postponed):
|
|
${K} -n ${NS} scale sts ${NS}-synapse-main --replicas=1
|
|
${K} -n ${NS} scale deploy ${NS}-matrix-authentication-service --replicas=1
|
|
|
|
SUMMARY
|