commit 84a19922ec9b920b726e23b6e57e7e933876bd6c Author: Alexander Rogov Date: Fri Jun 26 19:05:55 2026 +0300 initial: yandex-prod gitops infrastructure Gitea + ArgoCD + cert-manager + Traefik + CNPG + monitoring + loki + alloy Matrix homeservers for mrt0rtikize.ru, t0rt1k.tech, roglog.space diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..63127e9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +backups/ +kubeconfig diff --git a/BOOTSTRAP.md b/BOOTSTRAP.md new file mode 100644 index 0000000..7482b43 --- /dev/null +++ b/BOOTSTRAP.md @@ -0,0 +1,308 @@ +# Yandex Cloud Production Cluster — Bootstrap Guide + +## Prerequisites + +- [ ] `kubeconfig` file placed at `~/infra/yandex-prod/kubeconfig` +- [ ] `kubectl` context pointing to the new `yc-prod` cluster +- [ ] Domaster `prod.t01tt.tech` DNS managed (can be updated later in Phase 5) +- [ ] `git` and `helm` installed locally + +--- + +## Phase 0: Verify Cluster Access + +```bash +export KUBECONFIG=~/infra/yandex-prod/kubeconfig + +kubectl get nodes +# Expected: 3 nodes Ready, 2CPU/8GB each + +kubectl get sc +# Expected: yc-network-hdd (default), yc-network-ssd, yc-network-nvme, ... +``` + +--- + +## Phase 1: Bootstrap Gitea (internal access only) + +Gitea hosts the Git repo that ArgoCD reads. Deploy it first, but without ingress — we access it via port-forward. + +```bash +kubectl apply -f bootstrap/gitea/namespace.yaml +kubectl apply -f bootstrap/gitea/pvc.yaml +kubectl apply -f bootstrap/gitea/deployment.yaml +kubectl apply -f bootstrap/gitea/service.yaml +# NOTE: Do NOT apply ingress.yaml yet — no Traefik or cert-manager exists +``` + +Wait for Gitea to be ready, then port-forward and configure: + +```bash +kubectl wait deploy/gitea -n gitea --for=condition=available --timeout=120s + +# Port-forward in a separate terminal: +kubectl port-forward svc/gitea 3000:3000 -n gitea +``` + +1. Open **http://localhost:3000** in a browser +2. Fill out the install form: + - Database: **SQLite3** (default) + - Site Title: **Gitea** + - Domaster: **git.prod.t01tt.tech** + - Application URL: **https://git.prod.t01tt.tech** + - Create admin account (username/password/email — save these) +3. Click "Install Gitea" +4. Create a new repository: **`master`** (must be **public**, owned by admin) +5. Close the port-forward (Ctrl+C) + +--- + +## Phase 2: Push Repository to Gitea + +```bash +cd ~/infra/yandex-prod + +git init +git remote add origin http://localhost:3000/admin/master.git +# Or, once Gitea ingress works later, use: +# git remote add origin https://git.prod.t01tt.tech/admin/master.git + +git add -A +git commit -m "initial bootstrap: infrastructure manifests" +git push -u origin master +# Enter Gitea admin credentials when prompted +``` + +--- + +## Phase 3: Install ArgoCD (internal access only) + +```bash +bash bootstrap/argocd/install.sh +# Saves the admin password — copy it +``` + +Add the Gitea repository to ArgoCD: + +```bash +# Via port-forward: +kubectl port-forward svc/argocd-server 8080:80 -n argocd & +sleep 2 + +# Login and add repo: +ARGOCD_PASS=$(kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d) +argocd login localhost:8080 --username admin --password "${ARGOCD_PASS}" --insecure + +argocd repo add http://gitea.gitea.svc.cluster.local:3000/admin/master.git \ + --name yandex-prod \ + --type git +``` + +Deploy the root app: + +```bash +kubectl apply -f argocd/app-of-apps.yaml +``` + +ArgoCD will now sync child apps according to their sync waves. You can watch progress: + +```bash +argocd app list +``` + +--- + +## Phase 4: Let the Sync Waves Run + +Sync order (automated by ArgoCD via `argocd.argoproj.io/sync-wave` annotations): + +| Wave | App | What happens | +|------|-----|-------------| +| **-2** | `traefik` | DaemonSet deployed on all 3 nodes. NLB created → external IP provisioned | +| **-1** | `cert-manager` | cert-manager operator + CRDs installed | +| **0** | `cert-manager-issuers` | `letsencrypt-production` + `letsencrypt-staging` ClusterIssuers created | +| **0** | `monitoring` | VM k8s-stack (metrics) + Grafana ingress deployed | +| **0** | `loki` | Loki single-binary deployed | +| **0** | `cnpg-operator` | CloudNativePG operator installed | +| **1** | `cnpg-cluster` | `shared-pg` 3-node PostgreSQL cluster + 8 databases created | + +**Verify Traefik IP:** + +```bash +kubectl get svc traefik -n traefik -w +# Wait for EXTERNAL-IP to appear. Example output: +# traefik LoadBalancer 10.x.x.x 80:3xxxx/TCP,443:3xxxx/TCP 30s +# traefik LoadBalancer 10.x.x.x 158.160.x.x 80:3xxxx/TCP,443:3xxxx/TCP 60s +``` + +Take the EXTERNAL-IP — this is your NLB IP. You'll need it in Phase 5. + +**Verify state:** + +```bash +kubectl get pods -A +# Expected running pods: +# traefik: traefik-xxxxx (3 pods, DaemonSet) +# cert-manager: cert-manager-xxxxx, cert-manager-cainjector-xxxxx, cert-manager-webhook-xxxxx +# metrics: vm-k8s-stack-* pods (vmsingle, alertmanager, grafana, node-exporter, kube-state-metrics, vmagent) +# metrics: loki-0 +# cnpg-system: cnpg-operator-xxxxx +# cnpg: shared-pg-1, shared-pg-2, shared-pg-3 (may take a minute to start) + +kubectl get clusterissuer +# Expected: letsencrypt-production (True), letsencrypt-staging (True) + +kubectl get cluster -n cnpg +# Expected: shared-pg (3/3 instances ready) +``` + +--- + +## Phase 5: DNS + Expose Gitea & ArgoCD + +Now that Traefik has an external IP and cert-manager is running, we can: +1. Point DNS at the NLB IP +2. Create the Gitea and ArgoCD ingress resources (with TLS) + +### 5.1 Update DNS + +Point the following records to the Traefik NLB IP (from Phase 4): + +``` +git.prod.t01tt.tech → +argocd.prod.t01tt.tech → +grafana.prod.t01tt.tech → +``` + +Also create a wildcard for future hosts: +``` +*.prod.t01tt.tech → +``` + +### 5.2 Apply Ingresses + +```bash +kubectl apply -f bootstrap/gitea/ingress.yaml +kubectl apply -f bootstrap/argocd/ingress.yaml +``` + +### 5.3 Wait for TLS Certificates + +```bash +kubectl get certificate -A -w +# Wait for all to show Ready=True: +# gitea gitea-tls True +# argocd argocd-tls True +# metrics grafana-tls True +``` + +**Troubleshooting:** If certificates are stuck in `Pending`: +- Check DNS resolves: `dig git.prod.t01tt.tech` — must return the NLB IP +- Check cert-manager logs: `kubectl logs -n cert-manager deploy/cert-manager` +- Check challenge: `kubectl get challenges -A` + +--- + +## Phase 6: Verify Everything + +### Gitea +``` +https://git.prod.t01tt.tech +``` +Login with the admin credentials from Phase 1. Verify the `yandex-prod` repo exists. + +### ArgoCD +``` +https://argocd.prod.t01tt.tech +``` +Login with `admin` + password from Phase 3. All apps should show green (`Synced` + `Healthy`). + +The Ingress health may show `Healthy` immediately (by design — see `values.yaml` customization). + +### Grafana +``` +https://grafana.prod.t01tt.tech +``` +Login with `admin` / `change-me`. Check that VM k8s-stack dashboards are available. + +### PostgreSQL +```bash +kubectl get databases -n cnpg +# Expected: 8 Database resources, one per homeserver + +kubectl get pods -n cnpg +# Expected: shared-pg-1, shared-pg-2, shared-pg-3 (Running) +``` + +### ArgoCD Repo Connection +```bash +argocd repo list +# Expected: the Gitea repo with status "Successful" +``` + +If not connected, re-add via ArgoCD CLI: +```bash +argocd repo add http://gitea.gitea.svc.cluster.local:3000/admin/master.git \ + --name yandex-prod \ + --type git +``` + +Or in the ArgoCD UI: Settings → Repositories → Connect repo. + +--- + +## Phase 7: Post-Bootstrap Checklist + +- [ ] All ArgoCD apps `Synced` and `Healthy` +- [ ] `https://git.prod.t01tt.tech` — Gitea accessible, SSL valid +- [ ] `https://argocd.prod.t01tt.tech` — ArgoCD accessible, SSL valid +- [ ] `https://grafana.prod.t01tt.tech` — Grafana accessible, SSL valid, datasources working +- [ ] `kubectl get pv` — PVCs bound for all stateful components +- [ ] CNPG `shared-pg` cluster status: `kubectl get cluster -n cnpg` shows 3/3 ready +- [ ] Certificates all `Ready`: `kubectl get certificate -A | grep False` (should return nothing) + +--- + +## Quick Reference: Service URLs + +| Service | URL | Auth | +|---------|-----|------| +| Gitea | `https://git.prod.t01tt.tech` | Admin user from Phase 1 | +| ArgoCD | `https://argocd.prod.t01tt.tech` | `admin` / password from Phase 3 | +| Grafana | `https://grafana.prod.t01tt.tech` | `admin` / `change-me` | +| Traefik dashboard | `kubectl port-forward -n traefik daemonset/traefik 9000:9000` | Internal only | + +--- + +## Troubleshooting + +### Traefik NLB stuck in `` +Yandex Cloud NLB provisioning can take a few minutes. Check: +```bash +kubectl describe svc traefik -n traefik +``` +If it's stuck for >5 minutes, verify the Yandex annotations are correct. + +### Certificates stuck in `Pending` +1. Verify DNS: `dig git.prod.t01tt.tech` → must return the NLB IP +2. Check Traefik is listening: `curl -k https:// -H "Host: git.prod.t01tt.tech"` → should return 404 (expected, just verifying Traefik responds) +3. Check orders: `kubectl get orders -A` + +### CNPG cluster not becoming ready +```bash +kubectl describe cluster shared-pg -n cnpg +kubectl logs -n cnpg-system deploy/cnpg-controller-manager +``` +Common issue: pods can't schedule due to `podAntiAffinityType: required`. Ensure all 3 nodes exist and PVCs can bind. + +### Gitea UI shows wrong URL after first login +Gitea caches the ROOT_URL from the `deployment.yaml` env vars. If you change the domaster, update: +```bash +kubectl set env deploy/gitea -n gitea \ + GITEA__server__DOMAIN=git.prod.t01tt.tech \ + GITEA__server__ROOT_URL=https://git.prod.t01tt.tech +kubectl rollout restart deploy/gitea -n gitea +``` + +### ArgoCD apps showing "Unknown" health +This is normal for Ingress resources — the custom health check in `bootstrap/argocd/values.yaml` marks all Ingresses as `Healthy` once synced. For other resources, check the app details in ArgoCD UI for the specific error. diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..8f248e7 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,527 @@ +# Yandex Cloud Production Matrix Cluster — Migration Plan + +## 0. Current State + +### Existing Clusters + +| Cluster | Type | Nodes | Purpose | +|---------|------|-------|---------| +| k3s homelab (`~/infra/k3s`) | self-managed k3s | 2 (oracle, sentinel) | GitOps-based, hosts test `mrt0rtikize.ru` ESS Community | +| Yandex Cloud prod (`kubectl` default) | managed k8s v1.32.1 | 3x 2CPU/6GB | Manual Helm, hosts 3 production Matrix instances | + +### Current Prod Load (Yandex Cloud) + +| Node | CPU | RAM | What runs there | +|------|-----|-----|-----------------| +| `ofem` | 6% | 32% | element-web x3, element-call x2, well-known x3, alloy, cert-manager-cainjector | +| `uzig` | 19% | **94%** | Grafana(682Mi) + Prometheus(499Mi) + VictoriaMetrics(327Mi) + Synapse-t0rt1k(582Mi) + 3x PostgreSQL + Alertmanager + 3x Redis + 3x LiveKit SFU + Traefik | +| `efur` | 10% | 51% | Synapse-roglog(134Mi) + Synapse-uretra(137Mi) + Loki(304Mi) + 2x mas-postgresql + cert-manager + coredns | + +Key issue: **uzig at 94% RAM** — monitoring stack competes with busiest Synapse on same node. + +### Prod Matrix Instances + +| | `matrix-t0rt1k` | `matrix-roglog` | `matrix-uretra` | +|---|---|---|---| +| Domain | `t0rt1k.tech` | `roglog.space` | `uretra.space` | +| Age | 87d | 82d | 82d | +| Synapse RAM | 582Mi | 134Mi | 137Mi | +| Storage | 10+8+1 Gi (HDD) | 10+8+1 Gi (HDD) | 10+8+1 Gi (HDD) | +| Helm chart | `matrix-2.9.17` (NOT ESS) | same | same | +| MAS migration | Failed (`syn2mas` job) | OK | OK | +| Components per instance | Synapse, Element Web, Element Call, LiveKit SFU + Redis + JWT, MAS, 2x PostgreSQL, well-known | same | same | +| Ingress | Traefik, `158.160.164.95` | same LB | same LB | +| TLS | cert-manager + `letsencrypt-production` | same | same | + +### Test Instance (k3s homelab) + +| Property | Value | +|----------|-------| +| Domain | `mrt0rtikize.ru` | +| Chart | ESS Community (`oci://ghcr.io/element-hq/ess-helm/matrix-stack` v26.6.1) | +| Namespace | `matrix-mrt0rtikize` | +| Components | Synapse, MAS, Element Web, Element Admin, Matrix RTC, Hookshot, HAProxy | +| PostgreSQL | Built-in (chart-managed) | +| Storage | Longhorn | +| GitOps | ArgoCD, repo at `gitea.mrt0rtikize.ru` | + +--- + +## 1. New Cluster Architecture + +### 1.1 Platform + +- Yandex Cloud Managed Kubernetes (new cluster) +- Ability to add external nodes in future (supported experimentally, not needed now) +- Managed control plane, self-managed worker nodes + +### 1.2 GitOps Foundation + +| Component | How | Notes | +|-----------|-----|-------| +| Gitea | `kubectl apply` from `bootstrap/gitea/` | Self-hosted git server, deployed first (before ArgoCD) | +| ArgoCD | `helm install` via `bootstrap/argocd/install.sh` | Installed with `--insecure` (same as k3s), points to Gitea | +| Root App | `argocd/app-of-apps.yaml` | Scans `argocd/apps/*.yaml` recursively, deploys everything else | + +### 1.3 Infrastructure Components + +| Component | Type | Values | Notes | +|-----------|------|--------|-------| +| cert-manager | ArgoCD Helm app | `installCRDs: true`, `ClusterIssuer: letsencrypt-production` | TLS for all ingresses | +| CloudNativePG Operator | ArgoCD Helm app | `cluster.instances: 3`, `storageClass: yc-network-ssd`, `size: 50Gi`, `podAntiAffinityType: required` | HA PostgreSQL for all Matrix instances | +| Prometheus Stack | ArgoCD Helm app | Ported from `k3s/manifests/metrics/kube-prometheus-stack-values.yaml`, remoteWrite to VictoriaMetrics | Monitoring + Alertmanager | +| VictoriaMetrics | ArgoCD Helm app | Ported from `k3s/manifests/metrics/victoria-metrics-single-values.yaml` | Long-term metrics storage | +| Loki | ArgoCD Helm app | Log aggregation | — | +| Alloy/Grafana Alloy | ArgoCD Helm app | Agent for metrics/logs forwarding | — | +| Traefik | Managed by Yandex (or DaemonSet) | Cluster's built-in ingress controller | LB external IP provisioned by Yandex Cloud | + +### 1.4 ESS Instances + +Each Matrix homeserver is a separate ArgoCD Application referencing the ESS chart: + +``` +argocd/apps/ +├── matrix-mrt0rtikize.yaml (first, test migration) +├── matrix-t0rt1k.yaml (production, after procedure proven) +├── matrix-roglog.yaml +└── matrix-uretra.yaml +``` + +Each uses the **shared CloudNativePG cluster** (not built-in PostgreSQL). + +### 1.5 Directory Structure + +``` +~/infra/yandex-prod/ +├── bootstrap/ +│ ├── gitea/ +│ │ ├── namespace.yaml +│ │ ├── deployment.yaml +│ │ ├── service.yaml +│ │ ├── ingress.yaml +│ │ └── pvc.yaml +│ └── argocd/ +│ ├── install.sh +│ └── values.yaml +├── argocd/ +│ ├── app-of-apps.yaml +│ └── apps/ +│ ├── cert-manager.yaml +│ ├── cnpg-operator.yaml +│ ├── cnpg-cluster.yaml +│ ├── monitoring.yaml +│ ├── loki.yaml +│ ├── matrix-mrt0rtikize.yaml +│ ├── matrix-t0rt1k.yaml +│ ├── matrix-roglog.yaml +│ └── matrix-uretra.yaml +└── manifests/ + ├── cnpg/ + │ ├── namespace.yaml + │ ├── databases.yaml # Database CRs per homeserver + │ └── secrets.yaml # PG credentials per homeserver (or generated) + └── matrix-mrt0rtikize/ + └── (supplemental manifests, if any) +``` + +### 1.6 CloudNativePG Architecture + +``` +CloudNativePG Cluster "shared-pg" (namespace: cnpg, 3 instances) +├── Instance 1 (node A) +├── Instance 2 (node B) ← anti-affinity ensures spread +└── Instance 3 (node C) + +Databases (one pair per homeserver): +├── synapse_mrt0rtikize (owner: synapse_mrt0rtikize) +├── mas_mrt0rtikize (owner: mas_mrt0rtikize) +├── synapse_t0rt1k (owner: synapse_t0rt1k) +├── mas_t0rt1k (owner: mas_t0rt1k) +├── synapse_roglog (owner: synapse_roglog) +├── mas_roglog (owner: mas_roglog) +├── synapse_uretra (owner: synapse_uretra) +└── mas_uretra (owner: mas_uretra) + +Service: shared-pg-rw.cnpg.svc.cluster.local:5432 (primary, read-write) + shared-pg-ro.cnpg.svc.cluster.local:5432 (replicas, read-only) +``` + +Each homeserver has a dedicated PostgreSQL role and database within the same cluster. +Databases and roles are created via CNPG `Database` CRs (see `manifests/cnpg/databases.yaml`). + +Credentials are stored in per-homeserver Kubernetes Secrets (`manifests/cnpg/secrets.yaml`), +referenced by ESS via `existingSecret` / `existingSecretKey`. + +### 1.7 ESS Configuration (per instance) + +```yaml +# Shared across all instances: +serverName: +certManager: + clusterIssuer: letsencrypt-production +ingress: + className: traefik # or whatever Yandex provides + +# PostgreSQL — external, shared CNPG cluster: +postgres: + enabled: false + +synapse: + postgres: + host: shared-pg-rw.cnpg.svc.cluster.local + database: synapse_ + user: synapse_ + existingSecret: -pg-creds + existingSecretKey: synapse + media: + storage: + size: 10Gi # adjustable per instance load + storageClassName: yc-network-hdd # media is fine on HDD + ingress: + host: matrix. + +matrixAuthenticationService: + postgres: + host: shared-pg-rw.cnpg.svc.cluster.local + database: mas_ + user: mas_ + existingSecret: -pg-creds + existingSecretKey: mas + ingress: + host: account. + +elementWeb: + ingress: + host: chat. + +elementAdmin: + ingress: + host: admin. + +matrixRTC: + ingress: + host: mrtc. + +hookshot: + enabled: true + # ingress host if needed for webhooks +``` + +### 1.8 Boot Order + +``` +Step 1: kubectl apply bootstrap/gitea/ +Step 2: helm install argocd (bootstrap/argocd/install.sh) +Step 3: git push manifests/ + argocd/ to Gitea +Step 4: kubectl apply argocd/app-of-apps.yaml +Step 5: ArgoCD syncs cert-manager, CNPG operator, CNPG cluster, databases, monitoring, ESS +``` + +--- + +## 2. Migration Procedure: `mrt0rtikize.ru` (test instance) + +> Perform on the test instance first to validate the procedure before touching production. + +### 2.1 Backup (on k3s homelab) + +```bash +NS=matrix-mrt0rtikize + +# 1. Stop Synapse + MAS +kubectl scale sts -l "app.kubernetes.io/component=matrix-server" -n $NS --replicas=0 +kubectl scale deploy -l "app.kubernetes.io/component=matrix-authentication" -n $NS --replicas=0 + +# 2. Dump PostgreSQL (built-in PG, release name is "ess" but pods are named matrix-mrt0rtikize-*) +# The PG pod is named based on the ESS release. Find it: +PG_POD=$(kubectl get pods -n $NS -l "app.kubernetes.io/name=postgres" -o name | head -1) +kubectl exec -n $NS $PG_POD -- pg_dumpall -U postgres > dump-mrt0rtikize.sql + +# 3. Backup generated secrets (CRITICAL — contains signing key, MAS encryption key) +kubectl get secret matrix-mrt0rtikize-generated -n $NS -o yaml > secrets-mrt0rtikize.yaml + +# 4. Backup deployment markers +kubectl get configmap \ + -l "app.kubernetes.io/managed-by=matrix-tools-deployment-markers" \ + -n $NS -o yaml > markers-mrt0rtikize.yaml + +# 5. Backup media files +# Find PV path from the node: +kubectl get pv -n $NS -o yaml | grep -A5 "synapse-media" +# Copy from the reported path on the node to a safe location + +# 6. Save ESS values (from the ArgoCD Application or helm get values) +kubectl get application matrix-mrt0rtikize -n argocd -o yaml > app-mrt0rtikize.yaml +``` + +**Critical data that MUST be preserved:** + +| Data | Location | Why | +|------|----------|-----| +| `SYNAPSE_SIGNING_KEY` | `matrix-mrt0rtikize-generated` secret | Federation identity — all other servers know this key. Lose it = all rooms break. | +| `MAS_ENCRYPTION_SECRET` | same secret | User session encryption. Lose it = all users must re-login. | +| `MAS_RSA_PRIVATE_KEY` | same secret | OIDC signing. Lose it = re-auth needed. | +| `SYNAPSE_MACAROON` | same secret | Admin API access token. | +| PostgreSQL dump | `dump-mrt0rtikize.sql` | All user accounts, rooms, messages. | +| Media files | Synapse media PV | Uploaded images/files/avatars. | + +### 2.2 Restore (on new Yandex cluster) + +```bash +# 1. Create secrets in the matrix-mrt0rtikize namespace +NS=matrix-mrt0rtikize +kubectl create ns $NS + +# Apply the generated secrets (signing key etc — DO NOT let initSecrets regenerate it) +kubectl apply -f secrets-mrt0rtikize.yaml +kubectl apply -f markers-mrt0rtikize.yaml + +# 2. Restore PostgreSQL dumps +# CNPG service: shared-pg-rw.cnpg.svc.cluster.local +# Extract per-DB dumps from pg_dumpall or use pg_restore: +PG_POD=$(kubectl get pods -n cnpg -l "cnpg.io/cluster=shared-pg,cnpg.io/podRole=instance" -o name | head -1) + +# Restore Synapse DB: +kubectl exec -n cnpg $PG_POD -- psql -U synapse_mrt0rtikize \ + -d synapse_mrt0rtikize < dump-mrt0rtikize.sql + +# Restore MAS DB: +kubectl exec -n cnpg $PG_POD -- psql -U mas_mrt0rtikize \ + -d mas_mrt0rtikize < dump-mrt0rtikize.sql + +# (Note: pg_dumpall produces a single file for all databases. You may need to +# split it per-database first, or use pg_restore per-database.) + +# 3. Restore media files +# Copy from backup to the new PV (path depends on storage class) +# For Yandex Cloud CSI: mount the PV on a temp pod and copy files in + +# 4. Deploy ESS via ArgoCD +# The Application was already committed to git (argocd/apps/matrix-mrt0rtikize.yaml). +# ArgoCD syncs it. Since secrets + markers are pre-loaded, the chart initializes +# with the existing signing key and database credentials. + +# 5. Verify +# - Log in with an existing user +# - Check federation: https://federationtester.matrix.org/?server_name=mrt0rtikize.ru +# - Test Element Call (VoIP) +# - Monitor logs for errors +``` + +### 2.3 DNS Cutover + +Once validated: +``` +Old records: mrt0rtikize.ru → k3s cluster IP + *.mrt0rtikize.ru → k3s cluster IP + +New records: mrt0rtikize.ru → new cluster Traefik LB IP + matrix.mrt0rtikize.ru → new cluster LB + account.mrt0rtikize.ru → new cluster LB + chat.mrt0rtikize.ru → new cluster LB + admin.mrt0rtikize.ru → new cluster LB + mrtc.mrt0rtikize.ru → new cluster LB +``` + +Lower DNS TTLs 24h before cutover to minimize propagation delay. + +### 2.4 Rollback + +If migration fails: +1. Scale down Synapse + MAS on new cluster +2. Revert DNS to k3s cluster IP +3. Scale up Synapse + MAS on k3s homelab + +The old instance on k3s should still be functional (just stopped, not deleted). + +--- + +## 3. Production Migration (vague plan) + +> Repeat steps from Section 2 for each production instance, one at a time. + +### 3.1 Order + +| # | Instance | Synapse Load | Complexity | +|---|----------|--------------|------------| +| 1 | `mrt0rtikize.ru` | Minimal (test) | Low — prove procedure | +| 2 | `t0rt1k.tech` | **582Mi** (busiest) | High — schedule during low-traffic, may need extended downtime | +| 3 | `roglog.space` | 134Mi | Medium | +| 4 | `uretra.space` | 137Mi | Medium | + +### 3.2 Pre-migration Checklist (per instance) + +``` +[ ] Announce maintenance window to users +[ ] Lower DNS TTLs (24h before) +[ ] Full PostgreSQL dump + verify (pg_restore --list) +[ ] Backup media files + verify checksums +[ ] Backup generated secrets (verify signing key matches federation) +[ ] Save current Helm values (helm get values) +[ ] Document current ingress/DNS/Certificate setup +[ ] Prepare rollback procedure +``` + +### 3.3 Migration Steps (per instance) + +``` +1. Stop Synapse + MAS on old cluster +2. Create CNPG databases on new cluster +3. Restore PostgreSQL dump to CNPG +4. Restore media files to new PV +5. Apply secrets (signing key, MAS keys, macaroon) +6. Apply deployment markers +7. Deploy ESS via ArgoCD on new cluster +8. Wait for pods healthy, certs issued +9. Test: login, federation, Element Call +10. Cut over DNS +11. Monitor for 24h +12. If stable: remove old instance resources from old cluster +``` + +### 3.4 Special Considerations for Prod Instances + +**`t0rt1k.tech` (busiest instance, 582Mi Synapse):** +- Uses older `matrix-2.9.17` chart (NOT ESS). Migration means switching to ESS Community chart. +- Has a **failed `syn2mas` job** — MAS migration was incomplete. When deploying ESS which bundles MAS, the migration may need to be completed or re-done. +- The 582Mi memory usage suggests many concurrent users/rooms — dump may be large. Allocate enough storage and time for the SQL dump/restore. +- Consider running the new ESS in parallel (different hostnames) first, then switching DNS once proven. + +**`roglog.space` and `uretra.space`:** +- Lower load (134Mi/137Mi) — quicker backups, less downtime risk. +- Same chart switch (`matrix-2.9.17` → ESS). +- Can be done in shorter windows. + +**Chart migration (`matrix-2.9.17` → ESS):** +- The old chart uses separate Helm releases per component (`chat`, `element-call`, `livekit`). +- ESS bundles everything into one chart. The database schema may differ. +- Key difference: ESS uses MAS for auth (Matrix 2.0), old chart may use legacy Synapse auth. +- May need to run `syn2mas` migration or manual user migration. Investigate per-instance before cutover. + +--- + +## 4. PostgreSQL Backup (ongoing) + +CloudNativePG has built-in backup to S3-compatible storage. Configure once for automatic daily backups: + +```yaml +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: shared-pg-daily + namespace: cnpg +spec: + schedule: "0 3 * * *" # 03:00 UTC daily + backupOwnerReference: self + cluster: + name: shared-pg + immediate: false + target: prefer-standby +``` + +CNPG also supports continuous WAL archiving to S3 for point-in-time recovery. +Configure Yandex Object Storage as the S3 target. + +--- + +## 5. Architecture Diagram (text) + +``` +┌────────────────────────────────────────────────────────────┐ +│ Yandex Cloud Managed K8s │ +│ │ +│ ┌───────────────────┐ ┌──────────────────────────────┐ │ +│ │ Infrastructure │ │ Matrix Layer │ │ +│ │ │ │ │ │ +│ │ Gitea (git) │ │ ┌─────────────────────────┐ │ │ +│ │ ArgoCD (gitops) │ │ │ matrix-mrt0rtikize (ESS)│ │ │ +│ │ cert-manager │ │ │ - Synapse │ │ │ +│ │ Traefik (LB) │ │ │ - MAS │ │ │ +│ │ Prometheus/Grafana │ │ │ - Element Web/Admin │ │ │ +│ │ VictoriaMetrics │ │ │ - Matrix RTC (LiveKit) │ │ │ +│ │ Loki │ │ │ - Hookshot │ │ │ +│ │ Alloy │ │ │ - HAProxy │ │ │ +│ └───────────────────┘ │ └──────────┬──────────────┘ │ │ +│ │ │ │ │ +│ ┌───────────────────┐ │ ┌──────────▼──────────────┐ │ │ +│ │ CNPG Cluster │ │ │ matrix-t0rt1k (ESS) │ │ │ +│ │ (3 nodes, SSD) │◄──┤ │ (same structure) │ │ │ +│ │ │ │ └─────────────────────────┘ │ │ +│ │ synapse_mrt0rtikize│ │ ┌─────────────────────────┐ │ │ +│ │ mas_mrt0rtikize │ │ │ matrix-roglog (ESS) │ │ │ +│ │ synapse_t0rt1k │ │ │ (same structure) │ │ │ +│ │ mas_t0rt1k │ │ └─────────────────────────┘ │ │ +│ │ synapse_roglog │ │ ┌─────────────────────────┐ │ │ +│ │ mas_roglog │ │ │ matrix-uretra (ESS) │ │ │ +│ │ synapse_uretra │ │ │ (same structure) │ │ │ +│ │ mas_uretra │ │ └─────────────────────────┘ │ │ +│ └───────────────────┘ └──────────────────────────────┘ │ +│ │ +│ External LB: │ +└────────────────────────────────────────────────────────────┘ +``` + +--- + +## 6. Implementation Notes + +### 6.1 Secrets Management + +- ESS `initSecrets` generates 14 credentials. For migration, these MUST be restored from backup (not regenerated). +- `SYNAPSE_SIGNING_KEY` is the most critical — it identifies the server to the federation. Changing it breaks all existing rooms and federation relationships. +- The `matrix*-generated` secret and deployment markers ConfigMap must be applied **before** the first ArgoCD sync, so the ESS chart does not generate new (wrong) ones. +- For fresh ESS instances (new homeservers, not migrations), let `initSecrets` generate them normally. + +### 6.2 Image Registry + +- ESS pulls from `oci.element.io` (Synapse, Element Web, Element Admin, lk-jwt-service) and `ghcr.io` (matrix-tools, hookshot), and `docker.io` (livekit, postgres, redis). +- `oci.element.io` S3 backend (`oci-element-io-images-storage-prod.s3.eu-central-1.amazonaws.com`) was observed to fail intermittently from Russia with "connection reset by peer". Images eventually pulled on retry, but consider: + - Setting `image.pullPolicy: IfNotPresent` to reduce re-pulls + - Setting up a containerd registry mirror or local pull-through cache for `oci.element.io` + - Pre-pulling images to nodes during initial setup + +### 6.3 Resource Limits + +Set `resources.requests` and `resources.limits` on all ESS components to prevent the 94% node issue seen in prod: + +```yaml +synapse: + resources: + requests: + memory: 256Mi + cpu: 100m + limits: + memory: 1Gi + cpu: 1000m +``` + +Do similar for MAS, element-web, livekit-sfu, etc. ESS chart supports per-component resource configuration. + +### 6.4 Storage Classes + +| Workload | Storage Class | Reason | +|----------|--------------|--------| +| PostgreSQL (CNPG) | `yc-network-ssd` | Database — needs low latency / high IOPS | +| Synapse media | `yc-network-hdd` (default) | Media files — sequential access, SSD benefit is marginal | +| Prometheus TSDB | `yc-network-ssd` | Time-series DB — random writes benefit from SSD | +| Loki chunks | `yc-network-hdd` | Log storage — sequential writes, HDD is fine | + +--- + +## 7. Next Steps (for next session) + +When the new cluster is ready, open a new session and point to this file. The next session should: + +1. Read this plan +2. Explore the new cluster (nodes, storage classes, ingress config) +3. Implement Phase 0 (bootstrap GitOps foundation): + - Create `~/infra/yandex-prod/` directory structure + - Write `bootstrap/gitea/` manifests + - Write `bootstrap/argocd/install.sh` + `values.yaml` + - Write `argocd/app-of-apps.yaml` + - Write infrastructure apps (cert-manager, CNPG, monitoring) + - Write ESS apps + - Push to Gitea +4. Execute Phase 1 (backup `mrt0rtikize.ru` from k3s) +5. Execute Phase 2 (restore `mrt0rtikize.ru` to new cluster) +6. Validate and plan DNS cutover diff --git a/STATUS.md b/STATUS.md new file mode 100644 index 0000000..be301e5 --- /dev/null +++ b/STATUS.md @@ -0,0 +1,124 @@ +# Yandex Cloud Production Cluster — Current State + +> Last updated: 2026-06-12 + +--- + +## Cluster Overview + +| Property | Value | +|----------|-------| +| Nodes | 3x 2CPU/8GB, zones b/d/e | +| Egress | NAT Gateway `k8s-nat-gw` (shared, no per-node IPs) | +| Domain base | `*.prod.t01tt.tech` | +| NLB IP | `81.26.181.240` | +| Git repo (Gitea) | `admin/main.git`, branch `master` | +| ArgoCD sync mode | **Manual** (no auto-sync, no auto-prune) | + +--- + +## Infrastructure Status + +### Running & Healthy ✅ + +| Component | URL / Access | Notes | +|-----------|-------------|-------| +| Gitea | `https://git.prod.t01tt.tech` | Also `git@git.prod.t01tt.tech:admin/main.git` (SSH via NLB port 22) | +| ArgoCD | `https://argocd.prod.t01tt.tech` | Manual sync only | +| cert-manager | — | `letsencrypt-production` + `staging` ClusterIssuers | +| Traefik | NLB `81.26.181.240` | Ports 80, 443, 22 (SSH for Gitea) | +| Monitoring (VM k8s-stack) | — | VMSingle, VMAlertmanager, node-exporter, kube-state-metrics | +| Grafana | `https://grafana.prod.t01tt.tech` | `admin`/`change-me`, VictoriaMetrics datasource works | +| CNPG Operator | — | v0.28.3, running in `cnpg-system` | +| CNPG Cluster `shared-pg` | — | 3/3 nodes healthy, 20Gi SSD each, no databases created yet | +| Alloy | — | DaemonSet, collecting pod logs, shipping to Loki | +| Loki | `http://loki.metrics.svc.cluster.local:3100` | Single binary, 20Gi HDD, logs flowing, queryable via API | + +### Broken/Incomplete ⚠️ + +| Component | Issue | Next Step | +|-----------|-------|-----------| +| Grafana + Loki | Grafana 13.0.1 has a Loki datasource plugin bug (`unsupported protocol scheme ""`). Loki API itself works (direct queries return data). | Wait for VM k8s-stack Helm chart to bundle a newer Grafana version (13.1.0+). Then add `grafana.image.tag: "13.1.0"` to `monitoring.yaml`. OR: expose Loki via Traefik ingress + use `access: direct` in datasource. | +| CNPG Databases | `manifests/cnpg/databases.yaml` — `clusterRef` must be `cluster` (8 occurrences) for CNPG v1 API. `manifests/cnpg/secrets.yaml` — type must be `Opaque` with plain password strings. | Fix when creating ESS Matrix apps — databases and secrets are part of the Matrix instance setup, not standalone infra. | +| `cnpg-cluster` ArgoCD app | OutOfSync due to the two files above. | Sync after fixing databases + secrets as part of ESS setup. | + +### ArgoCD App Status + +| App | Sync | Health | Notes | +|-----|------|--------|-------| +| alloy | OutOfSync | Healthy | Pushed to Gitea, needs manual sync | +| cert-manager | Synced | Healthy | | +| cert-manager-issuers | Synced | Healthy | | +| cnpg-cluster | OutOfSync | Healthy | Blocked by databases.yaml + secrets.yaml | +| cnpg-operator | Synced | Healthy | | +| loki | OutOfSync | Healthy | Pushed to Gitea, needs manual sync | +| monitoring | Synced | Healthy | | +| root-app | Synced | Healthy | | +| traefik | Synced | Healthy | | + +--- + +## Directory Structure + +``` +~/infra/yandex-prod/ +├── BOOTSTRAP.md # Step-by-step bootstrap guide +├── PLAN.md # Original migration plan +├── STATUS.md # This file +├── kubeconfig # Cluster kubeconfig +├── bootstrap/ +│ ├── gitea/ # 5 manifests (namespace, pvc, deploy, svc, ingress) +│ └── argocd/ # install.sh, values.yaml, ingress.yaml +├── argocd/ +│ ├── app-of-apps.yaml # Root app: watches argocd/apps/*.yaml +│ └── apps/ +│ ├── traefik.yaml # DaemonSet + NLB (wave -2) +│ ├── cert-manager.yaml # Helm chart (wave -1) +│ ├── cert-manager-issuers.yaml # ClusterIssuer CRs (wave 0) +│ ├── cnpg-operator.yaml # CNPG Helm chart +│ ├── cnpg-cluster.yaml # Cluster + DB CRs from manifests/cnpg/ +│ ├── monitoring.yaml # VM k8s-stack (wave 0) +│ ├── loki.yaml # Loki single-binary (wave 0) +│ └── alloy.yaml # Alloy log collector (wave 0) +└── manifests/ + ├── cert-manager/ + │ └── cluster-issuers.yaml # letsencrypt-production + staging + ├── cnpg/ + │ ├── namespace.yaml + │ ├── cluster.yaml # shared-pg Cluster CR (3 nodes, 20Gi SSD) + │ ├── databases.yaml # 8 Database CRs (BROKEN: clusterRef) + │ └── secrets.yaml # 4 PG cred secrets (BROKEN: type+values) + ├── gitea/ + │ └── ingressroute-ssh.yaml # Traefik TCP route for Gitea SSH + └── metrics/ + ├── grafana/ + │ ├── namespace.yaml + │ ├── ingress.yaml # grafana.prod.t01tt.tech + │ └── loki-datasource.yaml # Loki datasource ConfigMap + └── (empty — namespace.yaml moved into grafana/) +``` + +--- + +## Key Decisions Made + +| Decision | Why | +|----------|-----| +| NAT Gateway instead of per-node IPs | IP quota limits; shared egress via `k8s-nat-gw` | +| Auto-sync disabled on all apps | Manual control during bootstrapping | +| VM k8s-stack instead of kube-prometheus-stack | Single Helm chart for metrics | +| Alloy for log collection | Ported working config from old `yc-playground` cluster | +| Gryphon the Grafekr 13 Loki bug | MVP: logs are collected and queryable via API; UI integration deferred | + +--- + +## Next Session Priorities + +1. **Sync alloy + loki** (already in Gitea, just needs manual ArgoCD sync) +2. **Fix Grafana + Loki** — either Grafana 13.1.0 via chart update or Loki ingress workaround +3. **Create ESS Matrix app for `mrt0rtikize.ru`** — the test migration (Phase 2 of PLAN.md): + - Fix `databases.yaml` and `secrets.yaml` as part of this + - Create `argocd/apps/matrix-mrt0rtikize.yaml` + - Backup from k3s cluster, restore to new cluster + - Test DNS cutover +4. **Create ESS apps for prod instances** (`t0rt1k.tech`, `roglog.space`, `uretra.space`) diff --git a/argocd/app-of-apps.yaml b/argocd/app-of-apps.yaml new file mode 100644 index 0000000..ade871b --- /dev/null +++ b/argocd/app-of-apps.yaml @@ -0,0 +1,22 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: root-app + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: http://gitea.gitea.svc.cluster.local:3000/admin/main.git + targetRevision: master + path: argocd/apps + directory: + recurse: true + include: "*.yaml" + destination: + server: https://kubernetes.default.svc + namespace: argocd + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/apps/alloy.yaml b/argocd/apps/alloy.yaml new file mode 100644 index 0000000..8b8fc7d --- /dev/null +++ b/argocd/apps/alloy.yaml @@ -0,0 +1,103 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: alloy + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "0" + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://grafana.github.io/helm-charts + chart: alloy + targetRevision: ">=1.5.0" + helm: + values: | + alloy: + configMap: + content: |- + // Discover Kubernetes pods + discovery.kubernetes "pods" { + role = "pod" + } + + // Relabel pods to intelligently map existing Kubernetes labels to service/component + discovery.relabel "pods" { + targets = discovery.kubernetes.pods.targets + + // Create service label - try multiple sources in priority order + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] + regex = "(.+)" + replacement = "${1}" + target_label = "service" + } + // 2. Second priority: app label + rule { + source_labels = ["__meta_kubernetes_pod_label_app", "service"] + regex = "^(.+);$" + replacement = "${1}" + target_label = "service" + } + // 3. Third priority: extract from pod name (remove hash suffix) + rule { + source_labels = ["__meta_kubernetes_pod_name", "service"] + regex = "^([a-z0-9-]+?)(?:-[a-f0-9]{5,10})?;$" + replacement = "${1}" + target_label = "service" + } + + // Create component label from app.kubernetes.io/component or component label + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_component"] + regex = ".+" + target_label = "component" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_component"] + regex = ".+" + target_label = "component" + } + + // Drop pods that we still can't identify + rule { + source_labels = ["service"] + regex = "^$" + action = "drop" + } + + // Map standard Kubernetes metadata to Loki labels + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + } + + // Collect logs from Kubernetes pods using the Kubernetes API + loki.source.kubernetes "pods" { + targets = discovery.relabel.pods.output + forward_to = [loki.write.loki.receiver] + } + + // Write logs to Loki + loki.write "loki" { + endpoint { + url = "http://loki.metrics.svc.cluster.local:3100/loki/api/v1/push" + } + } + destination: + server: https://kubernetes.default.svc + namespace: metrics + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/apps/cert-manager-issuers.yaml b/argocd/apps/cert-manager-issuers.yaml new file mode 100644 index 0000000..026e2f8 --- /dev/null +++ b/argocd/apps/cert-manager-issuers.yaml @@ -0,0 +1,22 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: cert-manager-issuers + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "0" + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: http://gitea.gitea.svc.cluster.local:3000/admin/main.git + targetRevision: master + path: manifests/cert-manager + directory: + recurse: true + include: "*.yaml" + destination: + server: https://kubernetes.default.svc + namespace: cert-manager + syncPolicy: {} diff --git a/argocd/apps/cert-manager.yaml b/argocd/apps/cert-manager.yaml new file mode 100644 index 0000000..5c8bcd2 --- /dev/null +++ b/argocd/apps/cert-manager.yaml @@ -0,0 +1,24 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: cert-manager + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "-1" + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://charts.jetstack.io + chart: cert-manager + targetRevision: ">=1.18.0" + helm: + values: | + installCRDs: true + destination: + server: https://kubernetes.default.svc + namespace: cert-manager + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/apps/cnpg-cluster.yaml b/argocd/apps/cnpg-cluster.yaml new file mode 100644 index 0000000..e038ae4 --- /dev/null +++ b/argocd/apps/cnpg-cluster.yaml @@ -0,0 +1,24 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: cnpg-cluster + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "1" + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: http://gitea.gitea.svc.cluster.local:3000/admin/main.git + targetRevision: master + path: manifests/cnpg + directory: + recurse: true + include: "*.yaml" + destination: + server: https://kubernetes.default.svc + namespace: cnpg + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/apps/cnpg-operator.yaml b/argocd/apps/cnpg-operator.yaml new file mode 100644 index 0000000..bb23cdc --- /dev/null +++ b/argocd/apps/cnpg-operator.yaml @@ -0,0 +1,20 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: cnpg-operator + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://cloudnative-pg.github.io/charts + chart: cloudnative-pg + targetRevision: ">=0.23.0" + destination: + server: https://kubernetes.default.svc + namespace: cnpg-system + syncPolicy: + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/argocd/apps/loki.yaml b/argocd/apps/loki.yaml new file mode 100644 index 0000000..2da8f27 --- /dev/null +++ b/argocd/apps/loki.yaml @@ -0,0 +1,92 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: loki + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "0" + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://grafana.github.io/helm-charts + chart: loki + targetRevision: ">=6.0.0" + helm: + values: | + deploymentMode: SingleBinary + + write: + replicas: 0 + read: + replicas: 0 + backend: + replicas: 0 + + loki: + auth_enabled: false + storage: + bucketNames: + chunks: chunks + ruler: ruler + admin: admin + type: filesystem + filesystem: + chunks_directory: /var/loki/chunks + rules_directory: /var/loki/rules + structuredConfig: + auth_enabled: false + + server: + http_listen_port: 3100 + grpc_listen_port: 9095 + + common: + path_prefix: /var/loki + replication_factor: 1 + + schema_config: + configs: + - from: "2025-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + + limits_config: + retention_period: 30d + reject_old_samples: true + reject_old_samples_max_age: 168h + + chunksCache: + enabled: false + resultsCache: + enabled: false + gateway: + enabled: false + test: + enabled: false + lokiCanary: + enabled: false + + singleBinary: + replicas: 1 + persistence: + enabled: true + size: 20Gi + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + destination: + server: https://kubernetes.default.svc + namespace: metrics + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/apps/matrix-mrt0rtikize.yaml b/argocd/apps/matrix-mrt0rtikize.yaml new file mode 100644 index 0000000..9e0675f --- /dev/null +++ b/argocd/apps/matrix-mrt0rtikize.yaml @@ -0,0 +1,89 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: matrix-mrt0rtikize + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + sources: + - repoURL: http://gitea.gitea.svc.cluster.local:3000/admin/main.git + targetRevision: master + path: manifests/matrix-mrt0rtikize + directory: + recurse: true + include: "*.yaml" + - repoURL: ghcr.io + chart: element-hq/ess-helm/matrix-stack + targetRevision: 26.6.1 + helm: + values: | + serverName: mrt0rtikize.ru + + certManager: + clusterIssuer: letsencrypt-production + + ingress: + className: traefik + + postgres: + enabled: false + + synapse: + postgres: + host: shared-pg-rw.cnpg.svc.cluster.local + database: synapse_mrt0rtikize + user: synapse_mrt0rtikize + password: + secret: pg-creds + secretKey: synapse + media: + storage: + size: 10Gi + ingress: + host: matrix.mrt0rtikize.ru + + matrixAuthenticationService: + postgres: + host: shared-pg-rw.cnpg.svc.cluster.local + database: mas_mrt0rtikize + user: mas_mrt0rtikize + password: + secret: pg-creds + secretKey: mas + ingress: + host: account.mrt0rtikize.ru + + elementWeb: + ingress: + host: chat.mrt0rtikize.ru + + elementAdmin: + ingress: + host: admin.mrt0rtikize.ru + + matrixRTC: + ingress: + host: mrtc.mrt0rtikize.ru + sfu: + manualIP: "81.26.181.240" + useStunToDiscoverPublicIP: false + exposedServices: + rtcTcp: + port: 30009 + portType: NodePort + nodePort: "" + rtcMuxedUdp: + port: 30008 + portType: NodePort + nodePort: "" + + hookshot: + enabled: true + destination: + server: https://kubernetes.default.svc + namespace: matrix-mrt0rtikize + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/apps/matrix-roglog.yaml b/argocd/apps/matrix-roglog.yaml new file mode 100644 index 0000000..da4fc67 --- /dev/null +++ b/argocd/apps/matrix-roglog.yaml @@ -0,0 +1,89 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: matrix-roglog + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + sources: + - repoURL: http://gitea.gitea.svc.cluster.local:3000/admin/main.git + targetRevision: master + path: manifests/matrix-roglog + directory: + recurse: true + include: "*.yaml" + - repoURL: ghcr.io + chart: element-hq/ess-helm/matrix-stack + targetRevision: 26.6.1 + helm: + values: | + serverName: roglog.space + + certManager: + clusterIssuer: letsencrypt-production + + ingress: + className: traefik + + postgres: + enabled: false + + synapse: + postgres: + host: shared-pg-rw.cnpg.svc.cluster.local + database: synapse_roglog + user: synapse_roglog + password: + secret: pg-creds + secretKey: synapse + media: + storage: + size: 10Gi + ingress: + host: matrix.roglog.space + + matrixAuthenticationService: + postgres: + host: shared-pg-rw.cnpg.svc.cluster.local + database: mas_roglog + user: mas_roglog + password: + secret: pg-creds + secretKey: mas + ingress: + host: auth.roglog.space + + elementWeb: + ingress: + host: chat.roglog.space + + elementAdmin: + ingress: + host: admin.roglog.space + + matrixRTC: + ingress: + host: rtc.roglog.space + sfu: + manualIP: "81.26.181.240" + useStunToDiscoverPublicIP: false + exposedServices: + rtcTcp: + port: 30005 + portType: NodePort + nodePort: "" + rtcMuxedUdp: + port: 30004 + portType: NodePort + nodePort: "" + + hookshot: + enabled: true + destination: + server: https://kubernetes.default.svc + namespace: matrix-roglog + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/apps/matrix-t0rt1k.yaml b/argocd/apps/matrix-t0rt1k.yaml new file mode 100644 index 0000000..f8af807 --- /dev/null +++ b/argocd/apps/matrix-t0rt1k.yaml @@ -0,0 +1,160 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: matrix-t0rt1k + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + sources: + - repoURL: http://gitea.gitea.svc.cluster.local:3000/admin/main.git + targetRevision: master + path: manifests/matrix-t0rt1k + directory: + recurse: true + include: "*.yaml" + - repoURL: ghcr.io + chart: element-hq/ess-helm/matrix-stack + targetRevision: 26.6.1 + helm: + values: | + serverName: t0rt1k.tech + + certManager: + clusterIssuer: letsencrypt-production + + ingress: + className: traefik + + postgres: + enabled: false + + synapse: + postgres: + host: shared-pg-rw.cnpg.svc.cluster.local + database: synapse_t0rt1k + user: synapse_t0rt1k + password: + secret: pg-creds + secretKey: synapse + media: + storage: + size: 10Gi + ingress: + host: matrix.t0rt1k.tech + resources: + requests: + memory: 256Mi + cpu: 100m + limits: + memory: 1.5Gi + workers: + federation-sender: + enabled: true + resources: + requests: + memory: 192Mi + cpu: 50m + limits: + memory: 256Mi + federation-reader: + enabled: true + resources: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 256Mi + client-reader: + enabled: true + replicas: 1 + resources: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 256Mi + initial-synchrotron: + enabled: true + resources: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 256Mi + synchrotron: + enabled: true + replicas: 1 + resources: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 256Mi + additional: + 0-federation-retry: + config: | + destination_min_retry_interval: "1m" + destination_retry_multiplier: 1.5 + destination_max_retry_interval: "1d" + + matrixAuthenticationService: + postgres: + host: shared-pg-rw.cnpg.svc.cluster.local + database: mas_t0rt1k + user: mas_t0rt1k + password: + secret: pg-creds + secretKey: mas + ingress: + host: auth.t0rt1k.tech + resources: + limits: + memory: 128Mi + + elementWeb: + ingress: + host: chat.t0rt1k.tech + + elementAdmin: + ingress: + host: admin.t0rt1k.tech + + matrixRTC: + ingress: + host: rtc.t0rt1k.tech + sfu: + manualIP: "81.26.181.240" + useStunToDiscoverPublicIP: false + exposedServices: + rtcTcp: + port: 30007 + portType: NodePort + nodePort: "" + rtcMuxedUdp: + port: 30006 + portType: NodePort + nodePort: "" + resources: + requests: + memory: 64Mi + cpu: 50m + limits: + memory: 512Mi + + hookshot: + enabled: true + resources: + limits: + memory: 256Mi + ingress: + host: hookshot.t0rt1k.tech + className: traefik + tlsEnabled: true + destination: + server: https://kubernetes.default.svc + namespace: matrix-t0rt1k + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/apps/monitoring.yaml b/argocd/apps/monitoring.yaml new file mode 100644 index 0000000..926ff6e --- /dev/null +++ b/argocd/apps/monitoring.yaml @@ -0,0 +1,142 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: monitoring + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "0" + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + sources: + - repoURL: http://gitea.gitea.svc.cluster.local:3000/admin/main.git + targetRevision: master + path: manifests/metrics/grafana + directory: + recurse: true + include: "*.yaml" + + - repoURL: https://victoriametrics.github.io/helm-charts/ + chart: victoria-metrics-k8s-stack + targetRevision: ">=0.30.0" + helm: + values: | + fullnameOverride: vm-k8s-stack + namespaceOverride: metrics + + victoria-metrics-operator: + resources: + requests: + memory: 128Mi + cpu: 25m + limits: + memory: 256Mi + + defaultDashboards: + dashboards: + node-exporter-full: + enabled: false + + vmsingle: + enabled: true + spec: + retentionPeriod: "30d" + replicaCount: 1 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 30Gi + + alertmanager: + enabled: true + spec: + replicaCount: 1 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 200m + memory: 512Mi + storage: + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + config: + route: + receiver: blackhole + receivers: + - name: blackhole + + grafana: + enabled: true + adminUser: admin + adminPassword: change-me + persistence: + enabled: true + size: 2Gi + resources: + requests: + cpu: 50m + memory: 256Mi + limits: + cpu: 200m + memory: 512Mi + + prometheus-node-exporter: + enabled: true + + kube-state-metrics: + enabled: true + + kubelet: + enabled: true + + kubeApiServer: + enabled: false + + kubeControllerManager: + enabled: false + + kubeScheduler: + enabled: false + + kubeProxy: + enabled: false + + kubeEtcd: + enabled: false + + destination: + server: https://kubernetes.default.svc + namespace: metrics + ignoreDifferences: + - group: operator.victoriametrics.com + kind: VMAlertmanager + jsonPointers: + - /spec/config + - kind: Secret + name: monitoring-victoria-metrics-operator-validation + jsonPointers: + - /data + - kind: ValidatingWebhookConfiguration + name: monitoring-victoria-metrics-operator-admission + jsonPointers: + - /webhooks/0/clientConfig/caBundle + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/apps/traefik.yaml b/argocd/apps/traefik.yaml new file mode 100644 index 0000000..1ab61ab --- /dev/null +++ b/argocd/apps/traefik.yaml @@ -0,0 +1,139 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: traefik + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "-2" + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://traefik.github.io/charts + chart: traefik + targetRevision: ">=37.0.0" + helm: + values: | + deployment: + kind: DaemonSet + podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9100" + + ingressClass: + enabled: true + isDefaultClass: true + + additionalArguments: + - "--api.dashboard=true" + - "--ping=true" + - "--metrics.prometheus=true" + - "--metrics.prometheus.entrypoint=metrics" + - "--entryPoints.ssh.address=:2222/tcp" + - "--entryPoints.rtc-tcp-mrt0rtikize.address=:30009/tcp" + - "--entryPoints.rtc-udp-mrt0rtikize.address=:30008/udp" + - "--entryPoints.rtc-tcp-t0rt1k.address=:30007/tcp" + - "--entryPoints.rtc-udp-t0rt1k.address=:30006/udp" + - "--entryPoints.rtc-tcp-roglog.address=:30005/tcp" + - "--entryPoints.rtc-udp-roglog.address=:30004/udp" + - "--providers.kubernetesingress.ingressclass=traefik" + - "--providers.kubernetesingress.ingressendpoint.publishedservice=traefik/traefik" + - "--accesslog=true" + - "--log.level=INFO" + + ports: + web: + port: 8080 + exposedPort: 80 + http: + redirections: + entryPoint: + to: websecure + scheme: https + permanent: true + websecure: + port: 8443 + exposedPort: 443 + ssh: + port: 2222 + exposedPort: 22 + protocol: TCP + expose: + default: true + rtc-tcp-mrt0rtikize: + port: 30009 + exposedPort: 30009 + protocol: TCP + expose: + default: true + rtc-udp-mrt0rtikize: + port: 30008 + exposedPort: 30008 + protocol: UDP + expose: + default: true + rtc-tcp-t0rt1k: + port: 30007 + exposedPort: 30007 + protocol: TCP + expose: + default: true + rtc-udp-t0rt1k: + port: 30006 + exposedPort: 30006 + protocol: UDP + expose: + default: true + rtc-tcp-roglog: + port: 30005 + exposedPort: 30005 + protocol: TCP + expose: + default: true + rtc-udp-roglog: + port: 30004 + exposedPort: 30004 + protocol: UDP + expose: + default: true + metrics: + port: 9100 + expose: + default: false + traefik: + port: 9000 + expose: + default: false + + service: + type: LoadBalancer + annotations: + service.beta.kubernetes.io/yandex-load-balancer-name: traefik + service.beta.kubernetes.io/yandex-load-balancer-specification: '{"type": "network-load-balancer"}' + service.beta.kubernetes.io/yandex-load-balancer-type: external + + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + + destination: + server: https://kubernetes.default.svc + namespace: traefik + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/bootstrap/argocd/ingress.yaml b/bootstrap/argocd/ingress.yaml new file mode 100644 index 0000000..e6e29a1 --- /dev/null +++ b/bootstrap/argocd/ingress.yaml @@ -0,0 +1,26 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: argocd + namespace: argocd + annotations: + cert-manager.io/cluster-issuer: letsencrypt-production + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" +spec: + ingressClassName: traefik + tls: + - hosts: + - argocd.prod.t01tt.tech + secretName: argocd-tls + rules: + - host: argocd.prod.t01tt.tech + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: argocd-server + port: + number: 80 diff --git a/bootstrap/argocd/install.sh b/bootstrap/argocd/install.sh new file mode 100755 index 0000000..c3bfedc --- /dev/null +++ b/bootstrap/argocd/install.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -e + +export KUBECONFIG="$(dirname "$(realpath "$0")")/../../kubeconfig" + +echo "=== Installing ArgoCD ===" + +helm repo add argo https://argoproj.github.io/argo-helm 2>/dev/null || true +helm repo update argo + +helm upgrade --install argocd argo/argo-cd \ + --namespace argocd \ + --create-namespace \ + --values "$(dirname "$0")/values.yaml" \ + --wait \ + --timeout 300s + +echo "" +echo "=== ArgoCD installed ===" +echo "" +echo "To access ArgoCD UI:" +echo " kubectl port-forward svc/argocd-server -n argocd 8080:80" +echo "" +echo "Admin password:" +kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d +echo "" +echo "" +echo "Login with username: admin" diff --git a/bootstrap/argocd/values.yaml b/bootstrap/argocd/values.yaml new file mode 100644 index 0000000..46f17fe --- /dev/null +++ b/bootstrap/argocd/values.yaml @@ -0,0 +1,20 @@ +server: + extraArgs: + - --insecure + +configs: + params: + server.insecure: true + cm: + timeout.reconciliation: 180s + helm.timeoutSeconds: "300" + resource.customizations.health.networking.k8s.io_Ingress: | + hs = {} + hs.status = "Healthy" + hs.message = "Ingress is synced" + return hs + +redis: + image: + repository: docker.io/library/redis + tag: 7.4-alpine diff --git a/bootstrap/gitea/deployment.yaml b/bootstrap/gitea/deployment.yaml new file mode 100644 index 0000000..cec0658 --- /dev/null +++ b/bootstrap/gitea/deployment.yaml @@ -0,0 +1,64 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gitea + namespace: gitea +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: gitea + template: + metadata: + labels: + app: gitea + spec: + containers: + - name: gitea + image: gitea/gitea:1.24 + ports: + - containerPort: 3000 + name: http + - containerPort: 22 + name: ssh + env: + - name: GITEA__database__DB_TYPE + value: sqlite3 + - name: GITEA__server__DOMAIN + value: git.prod.t01tt.tech + - name: GITEA__server__ROOT_URL + value: https://git.prod.t01tt.tech + - name: GITEA__server__HTTP_PORT + value: "3000" + - name: GITEA__server__SSH_PORT + value: "22" + - name: GITEA__service__DISABLE_REGISTRATION + value: "true" + volumeMounts: + - name: data + mountPath: /data + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + livenessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 5 + volumes: + - name: data + persistentVolumeClaim: + claimName: gitea-data diff --git a/bootstrap/gitea/ingress.yaml b/bootstrap/gitea/ingress.yaml new file mode 100644 index 0000000..e99f3e6 --- /dev/null +++ b/bootstrap/gitea/ingress.yaml @@ -0,0 +1,26 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: gitea + namespace: gitea + annotations: + cert-manager.io/cluster-issuer: letsencrypt-production + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" +spec: + ingressClassName: traefik + tls: + - hosts: + - git.prod.t01tt.tech + secretName: gitea-tls + rules: + - host: git.prod.t01tt.tech + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: gitea + port: + number: 3000 diff --git a/bootstrap/gitea/namespace.yaml b/bootstrap/gitea/namespace.yaml new file mode 100644 index 0000000..09a988f --- /dev/null +++ b/bootstrap/gitea/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: gitea diff --git a/bootstrap/gitea/pvc.yaml b/bootstrap/gitea/pvc.yaml new file mode 100644 index 0000000..ea94e6e --- /dev/null +++ b/bootstrap/gitea/pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: gitea-data + namespace: gitea +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi diff --git a/bootstrap/gitea/service.yaml b/bootstrap/gitea/service.yaml new file mode 100644 index 0000000..378a351 --- /dev/null +++ b/bootstrap/gitea/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: gitea + namespace: gitea +spec: + selector: + app: gitea + ports: + - name: http + port: 3000 + targetPort: 3000 + - name: ssh + port: 22 + targetPort: 22 diff --git a/fqdns b/fqdns new file mode 100644 index 0000000..584eba8 --- /dev/null +++ b/fqdns @@ -0,0 +1,7 @@ +DNS records to add (all → 81.26.181.240) +matrix.t0rt1k.tech +auth.t0rt1k.tech +chat.t0rt1k.tech (already exists, points to old IP) +admin.t0rt1k.tech +rtc.t0rt1k.tech (already exists, points to old IP) +t0rt1k.tech (well-known) — already exists, update IP. call.t0rt1k.tech — remove (no longer needed). diff --git a/manifests/cert-manager/cluster-issuers.yaml b/manifests/cert-manager/cluster-issuers.yaml new file mode 100644 index 0000000..8e27ea0 --- /dev/null +++ b/manifests/cert-manager/cluster-issuers.yaml @@ -0,0 +1,29 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-production +spec: + acme: + email: i_am@rogov.al + privateKeySecretRef: + name: letsencrypt-production-account-key + server: https://acme-v02.api.letsencrypt.org/directory + solvers: + - http01: + ingress: + class: traefik +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-staging +spec: + acme: + email: i_am@rogov.al + privateKeySecretRef: + name: letsencrypt-staging-account-key + server: https://acme-staging-v02.api.letsencrypt.org/directory + solvers: + - http01: + ingress: + class: traefik diff --git a/manifests/cnpg/cluster.yaml b/manifests/cnpg/cluster.yaml new file mode 100644 index 0000000..4e837dd --- /dev/null +++ b/manifests/cnpg/cluster.yaml @@ -0,0 +1,41 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: shared-pg + namespace: cnpg +spec: + instances: 3 + imageName: ghcr.io/cloudnative-pg/postgresql:16 + + resources: + requests: + memory: 1Gi + cpu: 250m + limits: + memory: 2Gi + cpu: 1000m + + storage: + size: 20Gi + storageClass: yc-network-ssd + + affinity: + podAntiAffinityType: required + + bootstrap: + initdb: + database: postgres + owner: postgres + + postgresql: + parameters: + shared_buffers: "512MB" + effective_cache_size: "1536MB" + maintenance_work_mem: "64MB" + max_connections: "200" + work_mem: "16MB" + random_page_cost: "1.1" + effective_io_concurrency: "200" + + monitoring: + enablePodMonitor: true diff --git a/manifests/cnpg/databases.yaml b/manifests/cnpg/databases.yaml new file mode 100644 index 0000000..1a26bbc --- /dev/null +++ b/manifests/cnpg/databases.yaml @@ -0,0 +1,87 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: synapse-mrt0rtikize + namespace: cnpg +spec: + name: synapse_mrt0rtikize + owner: synapse_mrt0rtikize + cluster: + name: shared-pg +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: mas-mrt0rtikize + namespace: cnpg +spec: + name: mas_mrt0rtikize + owner: mas_mrt0rtikize + cluster: + name: shared-pg +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: synapse-t0rt1k + namespace: cnpg +spec: + name: synapse_t0rt1k + owner: synapse_t0rt1k + cluster: + name: shared-pg +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: mas-t0rt1k + namespace: cnpg +spec: + name: mas_t0rt1k + owner: mas_t0rt1k + cluster: + name: shared-pg +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: synapse-roglog + namespace: cnpg +spec: + name: synapse_roglog + owner: synapse_roglog + cluster: + name: shared-pg +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: mas-roglog + namespace: cnpg +spec: + name: mas_roglog + owner: mas_roglog + cluster: + name: shared-pg +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: synapse-uretra + namespace: cnpg +spec: + name: synapse_uretra + owner: synapse_uretra + cluster: + name: shared-pg +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: mas-uretra + namespace: cnpg +spec: + name: mas_uretra + owner: mas_uretra + cluster: + name: shared-pg diff --git a/manifests/cnpg/namespace.yaml b/manifests/cnpg/namespace.yaml new file mode 100644 index 0000000..6b7aad5 --- /dev/null +++ b/manifests/cnpg/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cnpg diff --git a/manifests/gitea/ingressroute-ssh.yaml b/manifests/gitea/ingressroute-ssh.yaml new file mode 100644 index 0000000..01bcb9f --- /dev/null +++ b/manifests/gitea/ingressroute-ssh.yaml @@ -0,0 +1,13 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteTCP +metadata: + name: gitea-ssh + namespace: gitea +spec: + entryPoints: + - ssh + routes: + - match: HostSNI(`*`) + services: + - name: gitea + port: 22 diff --git a/manifests/matrix-mrt0rtikize/ingressroute-rtc-tcp.yaml b/manifests/matrix-mrt0rtikize/ingressroute-rtc-tcp.yaml new file mode 100644 index 0000000..2fcdc09 --- /dev/null +++ b/manifests/matrix-mrt0rtikize/ingressroute-rtc-tcp.yaml @@ -0,0 +1,13 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteTCP +metadata: + name: rtc-tcp + namespace: matrix-mrt0rtikize +spec: + entryPoints: + - rtc-tcp-mrt0rtikize + routes: + - match: HostSNI(`*`) + services: + - name: matrix-mrt0rtikize-matrix-rtc-sfu-tcp + port: 30009 diff --git a/manifests/matrix-mrt0rtikize/ingressroute-rtc-udp.yaml b/manifests/matrix-mrt0rtikize/ingressroute-rtc-udp.yaml new file mode 100644 index 0000000..1bb0b7d --- /dev/null +++ b/manifests/matrix-mrt0rtikize/ingressroute-rtc-udp.yaml @@ -0,0 +1,12 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteUDP +metadata: + name: rtc-udp + namespace: matrix-mrt0rtikize +spec: + entryPoints: + - rtc-udp-mrt0rtikize + routes: + - services: + - name: matrix-mrt0rtikize-matrix-rtc-sfu-muxed-udp + port: 30008 diff --git a/manifests/matrix-roglog/ingressroute-rtc-tcp.yaml b/manifests/matrix-roglog/ingressroute-rtc-tcp.yaml new file mode 100644 index 0000000..5a800d8 --- /dev/null +++ b/manifests/matrix-roglog/ingressroute-rtc-tcp.yaml @@ -0,0 +1,13 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteTCP +metadata: + name: rtc-tcp + namespace: matrix-roglog +spec: + entryPoints: + - rtc-tcp-roglog + routes: + - match: HostSNI(`*`) + services: + - name: matrix-roglog-matrix-rtc-sfu-tcp + port: 30005 diff --git a/manifests/matrix-roglog/ingressroute-rtc-udp.yaml b/manifests/matrix-roglog/ingressroute-rtc-udp.yaml new file mode 100644 index 0000000..1379a1a --- /dev/null +++ b/manifests/matrix-roglog/ingressroute-rtc-udp.yaml @@ -0,0 +1,12 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteUDP +metadata: + name: rtc-udp + namespace: matrix-roglog +spec: + entryPoints: + - rtc-udp-roglog + routes: + - services: + - name: matrix-roglog-matrix-rtc-sfu-muxed-udp + port: 30004 diff --git a/manifests/matrix-t0rt1k/ingressroute-rtc-tcp.yaml b/manifests/matrix-t0rt1k/ingressroute-rtc-tcp.yaml new file mode 100644 index 0000000..550e807 --- /dev/null +++ b/manifests/matrix-t0rt1k/ingressroute-rtc-tcp.yaml @@ -0,0 +1,13 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteTCP +metadata: + name: rtc-tcp + namespace: matrix-t0rt1k +spec: + entryPoints: + - rtc-tcp-t0rt1k + routes: + - match: HostSNI(`*`) + services: + - name: matrix-t0rt1k-matrix-rtc-sfu-tcp + port: 30007 diff --git a/manifests/matrix-t0rt1k/ingressroute-rtc-udp.yaml b/manifests/matrix-t0rt1k/ingressroute-rtc-udp.yaml new file mode 100644 index 0000000..20dace5 --- /dev/null +++ b/manifests/matrix-t0rt1k/ingressroute-rtc-udp.yaml @@ -0,0 +1,12 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteUDP +metadata: + name: rtc-udp + namespace: matrix-t0rt1k +spec: + entryPoints: + - rtc-udp-t0rt1k + routes: + - services: + - name: matrix-t0rt1k-matrix-rtc-sfu-muxed-udp + port: 30006 diff --git a/manifests/matrix-t0rt1k/vmpodscrape-synapse.yaml b/manifests/matrix-t0rt1k/vmpodscrape-synapse.yaml new file mode 100644 index 0000000..3ff1608 --- /dev/null +++ b/manifests/matrix-t0rt1k/vmpodscrape-synapse.yaml @@ -0,0 +1,15 @@ +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: synapse + namespace: matrix-t0rt1k +spec: + namespaceSelector: + matchNames: + - matrix-t0rt1k + podMetricsEndpoints: + - port: synapse-metrics + path: /metrics + selector: + matchLabels: + app.kubernetes.io/component: matrix-server diff --git a/manifests/metrics/grafana/ingress.yaml b/manifests/metrics/grafana/ingress.yaml new file mode 100644 index 0000000..f063112 --- /dev/null +++ b/manifests/metrics/grafana/ingress.yaml @@ -0,0 +1,26 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: grafana + namespace: metrics + annotations: + cert-manager.io/cluster-issuer: letsencrypt-production + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" +spec: + ingressClassName: traefik + tls: + - hosts: + - grafana.prod.t01tt.tech + secretName: grafana-tls + rules: + - host: grafana.prod.t01tt.tech + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: monitoring-grafana + port: + number: 80 diff --git a/manifests/metrics/grafana/loki-datasource.yaml b/manifests/metrics/grafana/loki-datasource.yaml new file mode 100644 index 0000000..3ad5eb0 --- /dev/null +++ b/manifests/metrics/grafana/loki-datasource.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: loki-grafana-ds + namespace: metrics + labels: + grafana_datasource: "1" +data: + datasource.yaml: | + apiVersion: 1 + datasources: + - access: proxy + isDefault: false + name: Loki + type: loki + uid: Loki + url: http://loki.metrics.svc.cluster.local:3100 diff --git a/manifests/metrics/grafana/namespace.yaml b/manifests/metrics/grafana/namespace.yaml new file mode 100644 index 0000000..6d57933 --- /dev/null +++ b/manifests/metrics/grafana/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: metrics diff --git a/scripts/backup-mrt0rtikize.sh b/scripts/backup-mrt0rtikize.sh new file mode 100755 index 0000000..aefb7dd --- /dev/null +++ b/scripts/backup-mrt0rtikize.sh @@ -0,0 +1,244 @@ +#!/bin/bash +set -euo pipefail + +# ================================================================ +# Backup script for mrt0rtikize.ru Matrix instance (k3s cluster) +# ================================================================ +# Output: backups/mrt0rtikize-YYYYMMDD-HHMMSS/ +# +# Run this BEFORE switching DNS. Ensure TTL is already set to 60s +# on all mrt0rtikize.ru DNS records (24h before planned cutover). +# +# Steps: +# 1. Stop Synapse + MAS to prevent DB writes +# 2. Dump PostgreSQL (built-in PG from ESS chart) +# 3. Export generated secrets (CRITICAL: signing key, MAS keys) +# 4. Export deployment markers ConfigMap +# 5. Export ESS ArgoCD Application (values reference) +# 6. Export media file locations (manual restore, noted in README) +# ================================================================ + +readonly K3S_KUBECONFIG="${KUBECONFIG:-/home/mrt0rtikize/infra/k3s/config}" +readonly NS="matrix-mrt0rtikize" +readonly BACKUP_BASE="$(dirname "$(realpath "$0")")/../backups" +readonly TIMESTAMP="$(date +%Y%m%d-%H%M%S)" +readonly BACKUP_DIR="${BACKUP_BASE}/${NS}-${TIMESTAMP}" + +readonly K="${KUBECTL:-kubectl} --kubeconfig ${K3S_KUBECONFIG}" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +log() { echo -e "${GREEN}[$(date +%H:%M:%S)]${NC} $*"; } +warn() { echo -e "${YELLOW}[$(date +%H:%M:%S)] WARN:${NC} $*"; } +err() { echo -e "${RED}[$(date +%H:%M:%S)] ERROR:${NC} $*"; } + +# ------------------------------------------------------------------- +# Prerequisites +# ------------------------------------------------------------------- +log "=== Checking prerequisites ===" + +if ! ${K} get ns "${NS}" >/dev/null 2>&1; then + err "Namespace ${NS} not found on k3s cluster. Check KUBECONFIG (${K3S_KUBECONFIG})." + exit 1 +fi + +mkdir -p "${BACKUP_DIR}" +log "Backup directory: ${BACKUP_DIR}" + +# ------------------------------------------------------------------- +# Step 1: Backup Synapse media files (BEFORE stopping Synapse) +# ------------------------------------------------------------------- +log "=== Step 1: Backing up Synapse media ===" + +SYNAPSE_POD_NAME=$( ${K} -n "${NS}" get pods -l "app.kubernetes.io/component=matrix-server" -o jsonpath='{.items[?(@.status.phase=="Running")].metadata.name}' 2>/dev/null) + +if [ -z "${SYNAPSE_POD_NAME}" ]; then + warn "No running Synapse pod found. Cannot backup media." +else + log "Using Synapse pod: ${SYNAPSE_POD_NAME}" + ${K} exec -n "${NS}" "${SYNAPSE_POD_NAME}" -- tar czf /tmp/synapse-media.tar.gz -C /media media_store/ + ${K} cp "${NS}/${SYNAPSE_POD_NAME}:/tmp/synapse-media.tar.gz" "${BACKUP_DIR}/synapse-media.tar.gz" + ${K} exec -n "${NS}" "${SYNAPSE_POD_NAME}" -- rm /tmp/synapse-media.tar.gz + log "Synapse media saved: ${BACKUP_DIR}/synapse-media.tar.gz ($(du -h "${BACKUP_DIR}/synapse-media.tar.gz" | cut -f1))" +fi + +# ------------------------------------------------------------------- +# Step 2: Stop Synapse + MAS (start downtime window) +# ------------------------------------------------------------------- +log "=== Step 2: Stopping Synapse + MAS ===" + +SYNAPSE_READY=$(${K} -n "${NS}" get sts "${NS}-synapse-main" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0") +MAS_READY=$(${K} -n "${NS}" get deploy "${NS}-matrix-authentication-service" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0") + +if [ "${SYNAPSE_READY}" != "0" ]; then + log "Scaling Synapse to 0..." + ${K} -n "${NS}" scale sts "${NS}-synapse-main" --replicas=0 +else + log "Synapse already scaled to 0." +fi + +if [ "${MAS_READY}" != "0" ]; then + log "Scaling MAS to 0..." + ${K} -n "${NS}" scale deploy "${NS}-matrix-authentication-service" --replicas=0 +else + log "MAS already scaled to 0." +fi + +log "Waiting for Synapse + MAS pods to terminate..." +${K} -n "${NS}" wait --for=delete pod -l "app.kubernetes.io/component=matrix-server" --timeout=120s 2>/dev/null || warn "Some Synapse pods may still be terminating." +${K} -n "${NS}" wait --for=delete pod -l "app.kubernetes.io/component=matrix-authentication" --timeout=120s 2>/dev/null || warn "Some MAS pods may still be terminating." + +log "Synapse + MAS stopped." + +# ------------------------------------------------------------------- +# Step 3: Dump PostgreSQL +# ------------------------------------------------------------------- +log "=== Step 3: Dumping PostgreSQL ===" + +PG_POD=$(${K} -n "${NS}" get pods -l "app.kubernetes.io/name=postgres" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) +if [ -z "${PG_POD}" ]; then + PG_POD=$(${K} -n "${NS}" get pods -o name 2>/dev/null | grep postgres | head -1 | cut -d/ -f2) +fi + +if [ -z "${PG_POD}" ]; then + err "Could not find PostgreSQL pod in namespace ${NS}." + err "Available pods:" + ${K} -n "${NS}" get pods + exit 1 +fi + +log "Using PostgreSQL pod: ${PG_POD}" + +${K} exec -n "${NS}" "${PG_POD}" -- pg_dumpall -U postgres > "${BACKUP_DIR}/dump-all.sql" +log "PostgreSQL dump saved: ${BACKUP_DIR}/dump-all.sql ($(wc -c < "${BACKUP_DIR}/dump-all.sql") bytes)" + +${K} exec -n "${NS}" "${PG_POD}" -- pg_dump -U postgres -d synapse > "${BACKUP_DIR}/dump-synapse.sql" 2>/dev/null || \ + warn "Could not dump synapse DB individually (will use dump-all.sql for restore)." + +${K} exec -n "${NS}" "${PG_POD}" -- pg_dump -U postgres -d matrixauthenticationservice > "${BACKUP_DIR}/dump-mas.sql" 2>/dev/null || \ + warn "Could not dump MAS DB individually (will use dump-all.sql for restore)." + +# ------------------------------------------------------------------- +# Step 4: Export generated secrets (CRITICAL) +# ------------------------------------------------------------------- +log "=== Step 4: Exporting generated secrets ===" + +if ${K} -n "${NS}" get secret "${NS}-generated" >/dev/null 2>&1; then + ${K} -n "${NS}" get secret "${NS}-generated" -o yaml > "${BACKUP_DIR}/secret-generated.yaml" + log "Generated secret saved: ${BACKUP_DIR}/secret-generated.yaml" +else + err "CRITICAL: ${NS}-generated secret NOT FOUND!" + err "This contains SYNAPSE_SIGNING_KEY, MAS keys, and MACAROON." + err "Without it, federation identity is lost and all rooms break." + err "Available secrets:" + ${K} -n "${NS}" get secrets + exit 1 +fi + +log "Secret contents (verify these exist):" +${K} -n "${NS}" get secret "${NS}-generated" -o jsonpath='{.data}' | python3 -c " +import json, sys +keys = ['SYNAPSE_SIGNING_KEY', 'MAS_ENCRYPTION_SECRET', 'MAS_RSA_PRIVATE_KEY', + 'SYNAPSE_MACAROON', 'MAS_SYNAPSE_SHARED_SECRET', + 'POSTGRES_SYNAPSE_PASSWORD', 'POSTGRES_MATRIX_AUTHENTICATION_SERVICE_PASSWORD'] +d = json.load(sys.stdin) +for k in keys: + present = 'OK' if k in d else 'MISSING!' + print(f' {k}: {present}') +" 2>/dev/null || warn "Could not verify secret keys." + +# ------------------------------------------------------------------- +# Step 5: Export deployment markers +# ------------------------------------------------------------------- +log "=== Step 5: Exporting deployment markers ===" + +MARKER_CM=$(${K} -n "${NS}" get cm -l "app.kubernetes.io/managed-by=matrix-tools-deployment-markers" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) +if [ -n "${MARKER_CM}" ]; then + ${K} -n "${NS}" get cm "${MARKER_CM}" -o yaml > "${BACKUP_DIR}/markers.yaml" + log "Deployment markers saved: ${BACKUP_DIR}/markers.yaml" +else + warn "No deployment markers ConfigMap found (non-critical, ESS may regenerate)." +fi + +# ------------------------------------------------------------------- +# Step 6: Export ESS values from ArgoCD +# ------------------------------------------------------------------- +log "=== Step 6: Exporting ESS ArgoCD application ===" + +if ${K} -n argocd get application "${NS}" >/dev/null 2>&1; then + ${K} -n argocd get application "${NS}" -o yaml > "${BACKUP_DIR}/argo-app.yaml" + log "ArgoCD Application saved: ${BACKUP_DIR}/argo-app.yaml" +else + warn "ArgoCD Application '${NS}' not found (running without ArgoCD?)." + warn "Save your ESS values manually from helm get values or git." +fi + +# ------------------------------------------------------------------- +# Write README +# ------------------------------------------------------------------- +log "=== Writing README ===" + +cat > "${BACKUP_DIR}/README.txt" << READEOF +Backup for ${NS} — ${TIMESTAMP} +================================== + +Files: + dump-all.sql Full PostgreSQL dump (pg_dumpall) + secret-generated.yaml CRITICAL: contains SYNAPSE_SIGNING_KEY, MAS keys, MACAROON + markers.yaml Deployment markers ConfigMap (ESS state tracking) + argo-app.yaml ESS ArgoCD Application (values reference) + synapse-media.tar.gz Synapse media files (local + remote content) + dump-synapse.sql Synapse DB only (optional, for easier restore) + dump-mas.sql MAS DB only (optional, for easier restore) + README.txt This file + +CRITICAL: Do NOT lose secret-generated.yaml. + - SYNAPSE_SIGNING_KEY identifies this server to the Matrix federation. + Changing it breaks all existing rooms and federation relationships. + - MAS_ENCRYPTION_SECRET encrypts user sessions. + Changing it forces all users to re-login. + - SYNAPSE_MACAROON is the admin API token. + +Restore order on new cluster: + 1. Create CNPG databases + secrets on new cluster (see PLAN.md) + 2. Deploy ESS chart on new cluster (starts with empty DB) + 3. Stop Synapse + MAS on new cluster + 4. Restore PG dump into CNPG shared-pg + 5. Apply this secret-generated.yaml to new cluster's namespace + 6. Apply markers.yaml + 7. Restore media: kubectl cp synapse-media.tar.gz to new Synapse pod, untar to /media/ + 8. Restart Synapse + MAS on new cluster + 9. Verify: login, federation tester, Element Call + 10. Cut DNS to new NLB IP +READEOF + +log "README saved: ${BACKUP_DIR}/README.txt" + +# ------------------------------------------------------------------- +# Summary +# ------------------------------------------------------------------- +cat < [--no-downtime] +# Example: ./backup-yc-playground.sh t0rt1k +# ./backup-yc-playground.sh t0rt1k --no-downtime (test, no scaling) +# ================================================================ + +readonly YC_CONTEXT="yc-playground" +readonly K="${KUBECTL:-kubectl} --context ${YC_CONTEXT}" +readonly BACKUP_BASE="$(dirname "$(realpath "$0")")/../backups" +readonly TIMESTAMP="$(date +%Y%m%d-%H%M%S)" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' +log() { echo -e "${GREEN}[$(date +%H:%M:%S)]${NC} $*"; } +warn() { echo -e "${YELLOW}[$(date +%H:%M:%S)] WARN:${NC} $*"; } +err() { echo -e "${RED}[$(date +%H:%M:%S)] ERROR:${NC} $*"; } + +# ------------------------------------------------------------------- +# Parse args +# ------------------------------------------------------------------- +DOWNTIME=true +NAME="" +for arg in "$@"; do + case "$arg" in + --no-downtime) DOWNTIME=false ;; + *) NAME="$arg" ;; + esac +done + +if [ -z "${NAME}" ]; then + echo "Usage: $0 [--no-downtime]" + echo " name: t0rt1k | roglog | uretra" + echo " --no-downtime: skip scale up/down (test run)" + exit 1 +fi + +readonly NS="matrix-${NAME}" +readonly BACKUP_DIR="${BACKUP_BASE}/matrix-${NAME}-${TIMESTAMP}" + +# Synapse labels +readonly SYNAPSE_LABEL="app.kubernetes.io/instance=chat,app.kubernetes.io/name=matrix" +# MAS labels +readonly MAS_LABEL="app=mas" +readonly MAS_PG_LABEL="app=mas-postgresql" + +# ------------------------------------------------------------------- +# Prerequisites +# ------------------------------------------------------------------- +log "=== Backing up ${NS} (downtime=${DOWNTIME}) ===" + +if ! ${K} get ns "${NS}" >/dev/null 2>&1; then + err "Namespace ${NS} not found on yc-playground." + exit 1 +fi + +mkdir -p "${BACKUP_DIR}" +log "Backup directory: ${BACKUP_DIR}" + +# ------------------------------------------------------------------- +# Step 1: Backup Synapse media +# ------------------------------------------------------------------- +log "=== Step 1: Backing up Synapse media ===" + +SYNAPSE_POD=$(${K} -n "${NS}" get pods -l "${SYNAPSE_LABEL}" -o jsonpath='{.items[?(@.status.phase=="Running")].metadata.name}' 2>/dev/null) + +if [ -n "${SYNAPSE_POD}" ]; then + log "Synapse pod: ${SYNAPSE_POD}" + # Check if media directory exists + if ${K} exec -n "${NS}" "${SYNAPSE_POD}" -- test -d /data/media_store 2>/dev/null; then + ${K} exec -n "${NS}" "${SYNAPSE_POD}" -- tar czf /tmp/synapse-media.tar.gz -C /data media_store/ + ${K} cp "${NS}/${SYNAPSE_POD}:/tmp/synapse-media.tar.gz" "${BACKUP_DIR}/synapse-media.tar.gz" 2>/dev/null || \ + ${K} cp "${NS}/${SYNAPSE_POD}:tmp/synapse-media.tar.gz" "${BACKUP_DIR}/synapse-media.tar.gz" + ${K} exec -n "${NS}" "${SYNAPSE_POD}" -- rm -f /tmp/synapse-media.tar.gz + log "Media saved: $(du -h "${BACKUP_DIR}/synapse-media.tar.gz" | cut -f1)" + else + warn "/data/media_store not found in Synapse pod." + fi +else + warn "No running Synapse pod found — skipping media backup." +fi + +# ------------------------------------------------------------------- +# Step 2: Stop Synapse + MAS +# ------------------------------------------------------------------- +if ${DOWNTIME}; then + log "=== Step 2: Stopping Synapse + MAS ===" + + SYNAPSE_READY=$(${K} -n "${NS}" get deploy chat-matrix -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0") + MAS_READY=$(${K} -n "${NS}" get deploy mas -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0") + + if [ "${SYNAPSE_READY}" != "0" ]; then + ${K} -n "${NS}" scale deploy chat-matrix --replicas=0 + log "Synapse scaled to 0." + else + log "Synapse already scaled to 0." + fi + + if [ "${MAS_READY}" != "0" ]; then + ${K} -n "${NS}" scale deploy mas --replicas=0 + log "MAS scaled to 0." + else + log "MAS already scaled to 0." + fi + + ${K} -n "${NS}" wait --for=delete pod -l "${SYNAPSE_LABEL}" --timeout=120s 2>/dev/null || warn "Synapse may still be terminating." + ${K} -n "${NS}" wait --for=delete pod -l "${MAS_LABEL}" --timeout=120s 2>/dev/null || warn "MAS may still be terminating." + log "Synapse + MAS stopped." +else + log "=== Step 2: Skipping downtime (--no-downtime) ===" +fi + +# ------------------------------------------------------------------- +# Step 3: Dump Synapse PostgreSQL +# ------------------------------------------------------------------- +log "=== Step 3: Dumping Synapse PostgreSQL ===" + +# Get the postgres password from the secret +SYNAPSE_PG_PW=$(${K} -n "${NS}" get secret chat-postgresql -o jsonpath='{.data.postgres-password}' 2>/dev/null | base64 -d) + +if [ -z "${SYNAPSE_PG_PW}" ]; then + SYNAPSE_PG_PW=$(${K} -n "${NS}" get secret chat-postgresql -o jsonpath='{.data.password}' 2>/dev/null | base64 -d) +fi + +if [ -z "${SYNAPSE_PG_PW}" ]; then + warn "Could not read chat-postgresql secret — trying env var from pod." + SYNAPSE_PG_PW=$(${K} exec -n "${NS}" chat-postgresql-0 -c postgresql -- bash -c 'echo $POSTGRES_POSTGRES_PASSWORD' 2>/dev/null) +fi + +${K} exec -n "${NS}" chat-postgresql-0 -c postgresql -- bash -c "env PGPASSWORD='${SYNAPSE_PG_PW}' pg_dump -U postgres -d matrix -f /bitnami/postgresql/data/dump.sql" 2>&1 +${K} cp "${NS}/chat-postgresql-0:/bitnami/postgresql/data/dump.sql" "${BACKUP_DIR}/dump-synapse.sql" 2>/dev/null || true +${K} exec -n "${NS}" chat-postgresql-0 -c postgresql -- rm -f /bitnami/postgresql/data/dump.sql +log "Synapse dump saved: ${BACKUP_DIR}/dump-synapse.sql ($(wc -c < "${BACKUP_DIR}/dump-synapse.sql") bytes)" + +# ------------------------------------------------------------------- +# Step 4: Dump MAS PostgreSQL +# ------------------------------------------------------------------- +log "=== Step 4: Dumping MAS PostgreSQL ===" + +MAS_PG_POD=$(${K} -n "${NS}" get pods -l "${MAS_PG_LABEL}" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + +if [ -n "${MAS_PG_POD}" ]; then + MAS_PG_PW=$(${K} exec -n "${NS}" "${MAS_PG_POD}" -c postgresql -- bash -c 'echo $POSTGRESQL_PASSWORD' 2>/dev/null) + MAS_DB=$(${K} exec -n "${NS}" "${MAS_PG_POD}" -c postgresql -- bash -c 'echo $POSTGRESQL_DATABASE' 2>/dev/null) + + if [ -n "${MAS_PG_PW}" ]; then + ${K} exec -n "${NS}" "${MAS_PG_POD}" -c postgresql -- bash -c "env PGPASSWORD='${MAS_PG_PW}' pg_dump -U '${MAS_DB:-mas}' -d '${MAS_DB:-mas}' -f /bitnami/postgresql/data/dump.sql" 2>&1 + ${K} cp "${NS}/${MAS_PG_POD}:/bitnami/postgresql/data/dump.sql" "${BACKUP_DIR}/dump-mas.sql" 2>/dev/null || true + ${K} exec -n "${NS}" "${MAS_PG_POD}" -c postgresql -- rm -f /bitnami/postgresql/data/dump.sql + log "MAS dump saved: ${BACKUP_DIR}/dump-mas.sql ($(wc -c < "${BACKUP_DIR}/dump-mas.sql") bytes)" + else + warn "Could not read MAS PG password — skipping MAS dump." + fi +else + warn "No MAS PG pod found — skipping MAS dump." +fi + +# ------------------------------------------------------------------- +# Step 5: Export secrets +# ------------------------------------------------------------------- +log "=== Step 5: Exporting secrets ===" + +for secret in chat-matrix chat-postgresql mas matrixrtc-livekit; do + if ${K} -n "${NS}" get secret "${secret}" >/dev/null 2>&1; then + ${K} -n "${NS}" get secret "${secret}" -o yaml > "${BACKUP_DIR}/secret-${secret}.yaml" + log "Secret saved: secret-${secret}.yaml" + else + warn "Secret '${secret}' not found." + fi +done + +# ------------------------------------------------------------------- +# Step 6: Restart Synapse + MAS +# ------------------------------------------------------------------- +if ${DOWNTIME}; then + log "=== Step 6: Restarting Synapse + MAS ===" + ${K} -n "${NS}" scale deploy chat-matrix --replicas=1 2>/dev/null || warn "Could not scale Synapse." + ${K} -n "${NS}" scale deploy mas --replicas=1 2>/dev/null || warn "Could not scale MAS." + log "Synapse + MAS restarted." +else + log "=== Step 6: Skipped (--no-downtime) ===" +fi + +# ------------------------------------------------------------------- +# Step 7: Write README +# ------------------------------------------------------------------- +log "=== Step 7: Writing README ===" + +cat > "${BACKUP_DIR}/README.txt" << READEOF +Backup for ${NS} — ${TIMESTAMP} +================================== + +Instance: ${NAME} (namespace: ${NS}) +Chart: matrix-2.9.17 (old chart, NOT ESS) +Dump PG: Synapse (chat-postgresql) + MAS (mas-postgresql) + +Files: + dump-synapse.sql Synapse PostgreSQL dump (pg_dump -U postgres -d matrix) + dump-mas.sql MAS PostgreSQL dump (pg_dump -U mas -d mas) + synapse-media.tar.gz Synapse media files (/data/media_store/) + secret-chat-matrix.yaml Synapse secrets (signing.key, macaroon, etc.) + secret-chat-postgresql.yaml PG passwords + secret-mas.yaml MAS secrets (encryption-key, signing-key, shared-secret) + secret-matrixrtc-livekit.yaml LiveKit secrets + +CRITICAL for restore: + - secret-chat-matrix.yaml (SYNAPSE_SIGNING_KEY for federation identity) + - secret-mas.yaml (MAS encryption-key for user sessions) + - dump-synapse.sql (all user data, rooms, messages) + - dump-mas.sql (MAS user auth data) + +Migration to ESS chart notes: + - The old chart uses separate Helm releases per component (chat, element-call, livekit). + - ESS bundles everything into the matrix-stack chart. + - PostgreSQL is external (CNPG) on the new cluster. + - MAS keys must be restored EXACTLY as-is for user auth to work. + - The Synapse signing.key MUST match the federation identity. +READEOF + +log "README saved." + +# ------------------------------------------------------------------- +# Summary +# ------------------------------------------------------------------- +cat < +# +# Steps: +# 1. Scale Synapse + MAS to 0 on new cluster +# 2. Restore PostgreSQL dumps to CNPG shared-pg +# 3. Apply generated secrets (signing key, MAS keys, macaroon) +# 4. Apply deployment markers +# 5. Scale Synapse to 1 (now has signing key + DB) +# 6. Restore media files +# 7. Scale MAS to 1 +# ================================================================ + +readonly YC_KUBECONFIG="${KUBECONFIG:-/home/mrt0rtikize/infra/yandex-prod/kubeconfig}" +readonly K="${KUBECTL:-kubectl} --kubeconfig ${YC_KUBECONFIG}" + +readonly NS="matrix-mrt0rtikize" +readonly CNPG_NS="cnpg" +readonly CNPG_POD="shared-pg-1" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +log() { echo -e "${GREEN}[$(date +%H:%M:%S)]${NC} $*"; } +warn() { echo -e "${YELLOW}[$(date +%H:%M:%S)] WARN:${NC} $*"; } +err() { echo -e "${RED}[$(date +%H:%M:%S)] ERROR:${NC} $*"; } + +# ------------------------------------------------------------------- +# Usage +# ------------------------------------------------------------------- +if [ $# -ne 1 ]; then + echo "Usage: $0 " + echo "" + echo "Example: $0 backups/matrix-mrt0rtikize-20260613-192010/" + exit 1 +fi + +readonly BACKUP_DIR="$1" + +if [ ! -d "${BACKUP_DIR}" ]; then + err "Backup directory not found: ${BACKUP_DIR}" + exit 1 +fi + +# Verify critical files exist +for f in dump-synapse.sql dump-mas.sql secret-generated.yaml markers.yaml synapse-media.tar.gz; do + if [ ! -f "${BACKUP_DIR}/${f}" ]; then + warn "Missing: ${f}" + fi +done + +# ------------------------------------------------------------------- +# Prerequisites +# ------------------------------------------------------------------- +log "=== Checking prerequisites ===" + +if ! ${K} get ns "${NS}" >/dev/null 2>&1; then + err "Namespace ${NS} not found on new cluster. Deploy the ESS app first." + exit 1 +fi + +if ! ${K} -n "${CNPG_NS}" get pod "${CNPG_POD}" >/dev/null 2>&1; then + err "CNPG pod ${CNPG_POD} not found. Is the CNPG cluster running?" + exit 1 +fi + +log "Cluster access verified." + +# ------------------------------------------------------------------- +# Step 1: Scale Synapse + MAS to 0 +# ------------------------------------------------------------------- +log "=== Step 1: Stopping Synapse + MAS ===" + +${K} -n "${NS}" scale sts -l "app.kubernetes.io/component=matrix-server" --replicas=0 2>/dev/null || \ + ${K} -n "${NS}" scale sts "${NS}-synapse-main" --replicas=0 2>/dev/null || \ + warn "Could not scale Synapse via known names, trying by label..." + +${K} -n "${NS}" scale deploy -l "app.kubernetes.io/component=matrix-authentication" --replicas=0 2>/dev/null || \ + ${K} -n "${NS}" scale deploy "${NS}-matrix-authentication-service" --replicas=0 2>/dev/null || \ + warn "Could not scale MAS via known names..." + +log "Waiting for Synapse + MAS pods to terminate..." +${K} -n "${NS}" wait --for=delete pod -l "app.kubernetes.io/component=matrix-server" --timeout=120s 2>/dev/null || warn "Some Synapse pods may still be terminating." +${K} -n "${NS}" wait --for=delete pod -l "app.kubernetes.io/component=matrix-authentication" --timeout=120s 2>/dev/null || warn "Some MAS pods may still be terminating." + +log "Synapse + MAS stopped." + +# ------------------------------------------------------------------- +# Step 2: Clean schemas (DROP SCHEMA CASCADE — no connection races) +# ------------------------------------------------------------------- +log "=== Step 2: Cleaning database schemas ===" + +log "Reading PG credentials from cluster..." +SYNAPSE_PW=$(${K} get secret pg-creds -n "${NS}" -o jsonpath='{.data.synapse}' | base64 -d) +MAS_PW=$(${K} get secret pg-creds -n "${NS}" -o jsonpath='{.data.mas}' | base64 -d) + +log "Wiping synapse schema..." +${K} exec -n "${CNPG_NS}" "${CNPG_POD}" -- env PGPASSWORD="${SYNAPSE_PW}" \ + psql -U synapse_mrt0rtikize -d synapse_mrt0rtikize -h localhost -c \ + "DROP SCHEMA public CASCADE; CREATE SCHEMA public; GRANT ALL ON SCHEMA public TO synapse_mrt0rtikize;" 2>/dev/null || true + +log "Wiping MAS schema..." +${K} exec -n "${CNPG_NS}" "${CNPG_POD}" -- env PGPASSWORD="${MAS_PW}" \ + psql -U mas_mrt0rtikize -d mas_mrt0rtikize -h localhost -c \ + "DROP SCHEMA public CASCADE; CREATE SCHEMA public; GRANT ALL ON SCHEMA public TO mas_mrt0rtikize;" 2>/dev/null || true + +log "Schemas cleaned." + +# ------------------------------------------------------------------- +# Step 3: Restore PostgreSQL dumps +# ------------------------------------------------------------------- +log "=== Step 3: Restoring PostgreSQL dumps ===" + +log "Restoring Synapse database..." +${K} exec -i -n "${CNPG_NS}" "${CNPG_POD}" -- env PGPASSWORD="${SYNAPSE_PW}" \ + psql -U synapse_mrt0rtikize -d synapse_mrt0rtikize -h localhost < "${BACKUP_DIR}/dump-synapse.sql" +log "Synapse database restored." + +log "Restoring MAS database..." +${K} exec -i -n "${CNPG_NS}" "${CNPG_POD}" -- env PGPASSWORD="${MAS_PW}" \ + psql -U mas_mrt0rtikize -d mas_mrt0rtikize -h localhost < "${BACKUP_DIR}/dump-mas.sql" +log "MAS database restored." + +# ------------------------------------------------------------------- +# Step 3: Apply generated secrets (CRITICAL) +# ------------------------------------------------------------------- +log "=== Step 4: Applying generated secrets ===" + +if [ -f "${BACKUP_DIR}/secret-generated.yaml" ]; then + ${K} replace --force -f "${BACKUP_DIR}/secret-generated.yaml" + log "Generated secret replaced: ${NS}-generated (original signing key from backup)" +else + err "CRITICAL: secret-generated.yaml not found in backup!" + err "SYNAPSE_SIGNING_KEY and MAS keys will NOT be restored." + err "Federation identity is lost without this file." +fi + +# ------------------------------------------------------------------- +# Step 4: Apply deployment markers +# ------------------------------------------------------------------- +log "=== Step 5: Applying deployment markers ===" + +if [ -f "${BACKUP_DIR}/markers.yaml" ]; then + ${K} replace --force -f "${BACKUP_DIR}/markers.yaml" + log "Deployment markers replaced." +else + warn "markers.yaml not found in backup (non-critical)." +fi + +# ------------------------------------------------------------------- +# Step 5: Scale Synapse to 1 +# ------------------------------------------------------------------- +log "=== Step 6: Starting Synapse ===" + +${K} -n "${NS}" scale sts -l "app.kubernetes.io/component=matrix-server" --replicas=1 2>/dev/null || \ + ${K} -n "${NS}" scale sts "${NS}-synapse-main" --replicas=1 2>/dev/null + +log "Waiting for Synapse to start..." +${K} -n "${NS}" wait --for=condition=ready pod -l "app.kubernetes.io/component=matrix-server" --timeout=300s 2>/dev/null || warn "Synapse is not ready yet, check logs." + +# ------------------------------------------------------------------- +# Step 6: Restore media files +# ------------------------------------------------------------------- +log "=== Step 7: Restoring media files ===" + +if [ -f "${BACKUP_DIR}/synapse-media.tar.gz" ]; then + SYNAPSE_POD=$( ${K} -n "${NS}" get pods -l "app.kubernetes.io/component=matrix-server" -o jsonpath='{.items[?(@.status.phase=="Running")].metadata.name}' 2>/dev/null) + if [ -z "${SYNAPSE_POD}" ]; then + warn "No running Synapse pod found for media restore. Skip media." + warn "Re-run this step after Synapse is up:" + warn " kubectl cp synapse-media.tar.gz ${NS}/:/tmp/ && kubectl exec -- tar xzf /tmp/synapse-media.tar.gz -C /media/" + else + log "Copying media to Synapse pod: ${SYNAPSE_POD}" + ${K} cp "${BACKUP_DIR}/synapse-media.tar.gz" "${NS}/${SYNAPSE_POD}:/tmp/synapse-media.tar.gz" + ${K} exec -n "${NS}" "${SYNAPSE_POD}" -- tar xzf /tmp/synapse-media.tar.gz -C /media/ + ${K} exec -n "${NS}" "${SYNAPSE_POD}" -- rm /tmp/synapse-media.tar.gz + log "Media files restored." + fi +else + warn "synapse-media.tar.gz not found in backup." +fi + +# ------------------------------------------------------------------- +# Step 7: Scale MAS to 1 +# ------------------------------------------------------------------- +log "=== Step 8: Starting MAS ===" + +${K} -n "${NS}" scale deploy -l "app.kubernetes.io/component=matrix-authentication" --replicas=1 2>/dev/null || \ + ${K} -n "${NS}" scale deploy "${NS}-matrix-authentication-service" --replicas=1 2>/dev/null + +log "Waiting for MAS to start..." +${K} -n "${NS}" wait --for=condition=ready pod -l "app.kubernetes.io/component=matrix-authentication" --timeout=120s 2>/dev/null || warn "MAS is not ready yet, check logs." + +# ------------------------------------------------------------------- +# Summary +# ------------------------------------------------------------------- +cat < +# Example: ./restore-prod.sh t0rt1k backups/matrix-t0rt1k-20260614-161207/ +# ================================================================ + +readonly YC_KUBECONFIG="${KUBECONFIG:-/home/mrt0rtikize/infra/yandex-prod/kubeconfig}" +readonly K="${KUBECTL:-kubectl} --kubeconfig ${YC_KUBECONFIG}" + +readonly CNPG_NS="cnpg" +readonly CNPG_POD="shared-pg-1" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' +log() { echo -e "${GREEN}[$(date +%H:%M:%S)]${NC} $*"; } +warn() { echo -e "${YELLOW}[$(date +%H:%M:%S)] WARN:${NC} $*"; } +err() { echo -e "${RED}[$(date +%H:%M:%S)] ERROR:${NC} $*"; } + +# ------------------------------------------------------------------- +# Parse args +# ------------------------------------------------------------------- +if [ $# -ne 2 ]; then + echo "Usage: $0 " + echo "" + echo " name: t0rt1k | roglog | uretra" + echo " backup-dir: path to backup directory from backup-yc-playground.sh" + echo "" + echo "Example: $0 t0rt1k backups/matrix-t0rt1k-20260614-161207/" + exit 1 +fi + +readonly NAME="$1" +readonly NS="matrix-${NAME}" +readonly DB_SYNAPSE="synapse_${NAME}" +readonly DB_MAS="mas_${NAME}" +readonly BACKUP_DIR="$2" + +# ------------------------------------------------------------------- +# Prerequisites +# ------------------------------------------------------------------- +log "=== Restoring ${NS} from ${BACKUP_DIR} ===" + +if [ ! -d "${BACKUP_DIR}" ]; then + err "Backup directory not found: ${BACKUP_DIR}" + exit 1 +fi + +for f in dump-synapse.sql dump-mas.sql secret-chat-matrix.yaml secret-mas.yaml secret-chat-postgresql.yaml; do + if [ ! -f "${BACKUP_DIR}/${f}" ]; then + err "Missing: ${f}" + exit 1 + fi +done + +if ! ${K} get ns "${NS}" >/dev/null 2>&1; then + err "Namespace ${NS} not found. Deploy the ESS app and sync first." + exit 1 +fi + +if ! ${K} -n "${CNPG_NS}" get pod "${CNPG_POD}" >/dev/null 2>&1; then + err "CNPG pod ${CNPG_POD} not found." + exit 1 +fi + +log "Prerequisites OK." + +# ------------------------------------------------------------------- +# Step 1: Stop Synapse + MAS +# ------------------------------------------------------------------- +log "=== Step 1: Stopping Synapse + MAS ===" + +${K} -n "${NS}" scale deploy -l "app.kubernetes.io/component=matrix-server" --replicas=0 2>/dev/null || \ + ${K} -n "${NS}" scale sts "${NS}-synapse-main" --replicas=0 2>/dev/null || \ + ${K} -n "${NS}" scale deploy chat-matrix --replicas=0 2>/dev/null || \ + warn "Could not scale Synapse." + +${K} -n "${NS}" scale deploy -l "app.kubernetes.io/component=matrix-authentication" --replicas=0 2>/dev/null || \ + ${K} -n "${NS}" scale deploy "${NS}-matrix-authentication-service" --replicas=0 2>/dev/null || \ + ${K} -n "${NS}" scale deploy mas --replicas=0 2>/dev/null || \ + warn "Could not scale MAS." + +log "Waiting for pods to terminate..." +${K} -n "${NS}" wait --for=delete pod -l "app.kubernetes.io/component=matrix-server" --timeout=120s 2>/dev/null || warn "Synapse may still be terminating." +${K} -n "${NS}" wait --for=delete pod -l "app.kubernetes.io/component=matrix-authentication" --timeout=120s 2>/dev/null || warn "MAS may still be terminating." +${K} -n "${NS}" wait --for=delete pod -l "app=mas" --timeout=60s 2>/dev/null || true + +log "Synapse + MAS stopped." + +# ------------------------------------------------------------------- +# Step 2: Clean schemas +# ------------------------------------------------------------------- +log "=== Step 2: Cleaning database schemas ===" + +log "Reading PG credentials..." +SYNAPSE_PW=$(${K} get secret pg-creds -n "${NS}" -o jsonpath='{.data.synapse}' 2>/dev/null | base64 -d) +MAS_PW=$(${K} get secret pg-creds -n "${NS}" -o jsonpath='{.data.mas}' 2>/dev/null | base64 -d) + +log "Wiping ${DB_SYNAPSE} schema..." +${K} exec -n "${CNPG_NS}" "${CNPG_POD}" -- env PGPASSWORD="${SYNAPSE_PW}" \ + psql -U "${DB_SYNAPSE}" -d "${DB_SYNAPSE}" -h localhost -c \ + "DROP SCHEMA public CASCADE; CREATE SCHEMA public; GRANT ALL ON SCHEMA public TO ${DB_SYNAPSE};" 2>/dev/null || true + +log "Wiping ${DB_MAS} schema..." +${K} exec -n "${CNPG_NS}" "${CNPG_POD}" -- env PGPASSWORD="${MAS_PW}" \ + psql -U "${DB_MAS}" -d "${DB_MAS}" -h localhost -c \ + "DROP SCHEMA public CASCADE; CREATE SCHEMA public; GRANT ALL ON SCHEMA public TO ${DB_MAS};" 2>/dev/null || true + +log "Schemas cleaned." + +# ------------------------------------------------------------------- +# Step 3: Restore PostgreSQL dumps +# ------------------------------------------------------------------- +log "=== Step 3: Restoring PostgreSQL dumps ===" + +log "Restoring Synapse database..." +# Old chart used 'matrix' role — create it as superuser to suppress OWNER TO errors +${K} exec -n "${CNPG_NS}" "${CNPG_POD}" -- psql -U postgres -c \ + "DO \$\$ BEGIN IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'matrix') THEN CREATE ROLE matrix; END IF; END \$\$;" 2>/dev/null || true +${K} exec -n "${CNPG_NS}" "${CNPG_POD}" -- psql -U postgres -c \ + "GRANT matrix TO ${DB_SYNAPSE};" 2>/dev/null || true + +${K} exec -i -n "${CNPG_NS}" "${CNPG_POD}" -- env PGPASSWORD="${SYNAPSE_PW}" \ + psql -U "${DB_SYNAPSE}" -d "${DB_SYNAPSE}" -h localhost < "${BACKUP_DIR}/dump-synapse.sql" +log "Synapse database restored." + +log "Restoring MAS database..." +# Old chart used 'mas' role — create it as superuser to suppress OWNER TO errors +${K} exec -n "${CNPG_NS}" "${CNPG_POD}" -- psql -U postgres -c \ + "DO \$\$ BEGIN IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'mas') THEN CREATE ROLE mas; END IF; END \$\$;" 2>/dev/null || true +${K} exec -n "${CNPG_NS}" "${CNPG_POD}" -- psql -U postgres -c \ + "GRANT mas TO ${DB_MAS};" 2>/dev/null || true +${K} exec -i -n "${CNPG_NS}" "${CNPG_POD}" -- env PGPASSWORD="${MAS_PW}" \ + psql -U "${DB_MAS}" -d "${DB_MAS}" -h localhost < "${BACKUP_DIR}/dump-mas.sql" +log "MAS database restored." + +# ------------------------------------------------------------------- +# Step 4: Construct and apply the ESS -generated secret +# ------------------------------------------------------------------- +log "=== Step 4: Constructing ESS generated secret ===" + +# Read existing auto-generated secret (has keys the old chart didn't: ECDSA, LiveKit, Hookshot, etc.) +EXISTING_KEY_ECDSA="" +EXISTING_KEY_LIVEKIT="" +EXISTING_KEY_HOOKSHOT_REG="" +EXISTING_KEY_HOOKSHOT_PASS="" +EXISTING_KEY_SYNAPSE_EXTRA="" +EXISTING_KEY_PG_ADMIN="" + +if ${K} -n "${NS}" get secret "${NS}-generated" >/dev/null 2>&1; then + log "Reading existing auto-generated secret for non-migrated keys..." + EXISTING_KEY_ECDSA=$(${K} -n "${NS}" get secret "${NS}-generated" -o jsonpath='{.data.MAS_ECDSA_PRIME256V1_PRIVATE_KEY}' 2>/dev/null || echo "") + EXISTING_KEY_LIVEKIT=$(${K} -n "${NS}" get secret "${NS}-generated" -o jsonpath='{.data.ELEMENT_CALL_LIVEKIT_SECRET}' 2>/dev/null || echo "") + EXISTING_KEY_HOOKSHOT_REG=$(${K} -n "${NS}" get secret "${NS}-generated" -o jsonpath='{.data.HOOKSHOT_REGISTRATION}' 2>/dev/null || echo "") + EXISTING_KEY_HOOKSHOT_PASS=$(${K} -n "${NS}" get secret "${NS}-generated" -o jsonpath='{.data.HOOKSHOT_RSA_PASSKEY}' 2>/dev/null || echo "") + EXISTING_KEY_SYNAPSE_EXTRA=$(${K} -n "${NS}" get secret "${NS}-generated" -o jsonpath='{.data.SYNAPSE_EXTRA}' 2>/dev/null || echo "") + EXISTING_KEY_PG_ADMIN=$(${K} -n "${NS}" get secret "${NS}-generated" -o jsonpath='{.data.POSTGRES_ADMIN_PASSWORD}' 2>/dev/null || echo "") + log "Preserved non-migrated keys from existing secret." +else + warn "No existing ${NS}-generated secret found. Some keys may be missing." +fi + +# Read old secrets from backup +log "Reading old secrets from backup..." +OLD_SIGNING_KEY=$(python3 -c " +import yaml, sys +with open('${BACKUP_DIR}/secret-chat-matrix.yaml') as f: + s = yaml.safe_load(f.read()) +print(s['data']['signing.key']) +" 2>/dev/null) + +OLD_MACAROON=$(python3 -c " +import yaml, sys +with open('${BACKUP_DIR}/secret-chat-matrix.yaml') as f: + s = yaml.safe_load(f.read()) +print(s['data']['macaroon_secret_key']) +" 2>/dev/null) + +OLD_REGISTRATION=$(python3 -c " +import yaml, sys +with open('${BACKUP_DIR}/secret-chat-matrix.yaml') as f: + s = yaml.safe_load(f.read()) +print(s['data']['registration_shared_secret']) +" 2>/dev/null) + +OLD_MAS_ENCRYPTION=$(python3 -c " +import yaml, sys +with open('${BACKUP_DIR}/secret-mas.yaml') as f: + s = yaml.safe_load(f.read()) +print(s['data']['encryption-key']) +" 2>/dev/null) + +OLD_MAS_RSA=$(python3 -c " +import yaml, sys +with open('${BACKUP_DIR}/secret-mas.yaml') as f: + s = yaml.safe_load(f.read()) +print(s['data']['signing-key.pem']) +" 2>/dev/null) + +OLD_MAS_SHARED=$(python3 -c " +import yaml, sys +with open('${BACKUP_DIR}/secret-mas.yaml') as f: + s = yaml.safe_load(f.read()) +print(s['data']['shared-secret']) +" 2>/dev/null) + +OLD_SYNAPSE_PG_PW=$(python3 -c " +import yaml, sys +with open('${BACKUP_DIR}/secret-chat-postgresql.yaml') as f: + s = yaml.safe_load(f.read()) +print(s['data']['password']) +" 2>/dev/null) + +OLD_MAS_DB_PW=$(python3 -c " +import yaml, sys +with open('${BACKUP_DIR}/secret-mas.yaml') as f: + s = yaml.safe_load(f.read()) +print(s['data']['mas-db-password']) +" 2>/dev/null) + +# Build the new secret +log "Applying generated secret with migrated keys..." + +PYTHON_BODY=$(cat <<'PYEOF' +import yaml, sys, base64 + +# Read existing secret structure from stdin (if any) +existing = {} +try: + existing = yaml.safe_load(sys.stdin) +except: + pass + +data = existing.get('data', {}) if existing else {} + +# Overwrite with old keys (these are base64 encoded already) +data['SYNAPSE_SIGNING_KEY'] = sys.argv[1] if sys.argv[1] else data.get('SYNAPSE_SIGNING_KEY','') +data['SYNAPSE_MACAROON'] = sys.argv[2] if sys.argv[2] else data.get('SYNAPSE_MACAROON','') +data['SYNAPSE_REGISTRATION_SHARED_SECRET'] = sys.argv[3] if sys.argv[3] else data.get('SYNAPSE_REGISTRATION_SHARED_SECRET','') +data['MAS_ENCRYPTION_SECRET'] = sys.argv[4] if sys.argv[4] else data.get('MAS_ENCRYPTION_SECRET','') +data['MAS_RSA_PRIVATE_KEY'] = sys.argv[5] if sys.argv[5] else data.get('MAS_RSA_PRIVATE_KEY','') + +data['MAS_SYNAPSE_SHARED_SECRET'] = sys.argv[6] if sys.argv[6] else data.get('MAS_SYNAPSE_SHARED_SECRET','') +data['POSTGRES_SYNAPSE_PASSWORD'] = sys.argv[7] if sys.argv[7] else data.get('POSTGRES_SYNAPSE_PASSWORD','') +data['POSTGRES_MATRIX_AUTHENTICATION_SERVICE_PASSWORD'] = sys.argv[8] if sys.argv[8] else data.get('POSTGRES_MATRIX_AUTHENTICATION_SERVICE_PASSWORD','') + +# Preserve existing non-migrated keys if provided +if sys.argv[9]: data['MAS_ECDSA_PRIME256V1_PRIVATE_KEY'] = sys.argv[9] +if sys.argv[10]: data['ELEMENT_CALL_LIVEKIT_SECRET'] = sys.argv[10] +if sys.argv[11]: data['HOOKSHOT_REGISTRATION'] = sys.argv[11] +if sys.argv[12]: data['HOOKSHOT_RSA_PASSKEY'] = sys.argv[12] +if sys.argv[13]: data['SYNAPSE_EXTRA'] = sys.argv[13] +if sys.argv[14]: data['POSTGRES_ADMIN_PASSWORD'] = sys.argv[14] + +secret = { + 'apiVersion': 'v1', + 'kind': 'Secret', + 'metadata': { + 'name': f'{sys.argv[15]}-generated', + 'namespace': sys.argv[16], + }, + 'type': 'Opaque', + 'data': data, +} + +yaml.dump(secret, sys.stdout, default_flow_style=False) +PYEOF +) + +# Get existing secret for merge +${K} -n "${NS}" get secret "${NS}-generated" -o yaml 2>/dev/null | \ + python3 -c "${PYTHON_BODY}" \ + "${OLD_SIGNING_KEY}" "${OLD_MACAROON}" "${OLD_REGISTRATION}" \ + "${OLD_MAS_ENCRYPTION}" "${OLD_MAS_RSA}" "${OLD_MAS_SHARED}" \ + "${OLD_SYNAPSE_PG_PW}" "${OLD_MAS_DB_PW}" \ + "${EXISTING_KEY_ECDSA}" "${EXISTING_KEY_LIVEKIT}" "${EXISTING_KEY_HOOKSHOT_REG}" \ + "${EXISTING_KEY_HOOKSHOT_PASS}" "${EXISTING_KEY_SYNAPSE_EXTRA}" "${EXISTING_KEY_PG_ADMIN}" \ + "${NS}" "${NS}" | \ + ${K} replace --force -f - 2>/dev/null || \ + ${K} create -f - 2>/dev/null + +log "Generated secret applied with migrated keys from old chart." + +# ------------------------------------------------------------------- +# Step 5: Scale Synapse to 1 +# ------------------------------------------------------------------- +log "=== Step 5: Starting Synapse ===" + +${K} -n "${NS}" scale sts -l "app.kubernetes.io/component=matrix-server" --replicas=1 2>/dev/null || \ + ${K} -n "${NS}" scale sts "${NS}-synapse-main" --replicas=1 2>/dev/null || \ + ${K} -n "${NS}" scale deploy chat-matrix --replicas=1 2>/dev/null + +log "Waiting for Synapse..." +${K} -n "${NS}" wait --for=condition=ready pod -l "app.kubernetes.io/component=matrix-server" --timeout=300s 2>/dev/null || warn "Synapse not ready yet." + +# ------------------------------------------------------------------- +# Step 6: Restore media files +# ------------------------------------------------------------------- +log "=== Step 6: Restoring media files ===" + +if [ -f "${BACKUP_DIR}/synapse-media.tar.gz" ]; then + SYNAPSE_POD=$( ${K} -n "${NS}" get pods -l "app.kubernetes.io/component=matrix-server" -o jsonpath='{.items[?(@.status.phase=="Running")].metadata.name}' 2>/dev/null) + if [ -z "${SYNAPSE_POD}" ]; then + warn "No running Synapse pod for media restore." + else + log "Copying media to Synapse pod: ${SYNAPSE_POD}" + ${K} cp "${BACKUP_DIR}/synapse-media.tar.gz" "${NS}/${SYNAPSE_POD}:/tmp/synapse-media.tar.gz" + ${K} exec -n "${NS}" "${SYNAPSE_POD}" -- tar xzf /tmp/synapse-media.tar.gz -C /media/ + ${K} exec -n "${NS}" "${SYNAPSE_POD}" -- rm /tmp/synapse-media.tar.gz + log "Media files restored." + fi +else + warn "synapse-media.tar.gz not found in backup." +fi + +# ------------------------------------------------------------------- +# Step 7: Scale MAS to 1 +# ------------------------------------------------------------------- +log "=== Step 7: Starting MAS ===" + +${K} -n "${NS}" scale deploy -l "app.kubernetes.io/component=matrix-authentication" --replicas=1 2>/dev/null || \ + ${K} -n "${NS}" scale deploy "${NS}-matrix-authentication-service" --replicas=1 2>/dev/null || \ + ${K} -n "${NS}" scale deploy mas --replicas=1 2>/dev/null + +log "Waiting for MAS..." +${K} -n "${NS}" wait --for=condition=ready pod -l "app.kubernetes.io/component=matrix-authentication" --timeout=120s 2>/dev/null || warn "MAS not ready yet." + +# ------------------------------------------------------------------- +# Summary +# ------------------------------------------------------------------- +cat < +# Example: ./setup-pg-creds.sh mrt0rtikize +# ================================================================ + +readonly YC_KUBECONFIG="${KUBECONFIG:-/home/mrt0rtikize/infra/yandex-prod/kubeconfig}" +readonly K="${KUBECTL:-kubectl} --kubeconfig ${YC_KUBECONFIG}" +readonly REPO_DIR="$(dirname "$(realpath "$0")")/.." + +readonly CNPG_NS="cnpg" +readonly CNPG_POD="shared-pg-1" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' +log() { echo -e "${GREEN}[$(date +%H:%M:%S)]${NC} $*"; } +err() { echo -e "${RED}[$(date +%H:%M:%S)] ERROR:${NC} $*"; } + +# ------------------------------------------------------------------- +# Parse argument +# ------------------------------------------------------------------- +NAME="${1:?Usage: $0 }" +NS="matrix-${NAME}" +DB_SYNAPSE="synapse_${NAME}" +DB_MAS="mas_${NAME}" +USER_SYNAPSE="synapse_${NAME}" +USER_MAS="mas_${NAME}" + +# ------------------------------------------------------------------- +# Prerequisites +# ------------------------------------------------------------------- +log "=== Setting up PG credentials for ${NAME} ===" + +if ! ${K} get ns "${NS}" >/dev/null 2>&1; then + err "Namespace ${NS} not found. Deploy the ESS app first." + exit 1 +fi + +if ! ${K} -n "${CNPG_NS}" get pod "${CNPG_POD}" >/dev/null 2>&1; then + err "CNPG pod ${CNPG_POD} not found." + exit 1 +fi + +# ------------------------------------------------------------------- +# Generate passwords +# ------------------------------------------------------------------- +log "Generating passwords..." + +SYNAPSE_PW=$(openssl rand -base64 24 | tr -d '\n') +MAS_PW=$(openssl rand -base64 24 | tr -d '\n') + +# ------------------------------------------------------------------- +# Update PostgreSQL roles +# ------------------------------------------------------------------- +log "Creating/updating CNPG role: ${USER_SYNAPSE}" +${K} exec -n "${CNPG_NS}" "${CNPG_POD}" -- \ + psql -U postgres -c "DO \$\$ BEGIN IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${USER_SYNAPSE}') THEN CREATE ROLE ${USER_SYNAPSE} WITH LOGIN PASSWORD '${SYNAPSE_PW}'; ELSE ALTER ROLE ${USER_SYNAPSE} WITH PASSWORD '${SYNAPSE_PW}'; END IF; END \$\$;" + +log "Creating/updating CNPG role: ${USER_MAS}" +${K} exec -n "${CNPG_NS}" "${CNPG_POD}" -- \ + psql -U postgres -c "DO \$\$ BEGIN IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${USER_MAS}') THEN CREATE ROLE ${USER_MAS} WITH LOGIN PASSWORD '${MAS_PW}'; ELSE ALTER ROLE ${USER_MAS} WITH PASSWORD '${MAS_PW}'; END IF; END \$\$;" + +# ------------------------------------------------------------------- +# Apply Kubernetes Secret +# ------------------------------------------------------------------- +log "Creating/updating Kubernetes Secret pg-creds in ${NS}..." + +${K} create secret generic pg-creds -n "${NS}" \ + --from-literal=synapse="${SYNAPSE_PW}" \ + --from-literal=mas="${MAS_PW}" \ + --dry-run=client -o yaml | ${K} apply -f - + +# ------------------------------------------------------------------- +# Update repo file +# ------------------------------------------------------------------- +SECRET_FILE="${REPO_DIR}/manifests/${NS}/pg-secret.yaml" +if [ -f "${SECRET_FILE}" ]; then + log "Updating ${SECRET_FILE}..." + + cat > "${SECRET_FILE}" </dev/null +${K} delete pod -n "${NS}" -l "app.kubernetes.io/component=matrix-authentication" --ignore-not-found 2>/dev/null + +# ------------------------------------------------------------------- +# Summary +# ------------------------------------------------------------------- +echo "" +echo "Synapse password: ${SYNAPSE_PW}" +echo "MAS password: ${MAS_PW}" +echo "" +echo "Done. Synapse + MAS pods are restarting with new credentials."