From c7479acfc9d16d29c13d745f3dffcb3a540cb2ee Mon Sep 17 00:00:00 2001 From: Alexander Rogov Date: Fri, 26 Jun 2026 19:05:35 +0300 Subject: [PATCH] initial: yandex-prod gitops infrastructure Gitea + ArgoCD + cert-manager + Traefik + CNPG + monitoring + loki + alloy matrix homeservers for mrt0rtikize.ru, t0rt1k.tech, roglog.space --- .gitignore | 2 + BOOTSTRAP.md | 18 +- PLAN.md | 527 ++++++++++++++++++ argocd/app-of-apps.yaml | 7 +- argocd/apps/alloy.yaml | 103 ++++ argocd/apps/cert-manager-issuers.yaml | 9 +- argocd/apps/cnpg-cluster.yaml | 7 +- argocd/apps/cnpg-operator.yaml | 1 + argocd/apps/loki.yaml | 66 ++- argocd/apps/matrix-mrt0rtikize.yaml | 89 +++ argocd/apps/matrix-roglog.yaml | 89 +++ argocd/apps/matrix-t0rt1k.yaml | 160 ++++++ argocd/apps/monitoring.yaml | 45 +- argocd/apps/traefik.yaml | 69 ++- bootstrap/argocd/install.sh | 9 +- kubeconfig | 5 - manifests/cnpg/cluster.yaml | 15 +- manifests/cnpg/databases.yaml | 16 +- manifests/cnpg/secrets.yaml | 63 --- manifests/gitea/ingressroute-ssh.yaml | 13 + .../ingressroute-rtc-tcp.yaml | 13 + .../ingressroute-rtc-udp.yaml | 12 + .../matrix-roglog/ingressroute-rtc-tcp.yaml | 13 + .../matrix-roglog/ingressroute-rtc-udp.yaml | 12 + .../matrix-t0rt1k/ingressroute-rtc-tcp.yaml | 13 + .../matrix-t0rt1k/ingressroute-rtc-udp.yaml | 12 + .../matrix-t0rt1k/vmpodscrape-synapse.yaml | 15 + manifests/metrics/grafana/ingress.yaml | 2 +- .../metrics/grafana/loki-datasource.yaml | 17 + .../metrics/{ => grafana}/namespace.yaml | 0 30 files changed, 1280 insertions(+), 142 deletions(-) create mode 100644 .gitignore create mode 100644 PLAN.md create mode 100644 argocd/apps/alloy.yaml create mode 100644 argocd/apps/matrix-mrt0rtikize.yaml create mode 100644 argocd/apps/matrix-roglog.yaml create mode 100644 argocd/apps/matrix-t0rt1k.yaml delete mode 100644 kubeconfig delete mode 100644 manifests/cnpg/secrets.yaml create mode 100644 manifests/gitea/ingressroute-ssh.yaml create mode 100644 manifests/matrix-mrt0rtikize/ingressroute-rtc-tcp.yaml create mode 100644 manifests/matrix-mrt0rtikize/ingressroute-rtc-udp.yaml create mode 100644 manifests/matrix-roglog/ingressroute-rtc-tcp.yaml create mode 100644 manifests/matrix-roglog/ingressroute-rtc-udp.yaml create mode 100644 manifests/matrix-t0rt1k/ingressroute-rtc-tcp.yaml create mode 100644 manifests/matrix-t0rt1k/ingressroute-rtc-udp.yaml create mode 100644 manifests/matrix-t0rt1k/vmpodscrape-synapse.yaml create mode 100644 manifests/metrics/grafana/loki-datasource.yaml rename manifests/metrics/{ => grafana}/namespace.yaml (100%) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..63127e9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +backups/ +kubeconfig diff --git a/BOOTSTRAP.md b/BOOTSTRAP.md index 8d89927..7482b43 100644 --- a/BOOTSTRAP.md +++ b/BOOTSTRAP.md @@ -4,7 +4,7 @@ - [ ] `kubeconfig` file placed at `~/infra/yandex-prod/kubeconfig` - [ ] `kubectl` context pointing to the new `yc-prod` cluster -- [ ] Domain `prod.t01tt.tech` DNS managed (can be updated later in Phase 5) +- [ ] Domaster `prod.t01tt.tech` DNS managed (can be updated later in Phase 5) - [ ] `git` and `helm` installed locally --- @@ -48,11 +48,11 @@ kubectl port-forward svc/gitea 3000:3000 -n gitea 2. Fill out the install form: - Database: **SQLite3** (default) - Site Title: **Gitea** - - Domain: **git.prod.t01tt.tech** + - Domaster: **git.prod.t01tt.tech** - Application URL: **https://git.prod.t01tt.tech** - Create admin account (username/password/email — save these) 3. Click "Install Gitea" -4. Create a new repository: **`yandex-prod`** (must be **public**, owned by admin) +4. Create a new repository: **`master`** (must be **public**, owned by admin) 5. Close the port-forward (Ctrl+C) --- @@ -63,13 +63,13 @@ kubectl port-forward svc/gitea 3000:3000 -n gitea cd ~/infra/yandex-prod git init -git remote add origin http://localhost:3000//yandex-prod.git +git remote add origin http://localhost:3000/admin/master.git # Or, once Gitea ingress works later, use: -# git remote add origin https://git.prod.t01tt.tech//yandex-prod.git +# git remote add origin https://git.prod.t01tt.tech/admin/master.git git add -A git commit -m "initial bootstrap: infrastructure manifests" -git push -u origin main +git push -u origin master # Enter Gitea admin credentials when prompted ``` @@ -93,7 +93,7 @@ sleep 2 ARGOCD_PASS=$(kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d) argocd login localhost:8080 --username admin --password "${ARGOCD_PASS}" --insecure -argocd repo add http://gitea.gitea.svc.cluster.local:3000//yandex-prod.git \ +argocd repo add http://gitea.gitea.svc.cluster.local:3000/admin/master.git \ --name yandex-prod \ --type git ``` @@ -242,7 +242,7 @@ argocd repo list If not connected, re-add via ArgoCD CLI: ```bash -argocd repo add http://gitea.gitea.svc.cluster.local:3000//yandex-prod.git \ +argocd repo add http://gitea.gitea.svc.cluster.local:3000/admin/master.git \ --name yandex-prod \ --type git ``` @@ -296,7 +296,7 @@ kubectl logs -n cnpg-system deploy/cnpg-controller-manager Common issue: pods can't schedule due to `podAntiAffinityType: required`. Ensure all 3 nodes exist and PVCs can bind. ### Gitea UI shows wrong URL after first login -Gitea caches the ROOT_URL from the `deployment.yaml` env vars. If you change the domain, update: +Gitea caches the ROOT_URL from the `deployment.yaml` env vars. If you change the domaster, update: ```bash kubectl set env deploy/gitea -n gitea \ GITEA__server__DOMAIN=git.prod.t01tt.tech \ diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..8f248e7 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,527 @@ +# Yandex Cloud Production Matrix Cluster — Migration Plan + +## 0. Current State + +### Existing Clusters + +| Cluster | Type | Nodes | Purpose | +|---------|------|-------|---------| +| k3s homelab (`~/infra/k3s`) | self-managed k3s | 2 (oracle, sentinel) | GitOps-based, hosts test `mrt0rtikize.ru` ESS Community | +| Yandex Cloud prod (`kubectl` default) | managed k8s v1.32.1 | 3x 2CPU/6GB | Manual Helm, hosts 3 production Matrix instances | + +### Current Prod Load (Yandex Cloud) + +| Node | CPU | RAM | What runs there | +|------|-----|-----|-----------------| +| `ofem` | 6% | 32% | element-web x3, element-call x2, well-known x3, alloy, cert-manager-cainjector | +| `uzig` | 19% | **94%** | Grafana(682Mi) + Prometheus(499Mi) + VictoriaMetrics(327Mi) + Synapse-t0rt1k(582Mi) + 3x PostgreSQL + Alertmanager + 3x Redis + 3x LiveKit SFU + Traefik | +| `efur` | 10% | 51% | Synapse-roglog(134Mi) + Synapse-uretra(137Mi) + Loki(304Mi) + 2x mas-postgresql + cert-manager + coredns | + +Key issue: **uzig at 94% RAM** — monitoring stack competes with busiest Synapse on same node. + +### Prod Matrix Instances + +| | `matrix-t0rt1k` | `matrix-roglog` | `matrix-uretra` | +|---|---|---|---| +| Domain | `t0rt1k.tech` | `roglog.space` | `uretra.space` | +| Age | 87d | 82d | 82d | +| Synapse RAM | 582Mi | 134Mi | 137Mi | +| Storage | 10+8+1 Gi (HDD) | 10+8+1 Gi (HDD) | 10+8+1 Gi (HDD) | +| Helm chart | `matrix-2.9.17` (NOT ESS) | same | same | +| MAS migration | Failed (`syn2mas` job) | OK | OK | +| Components per instance | Synapse, Element Web, Element Call, LiveKit SFU + Redis + JWT, MAS, 2x PostgreSQL, well-known | same | same | +| Ingress | Traefik, `158.160.164.95` | same LB | same LB | +| TLS | cert-manager + `letsencrypt-production` | same | same | + +### Test Instance (k3s homelab) + +| Property | Value | +|----------|-------| +| Domain | `mrt0rtikize.ru` | +| Chart | ESS Community (`oci://ghcr.io/element-hq/ess-helm/matrix-stack` v26.6.1) | +| Namespace | `matrix-mrt0rtikize` | +| Components | Synapse, MAS, Element Web, Element Admin, Matrix RTC, Hookshot, HAProxy | +| PostgreSQL | Built-in (chart-managed) | +| Storage | Longhorn | +| GitOps | ArgoCD, repo at `gitea.mrt0rtikize.ru` | + +--- + +## 1. New Cluster Architecture + +### 1.1 Platform + +- Yandex Cloud Managed Kubernetes (new cluster) +- Ability to add external nodes in future (supported experimentally, not needed now) +- Managed control plane, self-managed worker nodes + +### 1.2 GitOps Foundation + +| Component | How | Notes | +|-----------|-----|-------| +| Gitea | `kubectl apply` from `bootstrap/gitea/` | Self-hosted git server, deployed first (before ArgoCD) | +| ArgoCD | `helm install` via `bootstrap/argocd/install.sh` | Installed with `--insecure` (same as k3s), points to Gitea | +| Root App | `argocd/app-of-apps.yaml` | Scans `argocd/apps/*.yaml` recursively, deploys everything else | + +### 1.3 Infrastructure Components + +| Component | Type | Values | Notes | +|-----------|------|--------|-------| +| cert-manager | ArgoCD Helm app | `installCRDs: true`, `ClusterIssuer: letsencrypt-production` | TLS for all ingresses | +| CloudNativePG Operator | ArgoCD Helm app | `cluster.instances: 3`, `storageClass: yc-network-ssd`, `size: 50Gi`, `podAntiAffinityType: required` | HA PostgreSQL for all Matrix instances | +| Prometheus Stack | ArgoCD Helm app | Ported from `k3s/manifests/metrics/kube-prometheus-stack-values.yaml`, remoteWrite to VictoriaMetrics | Monitoring + Alertmanager | +| VictoriaMetrics | ArgoCD Helm app | Ported from `k3s/manifests/metrics/victoria-metrics-single-values.yaml` | Long-term metrics storage | +| Loki | ArgoCD Helm app | Log aggregation | — | +| Alloy/Grafana Alloy | ArgoCD Helm app | Agent for metrics/logs forwarding | — | +| Traefik | Managed by Yandex (or DaemonSet) | Cluster's built-in ingress controller | LB external IP provisioned by Yandex Cloud | + +### 1.4 ESS Instances + +Each Matrix homeserver is a separate ArgoCD Application referencing the ESS chart: + +``` +argocd/apps/ +├── matrix-mrt0rtikize.yaml (first, test migration) +├── matrix-t0rt1k.yaml (production, after procedure proven) +├── matrix-roglog.yaml +└── matrix-uretra.yaml +``` + +Each uses the **shared CloudNativePG cluster** (not built-in PostgreSQL). + +### 1.5 Directory Structure + +``` +~/infra/yandex-prod/ +├── bootstrap/ +│ ├── gitea/ +│ │ ├── namespace.yaml +│ │ ├── deployment.yaml +│ │ ├── service.yaml +│ │ ├── ingress.yaml +│ │ └── pvc.yaml +│ └── argocd/ +│ ├── install.sh +│ └── values.yaml +├── argocd/ +│ ├── app-of-apps.yaml +│ └── apps/ +│ ├── cert-manager.yaml +│ ├── cnpg-operator.yaml +│ ├── cnpg-cluster.yaml +│ ├── monitoring.yaml +│ ├── loki.yaml +│ ├── matrix-mrt0rtikize.yaml +│ ├── matrix-t0rt1k.yaml +│ ├── matrix-roglog.yaml +│ └── matrix-uretra.yaml +└── manifests/ + ├── cnpg/ + │ ├── namespace.yaml + │ ├── databases.yaml # Database CRs per homeserver + │ └── secrets.yaml # PG credentials per homeserver (or generated) + └── matrix-mrt0rtikize/ + └── (supplemental manifests, if any) +``` + +### 1.6 CloudNativePG Architecture + +``` +CloudNativePG Cluster "shared-pg" (namespace: cnpg, 3 instances) +├── Instance 1 (node A) +├── Instance 2 (node B) ← anti-affinity ensures spread +└── Instance 3 (node C) + +Databases (one pair per homeserver): +├── synapse_mrt0rtikize (owner: synapse_mrt0rtikize) +├── mas_mrt0rtikize (owner: mas_mrt0rtikize) +├── synapse_t0rt1k (owner: synapse_t0rt1k) +├── mas_t0rt1k (owner: mas_t0rt1k) +├── synapse_roglog (owner: synapse_roglog) +├── mas_roglog (owner: mas_roglog) +├── synapse_uretra (owner: synapse_uretra) +└── mas_uretra (owner: mas_uretra) + +Service: shared-pg-rw.cnpg.svc.cluster.local:5432 (primary, read-write) + shared-pg-ro.cnpg.svc.cluster.local:5432 (replicas, read-only) +``` + +Each homeserver has a dedicated PostgreSQL role and database within the same cluster. +Databases and roles are created via CNPG `Database` CRs (see `manifests/cnpg/databases.yaml`). + +Credentials are stored in per-homeserver Kubernetes Secrets (`manifests/cnpg/secrets.yaml`), +referenced by ESS via `existingSecret` / `existingSecretKey`. + +### 1.7 ESS Configuration (per instance) + +```yaml +# Shared across all instances: +serverName: +certManager: + clusterIssuer: letsencrypt-production +ingress: + className: traefik # or whatever Yandex provides + +# PostgreSQL — external, shared CNPG cluster: +postgres: + enabled: false + +synapse: + postgres: + host: shared-pg-rw.cnpg.svc.cluster.local + database: synapse_ + user: synapse_ + existingSecret: -pg-creds + existingSecretKey: synapse + media: + storage: + size: 10Gi # adjustable per instance load + storageClassName: yc-network-hdd # media is fine on HDD + ingress: + host: matrix. + +matrixAuthenticationService: + postgres: + host: shared-pg-rw.cnpg.svc.cluster.local + database: mas_ + user: mas_ + existingSecret: -pg-creds + existingSecretKey: mas + ingress: + host: account. + +elementWeb: + ingress: + host: chat. + +elementAdmin: + ingress: + host: admin. + +matrixRTC: + ingress: + host: mrtc. + +hookshot: + enabled: true + # ingress host if needed for webhooks +``` + +### 1.8 Boot Order + +``` +Step 1: kubectl apply bootstrap/gitea/ +Step 2: helm install argocd (bootstrap/argocd/install.sh) +Step 3: git push manifests/ + argocd/ to Gitea +Step 4: kubectl apply argocd/app-of-apps.yaml +Step 5: ArgoCD syncs cert-manager, CNPG operator, CNPG cluster, databases, monitoring, ESS +``` + +--- + +## 2. Migration Procedure: `mrt0rtikize.ru` (test instance) + +> Perform on the test instance first to validate the procedure before touching production. + +### 2.1 Backup (on k3s homelab) + +```bash +NS=matrix-mrt0rtikize + +# 1. Stop Synapse + MAS +kubectl scale sts -l "app.kubernetes.io/component=matrix-server" -n $NS --replicas=0 +kubectl scale deploy -l "app.kubernetes.io/component=matrix-authentication" -n $NS --replicas=0 + +# 2. Dump PostgreSQL (built-in PG, release name is "ess" but pods are named matrix-mrt0rtikize-*) +# The PG pod is named based on the ESS release. Find it: +PG_POD=$(kubectl get pods -n $NS -l "app.kubernetes.io/name=postgres" -o name | head -1) +kubectl exec -n $NS $PG_POD -- pg_dumpall -U postgres > dump-mrt0rtikize.sql + +# 3. Backup generated secrets (CRITICAL — contains signing key, MAS encryption key) +kubectl get secret matrix-mrt0rtikize-generated -n $NS -o yaml > secrets-mrt0rtikize.yaml + +# 4. Backup deployment markers +kubectl get configmap \ + -l "app.kubernetes.io/managed-by=matrix-tools-deployment-markers" \ + -n $NS -o yaml > markers-mrt0rtikize.yaml + +# 5. Backup media files +# Find PV path from the node: +kubectl get pv -n $NS -o yaml | grep -A5 "synapse-media" +# Copy from the reported path on the node to a safe location + +# 6. Save ESS values (from the ArgoCD Application or helm get values) +kubectl get application matrix-mrt0rtikize -n argocd -o yaml > app-mrt0rtikize.yaml +``` + +**Critical data that MUST be preserved:** + +| Data | Location | Why | +|------|----------|-----| +| `SYNAPSE_SIGNING_KEY` | `matrix-mrt0rtikize-generated` secret | Federation identity — all other servers know this key. Lose it = all rooms break. | +| `MAS_ENCRYPTION_SECRET` | same secret | User session encryption. Lose it = all users must re-login. | +| `MAS_RSA_PRIVATE_KEY` | same secret | OIDC signing. Lose it = re-auth needed. | +| `SYNAPSE_MACAROON` | same secret | Admin API access token. | +| PostgreSQL dump | `dump-mrt0rtikize.sql` | All user accounts, rooms, messages. | +| Media files | Synapse media PV | Uploaded images/files/avatars. | + +### 2.2 Restore (on new Yandex cluster) + +```bash +# 1. Create secrets in the matrix-mrt0rtikize namespace +NS=matrix-mrt0rtikize +kubectl create ns $NS + +# Apply the generated secrets (signing key etc — DO NOT let initSecrets regenerate it) +kubectl apply -f secrets-mrt0rtikize.yaml +kubectl apply -f markers-mrt0rtikize.yaml + +# 2. Restore PostgreSQL dumps +# CNPG service: shared-pg-rw.cnpg.svc.cluster.local +# Extract per-DB dumps from pg_dumpall or use pg_restore: +PG_POD=$(kubectl get pods -n cnpg -l "cnpg.io/cluster=shared-pg,cnpg.io/podRole=instance" -o name | head -1) + +# Restore Synapse DB: +kubectl exec -n cnpg $PG_POD -- psql -U synapse_mrt0rtikize \ + -d synapse_mrt0rtikize < dump-mrt0rtikize.sql + +# Restore MAS DB: +kubectl exec -n cnpg $PG_POD -- psql -U mas_mrt0rtikize \ + -d mas_mrt0rtikize < dump-mrt0rtikize.sql + +# (Note: pg_dumpall produces a single file for all databases. You may need to +# split it per-database first, or use pg_restore per-database.) + +# 3. Restore media files +# Copy from backup to the new PV (path depends on storage class) +# For Yandex Cloud CSI: mount the PV on a temp pod and copy files in + +# 4. Deploy ESS via ArgoCD +# The Application was already committed to git (argocd/apps/matrix-mrt0rtikize.yaml). +# ArgoCD syncs it. Since secrets + markers are pre-loaded, the chart initializes +# with the existing signing key and database credentials. + +# 5. Verify +# - Log in with an existing user +# - Check federation: https://federationtester.matrix.org/?server_name=mrt0rtikize.ru +# - Test Element Call (VoIP) +# - Monitor logs for errors +``` + +### 2.3 DNS Cutover + +Once validated: +``` +Old records: mrt0rtikize.ru → k3s cluster IP + *.mrt0rtikize.ru → k3s cluster IP + +New records: mrt0rtikize.ru → new cluster Traefik LB IP + matrix.mrt0rtikize.ru → new cluster LB + account.mrt0rtikize.ru → new cluster LB + chat.mrt0rtikize.ru → new cluster LB + admin.mrt0rtikize.ru → new cluster LB + mrtc.mrt0rtikize.ru → new cluster LB +``` + +Lower DNS TTLs 24h before cutover to minimize propagation delay. + +### 2.4 Rollback + +If migration fails: +1. Scale down Synapse + MAS on new cluster +2. Revert DNS to k3s cluster IP +3. Scale up Synapse + MAS on k3s homelab + +The old instance on k3s should still be functional (just stopped, not deleted). + +--- + +## 3. Production Migration (vague plan) + +> Repeat steps from Section 2 for each production instance, one at a time. + +### 3.1 Order + +| # | Instance | Synapse Load | Complexity | +|---|----------|--------------|------------| +| 1 | `mrt0rtikize.ru` | Minimal (test) | Low — prove procedure | +| 2 | `t0rt1k.tech` | **582Mi** (busiest) | High — schedule during low-traffic, may need extended downtime | +| 3 | `roglog.space` | 134Mi | Medium | +| 4 | `uretra.space` | 137Mi | Medium | + +### 3.2 Pre-migration Checklist (per instance) + +``` +[ ] Announce maintenance window to users +[ ] Lower DNS TTLs (24h before) +[ ] Full PostgreSQL dump + verify (pg_restore --list) +[ ] Backup media files + verify checksums +[ ] Backup generated secrets (verify signing key matches federation) +[ ] Save current Helm values (helm get values) +[ ] Document current ingress/DNS/Certificate setup +[ ] Prepare rollback procedure +``` + +### 3.3 Migration Steps (per instance) + +``` +1. Stop Synapse + MAS on old cluster +2. Create CNPG databases on new cluster +3. Restore PostgreSQL dump to CNPG +4. Restore media files to new PV +5. Apply secrets (signing key, MAS keys, macaroon) +6. Apply deployment markers +7. Deploy ESS via ArgoCD on new cluster +8. Wait for pods healthy, certs issued +9. Test: login, federation, Element Call +10. Cut over DNS +11. Monitor for 24h +12. If stable: remove old instance resources from old cluster +``` + +### 3.4 Special Considerations for Prod Instances + +**`t0rt1k.tech` (busiest instance, 582Mi Synapse):** +- Uses older `matrix-2.9.17` chart (NOT ESS). Migration means switching to ESS Community chart. +- Has a **failed `syn2mas` job** — MAS migration was incomplete. When deploying ESS which bundles MAS, the migration may need to be completed or re-done. +- The 582Mi memory usage suggests many concurrent users/rooms — dump may be large. Allocate enough storage and time for the SQL dump/restore. +- Consider running the new ESS in parallel (different hostnames) first, then switching DNS once proven. + +**`roglog.space` and `uretra.space`:** +- Lower load (134Mi/137Mi) — quicker backups, less downtime risk. +- Same chart switch (`matrix-2.9.17` → ESS). +- Can be done in shorter windows. + +**Chart migration (`matrix-2.9.17` → ESS):** +- The old chart uses separate Helm releases per component (`chat`, `element-call`, `livekit`). +- ESS bundles everything into one chart. The database schema may differ. +- Key difference: ESS uses MAS for auth (Matrix 2.0), old chart may use legacy Synapse auth. +- May need to run `syn2mas` migration or manual user migration. Investigate per-instance before cutover. + +--- + +## 4. PostgreSQL Backup (ongoing) + +CloudNativePG has built-in backup to S3-compatible storage. Configure once for automatic daily backups: + +```yaml +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: shared-pg-daily + namespace: cnpg +spec: + schedule: "0 3 * * *" # 03:00 UTC daily + backupOwnerReference: self + cluster: + name: shared-pg + immediate: false + target: prefer-standby +``` + +CNPG also supports continuous WAL archiving to S3 for point-in-time recovery. +Configure Yandex Object Storage as the S3 target. + +--- + +## 5. Architecture Diagram (text) + +``` +┌────────────────────────────────────────────────────────────┐ +│ Yandex Cloud Managed K8s │ +│ │ +│ ┌───────────────────┐ ┌──────────────────────────────┐ │ +│ │ Infrastructure │ │ Matrix Layer │ │ +│ │ │ │ │ │ +│ │ Gitea (git) │ │ ┌─────────────────────────┐ │ │ +│ │ ArgoCD (gitops) │ │ │ matrix-mrt0rtikize (ESS)│ │ │ +│ │ cert-manager │ │ │ - Synapse │ │ │ +│ │ Traefik (LB) │ │ │ - MAS │ │ │ +│ │ Prometheus/Grafana │ │ │ - Element Web/Admin │ │ │ +│ │ VictoriaMetrics │ │ │ - Matrix RTC (LiveKit) │ │ │ +│ │ Loki │ │ │ - Hookshot │ │ │ +│ │ Alloy │ │ │ - HAProxy │ │ │ +│ └───────────────────┘ │ └──────────┬──────────────┘ │ │ +│ │ │ │ │ +│ ┌───────────────────┐ │ ┌──────────▼──────────────┐ │ │ +│ │ CNPG Cluster │ │ │ matrix-t0rt1k (ESS) │ │ │ +│ │ (3 nodes, SSD) │◄──┤ │ (same structure) │ │ │ +│ │ │ │ └─────────────────────────┘ │ │ +│ │ synapse_mrt0rtikize│ │ ┌─────────────────────────┐ │ │ +│ │ mas_mrt0rtikize │ │ │ matrix-roglog (ESS) │ │ │ +│ │ synapse_t0rt1k │ │ │ (same structure) │ │ │ +│ │ mas_t0rt1k │ │ └─────────────────────────┘ │ │ +│ │ synapse_roglog │ │ ┌─────────────────────────┐ │ │ +│ │ mas_roglog │ │ │ matrix-uretra (ESS) │ │ │ +│ │ synapse_uretra │ │ │ (same structure) │ │ │ +│ │ mas_uretra │ │ └─────────────────────────┘ │ │ +│ └───────────────────┘ └──────────────────────────────┘ │ +│ │ +│ External LB: │ +└────────────────────────────────────────────────────────────┘ +``` + +--- + +## 6. Implementation Notes + +### 6.1 Secrets Management + +- ESS `initSecrets` generates 14 credentials. For migration, these MUST be restored from backup (not regenerated). +- `SYNAPSE_SIGNING_KEY` is the most critical — it identifies the server to the federation. Changing it breaks all existing rooms and federation relationships. +- The `matrix*-generated` secret and deployment markers ConfigMap must be applied **before** the first ArgoCD sync, so the ESS chart does not generate new (wrong) ones. +- For fresh ESS instances (new homeservers, not migrations), let `initSecrets` generate them normally. + +### 6.2 Image Registry + +- ESS pulls from `oci.element.io` (Synapse, Element Web, Element Admin, lk-jwt-service) and `ghcr.io` (matrix-tools, hookshot), and `docker.io` (livekit, postgres, redis). +- `oci.element.io` S3 backend (`oci-element-io-images-storage-prod.s3.eu-central-1.amazonaws.com`) was observed to fail intermittently from Russia with "connection reset by peer". Images eventually pulled on retry, but consider: + - Setting `image.pullPolicy: IfNotPresent` to reduce re-pulls + - Setting up a containerd registry mirror or local pull-through cache for `oci.element.io` + - Pre-pulling images to nodes during initial setup + +### 6.3 Resource Limits + +Set `resources.requests` and `resources.limits` on all ESS components to prevent the 94% node issue seen in prod: + +```yaml +synapse: + resources: + requests: + memory: 256Mi + cpu: 100m + limits: + memory: 1Gi + cpu: 1000m +``` + +Do similar for MAS, element-web, livekit-sfu, etc. ESS chart supports per-component resource configuration. + +### 6.4 Storage Classes + +| Workload | Storage Class | Reason | +|----------|--------------|--------| +| PostgreSQL (CNPG) | `yc-network-ssd` | Database — needs low latency / high IOPS | +| Synapse media | `yc-network-hdd` (default) | Media files — sequential access, SSD benefit is marginal | +| Prometheus TSDB | `yc-network-ssd` | Time-series DB — random writes benefit from SSD | +| Loki chunks | `yc-network-hdd` | Log storage — sequential writes, HDD is fine | + +--- + +## 7. Next Steps (for next session) + +When the new cluster is ready, open a new session and point to this file. The next session should: + +1. Read this plan +2. Explore the new cluster (nodes, storage classes, ingress config) +3. Implement Phase 0 (bootstrap GitOps foundation): + - Create `~/infra/yandex-prod/` directory structure + - Write `bootstrap/gitea/` manifests + - Write `bootstrap/argocd/install.sh` + `values.yaml` + - Write `argocd/app-of-apps.yaml` + - Write infrastructure apps (cert-manager, CNPG, monitoring) + - Write ESS apps + - Push to Gitea +4. Execute Phase 1 (backup `mrt0rtikize.ru` from k3s) +5. Execute Phase 2 (restore `mrt0rtikize.ru` to new cluster) +6. Validate and plan DNS cutover diff --git a/argocd/app-of-apps.yaml b/argocd/app-of-apps.yaml index 29d4cd5..ade871b 100644 --- a/argocd/app-of-apps.yaml +++ b/argocd/app-of-apps.yaml @@ -8,8 +8,8 @@ metadata: spec: project: default source: - repoURL: http://gitea.gitea.svc.cluster.local:3000/gitea/yandex-prod.git - targetRevision: main + repoURL: http://gitea.gitea.svc.cluster.local:3000/admin/main.git + targetRevision: master path: argocd/apps directory: recurse: true @@ -18,8 +18,5 @@ spec: server: https://kubernetes.default.svc namespace: argocd syncPolicy: - automated: - prune: true - selfHeal: true syncOptions: - CreateNamespace=true diff --git a/argocd/apps/alloy.yaml b/argocd/apps/alloy.yaml new file mode 100644 index 0000000..8b8fc7d --- /dev/null +++ b/argocd/apps/alloy.yaml @@ -0,0 +1,103 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: alloy + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "0" + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://grafana.github.io/helm-charts + chart: alloy + targetRevision: ">=1.5.0" + helm: + values: | + alloy: + configMap: + content: |- + // Discover Kubernetes pods + discovery.kubernetes "pods" { + role = "pod" + } + + // Relabel pods to intelligently map existing Kubernetes labels to service/component + discovery.relabel "pods" { + targets = discovery.kubernetes.pods.targets + + // Create service label - try multiple sources in priority order + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] + regex = "(.+)" + replacement = "${1}" + target_label = "service" + } + // 2. Second priority: app label + rule { + source_labels = ["__meta_kubernetes_pod_label_app", "service"] + regex = "^(.+);$" + replacement = "${1}" + target_label = "service" + } + // 3. Third priority: extract from pod name (remove hash suffix) + rule { + source_labels = ["__meta_kubernetes_pod_name", "service"] + regex = "^([a-z0-9-]+?)(?:-[a-f0-9]{5,10})?;$" + replacement = "${1}" + target_label = "service" + } + + // Create component label from app.kubernetes.io/component or component label + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_component"] + regex = ".+" + target_label = "component" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_component"] + regex = ".+" + target_label = "component" + } + + // Drop pods that we still can't identify + rule { + source_labels = ["service"] + regex = "^$" + action = "drop" + } + + // Map standard Kubernetes metadata to Loki labels + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + } + + // Collect logs from Kubernetes pods using the Kubernetes API + loki.source.kubernetes "pods" { + targets = discovery.relabel.pods.output + forward_to = [loki.write.loki.receiver] + } + + // Write logs to Loki + loki.write "loki" { + endpoint { + url = "http://loki.metrics.svc.cluster.local:3100/loki/api/v1/push" + } + } + destination: + server: https://kubernetes.default.svc + namespace: metrics + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/apps/cert-manager-issuers.yaml b/argocd/apps/cert-manager-issuers.yaml index 3c16c58..026e2f8 100644 --- a/argocd/apps/cert-manager-issuers.yaml +++ b/argocd/apps/cert-manager-issuers.yaml @@ -10,8 +10,8 @@ metadata: spec: project: default source: - repoURL: http://gitea.gitea.svc.cluster.local:3000/gitea/yandex-prod.git - targetRevision: main + repoURL: http://gitea.gitea.svc.cluster.local:3000/admin/main.git + targetRevision: master path: manifests/cert-manager directory: recurse: true @@ -19,7 +19,4 @@ spec: destination: server: https://kubernetes.default.svc namespace: cert-manager - syncPolicy: - automated: - prune: true - selfHeal: true + syncPolicy: {} diff --git a/argocd/apps/cnpg-cluster.yaml b/argocd/apps/cnpg-cluster.yaml index 1656175..e038ae4 100644 --- a/argocd/apps/cnpg-cluster.yaml +++ b/argocd/apps/cnpg-cluster.yaml @@ -10,8 +10,8 @@ metadata: spec: project: default source: - repoURL: http://gitea.gitea.svc.cluster.local:3000/gitea/yandex-prod.git - targetRevision: main + repoURL: http://gitea.gitea.svc.cluster.local:3000/admin/main.git + targetRevision: master path: manifests/cnpg directory: recurse: true @@ -20,8 +20,5 @@ spec: server: https://kubernetes.default.svc namespace: cnpg syncPolicy: - automated: - prune: true - selfHeal: true syncOptions: - CreateNamespace=true diff --git a/argocd/apps/cnpg-operator.yaml b/argocd/apps/cnpg-operator.yaml index 12f2372..bb23cdc 100644 --- a/argocd/apps/cnpg-operator.yaml +++ b/argocd/apps/cnpg-operator.yaml @@ -17,3 +17,4 @@ spec: syncPolicy: syncOptions: - CreateNamespace=true + - ServerSideApply=true diff --git a/argocd/apps/loki.yaml b/argocd/apps/loki.yaml index 4387317..2da8f27 100644 --- a/argocd/apps/loki.yaml +++ b/argocd/apps/loki.yaml @@ -16,25 +16,61 @@ spec: helm: values: | deploymentMode: SingleBinary + + write: + replicas: 0 + read: + replicas: 0 + backend: + replicas: 0 + loki: auth_enabled: false - commonConfig: - replication_factor: 1 storage: + bucketNames: + chunks: chunks + ruler: ruler + admin: admin type: filesystem - schemaConfig: - configs: - - from: "2025-01-01" - store: tsdb - objectStore: filesystem - schema: v13 - index: - prefix: index_ - period: 24h - limits_config: - retention_period: 30d - reject_old_samples: true - reject_old_samples_max_age: 168h + filesystem: + chunks_directory: /var/loki/chunks + rules_directory: /var/loki/rules + structuredConfig: + auth_enabled: false + + server: + http_listen_port: 3100 + grpc_listen_port: 9095 + + common: + path_prefix: /var/loki + replication_factor: 1 + + schema_config: + configs: + - from: "2025-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + + limits_config: + retention_period: 30d + reject_old_samples: true + reject_old_samples_max_age: 168h + + chunksCache: + enabled: false + resultsCache: + enabled: false + gateway: + enabled: false + test: + enabled: false + lokiCanary: + enabled: false singleBinary: replicas: 1 diff --git a/argocd/apps/matrix-mrt0rtikize.yaml b/argocd/apps/matrix-mrt0rtikize.yaml new file mode 100644 index 0000000..9e0675f --- /dev/null +++ b/argocd/apps/matrix-mrt0rtikize.yaml @@ -0,0 +1,89 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: matrix-mrt0rtikize + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + sources: + - repoURL: http://gitea.gitea.svc.cluster.local:3000/admin/main.git + targetRevision: master + path: manifests/matrix-mrt0rtikize + directory: + recurse: true + include: "*.yaml" + - repoURL: ghcr.io + chart: element-hq/ess-helm/matrix-stack + targetRevision: 26.6.1 + helm: + values: | + serverName: mrt0rtikize.ru + + certManager: + clusterIssuer: letsencrypt-production + + ingress: + className: traefik + + postgres: + enabled: false + + synapse: + postgres: + host: shared-pg-rw.cnpg.svc.cluster.local + database: synapse_mrt0rtikize + user: synapse_mrt0rtikize + password: + secret: pg-creds + secretKey: synapse + media: + storage: + size: 10Gi + ingress: + host: matrix.mrt0rtikize.ru + + matrixAuthenticationService: + postgres: + host: shared-pg-rw.cnpg.svc.cluster.local + database: mas_mrt0rtikize + user: mas_mrt0rtikize + password: + secret: pg-creds + secretKey: mas + ingress: + host: account.mrt0rtikize.ru + + elementWeb: + ingress: + host: chat.mrt0rtikize.ru + + elementAdmin: + ingress: + host: admin.mrt0rtikize.ru + + matrixRTC: + ingress: + host: mrtc.mrt0rtikize.ru + sfu: + manualIP: "81.26.181.240" + useStunToDiscoverPublicIP: false + exposedServices: + rtcTcp: + port: 30009 + portType: NodePort + nodePort: "" + rtcMuxedUdp: + port: 30008 + portType: NodePort + nodePort: "" + + hookshot: + enabled: true + destination: + server: https://kubernetes.default.svc + namespace: matrix-mrt0rtikize + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/apps/matrix-roglog.yaml b/argocd/apps/matrix-roglog.yaml new file mode 100644 index 0000000..da4fc67 --- /dev/null +++ b/argocd/apps/matrix-roglog.yaml @@ -0,0 +1,89 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: matrix-roglog + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + sources: + - repoURL: http://gitea.gitea.svc.cluster.local:3000/admin/main.git + targetRevision: master + path: manifests/matrix-roglog + directory: + recurse: true + include: "*.yaml" + - repoURL: ghcr.io + chart: element-hq/ess-helm/matrix-stack + targetRevision: 26.6.1 + helm: + values: | + serverName: roglog.space + + certManager: + clusterIssuer: letsencrypt-production + + ingress: + className: traefik + + postgres: + enabled: false + + synapse: + postgres: + host: shared-pg-rw.cnpg.svc.cluster.local + database: synapse_roglog + user: synapse_roglog + password: + secret: pg-creds + secretKey: synapse + media: + storage: + size: 10Gi + ingress: + host: matrix.roglog.space + + matrixAuthenticationService: + postgres: + host: shared-pg-rw.cnpg.svc.cluster.local + database: mas_roglog + user: mas_roglog + password: + secret: pg-creds + secretKey: mas + ingress: + host: auth.roglog.space + + elementWeb: + ingress: + host: chat.roglog.space + + elementAdmin: + ingress: + host: admin.roglog.space + + matrixRTC: + ingress: + host: rtc.roglog.space + sfu: + manualIP: "81.26.181.240" + useStunToDiscoverPublicIP: false + exposedServices: + rtcTcp: + port: 30005 + portType: NodePort + nodePort: "" + rtcMuxedUdp: + port: 30004 + portType: NodePort + nodePort: "" + + hookshot: + enabled: true + destination: + server: https://kubernetes.default.svc + namespace: matrix-roglog + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/apps/matrix-t0rt1k.yaml b/argocd/apps/matrix-t0rt1k.yaml new file mode 100644 index 0000000..f8af807 --- /dev/null +++ b/argocd/apps/matrix-t0rt1k.yaml @@ -0,0 +1,160 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: matrix-t0rt1k + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + sources: + - repoURL: http://gitea.gitea.svc.cluster.local:3000/admin/main.git + targetRevision: master + path: manifests/matrix-t0rt1k + directory: + recurse: true + include: "*.yaml" + - repoURL: ghcr.io + chart: element-hq/ess-helm/matrix-stack + targetRevision: 26.6.1 + helm: + values: | + serverName: t0rt1k.tech + + certManager: + clusterIssuer: letsencrypt-production + + ingress: + className: traefik + + postgres: + enabled: false + + synapse: + postgres: + host: shared-pg-rw.cnpg.svc.cluster.local + database: synapse_t0rt1k + user: synapse_t0rt1k + password: + secret: pg-creds + secretKey: synapse + media: + storage: + size: 10Gi + ingress: + host: matrix.t0rt1k.tech + resources: + requests: + memory: 256Mi + cpu: 100m + limits: + memory: 1.5Gi + workers: + federation-sender: + enabled: true + resources: + requests: + memory: 192Mi + cpu: 50m + limits: + memory: 256Mi + federation-reader: + enabled: true + resources: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 256Mi + client-reader: + enabled: true + replicas: 1 + resources: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 256Mi + initial-synchrotron: + enabled: true + resources: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 256Mi + synchrotron: + enabled: true + replicas: 1 + resources: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 256Mi + additional: + 0-federation-retry: + config: | + destination_min_retry_interval: "1m" + destination_retry_multiplier: 1.5 + destination_max_retry_interval: "1d" + + matrixAuthenticationService: + postgres: + host: shared-pg-rw.cnpg.svc.cluster.local + database: mas_t0rt1k + user: mas_t0rt1k + password: + secret: pg-creds + secretKey: mas + ingress: + host: auth.t0rt1k.tech + resources: + limits: + memory: 128Mi + + elementWeb: + ingress: + host: chat.t0rt1k.tech + + elementAdmin: + ingress: + host: admin.t0rt1k.tech + + matrixRTC: + ingress: + host: rtc.t0rt1k.tech + sfu: + manualIP: "81.26.181.240" + useStunToDiscoverPublicIP: false + exposedServices: + rtcTcp: + port: 30007 + portType: NodePort + nodePort: "" + rtcMuxedUdp: + port: 30006 + portType: NodePort + nodePort: "" + resources: + requests: + memory: 64Mi + cpu: 50m + limits: + memory: 512Mi + + hookshot: + enabled: true + resources: + limits: + memory: 256Mi + ingress: + host: hookshot.t0rt1k.tech + className: traefik + tlsEnabled: true + destination: + server: https://kubernetes.default.svc + namespace: matrix-t0rt1k + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/apps/monitoring.yaml b/argocd/apps/monitoring.yaml index 3df93d2..926ff6e 100644 --- a/argocd/apps/monitoring.yaml +++ b/argocd/apps/monitoring.yaml @@ -10,6 +10,13 @@ metadata: spec: project: default sources: + - repoURL: http://gitea.gitea.svc.cluster.local:3000/admin/main.git + targetRevision: master + path: manifests/metrics/grafana + directory: + recurse: true + include: "*.yaml" + - repoURL: https://victoriametrics.github.io/helm-charts/ chart: victoria-metrics-k8s-stack targetRevision: ">=0.30.0" @@ -18,6 +25,19 @@ spec: fullnameOverride: vm-k8s-stack namespaceOverride: metrics + victoria-metrics-operator: + resources: + requests: + memory: 128Mi + cpu: 25m + limits: + memory: 256Mi + + defaultDashboards: + dashboards: + node-exporter-full: + enabled: false + vmsingle: enabled: true spec: @@ -101,19 +121,22 @@ spec: kubeEtcd: enabled: false - - repoURL: http://gitea.gitea.svc.cluster.local:3000/gitea/yandex-prod.git - targetRevision: main - path: manifests/metrics/grafana - directory: - recurse: true - include: "*.yaml" - destination: server: https://kubernetes.default.svc - namespace: argocd + namespace: metrics + ignoreDifferences: + - group: operator.victoriametrics.com + kind: VMAlertmanager + jsonPointers: + - /spec/config + - kind: Secret + name: monitoring-victoria-metrics-operator-validation + jsonPointers: + - /data + - kind: ValidatingWebhookConfiguration + name: monitoring-victoria-metrics-operator-admission + jsonPointers: + - /webhooks/0/clientConfig/caBundle syncPolicy: - automated: - prune: true - selfHeal: true syncOptions: - CreateNamespace=true diff --git a/argocd/apps/traefik.yaml b/argocd/apps/traefik.yaml index 0f041a3..1ab61ab 100644 --- a/argocd/apps/traefik.yaml +++ b/argocd/apps/traefik.yaml @@ -17,6 +17,9 @@ spec: values: | deployment: kind: DaemonSet + podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9100" ingressClass: enabled: true @@ -27,6 +30,13 @@ spec: - "--ping=true" - "--metrics.prometheus=true" - "--metrics.prometheus.entrypoint=metrics" + - "--entryPoints.ssh.address=:2222/tcp" + - "--entryPoints.rtc-tcp-mrt0rtikize.address=:30009/tcp" + - "--entryPoints.rtc-udp-mrt0rtikize.address=:30008/udp" + - "--entryPoints.rtc-tcp-t0rt1k.address=:30007/tcp" + - "--entryPoints.rtc-udp-t0rt1k.address=:30006/udp" + - "--entryPoints.rtc-tcp-roglog.address=:30005/tcp" + - "--entryPoints.rtc-udp-roglog.address=:30004/udp" - "--providers.kubernetesingress.ingressclass=traefik" - "--providers.kubernetesingress.ingressendpoint.publishedservice=traefik/traefik" - "--accesslog=true" @@ -36,16 +46,65 @@ spec: web: port: 8080 exposedPort: 80 - redirectTo: websecure + http: + redirections: + entryPoint: + to: websecure + scheme: https + permanent: true websecure: port: 8443 exposedPort: 443 + ssh: + port: 2222 + exposedPort: 22 + protocol: TCP + expose: + default: true + rtc-tcp-mrt0rtikize: + port: 30009 + exposedPort: 30009 + protocol: TCP + expose: + default: true + rtc-udp-mrt0rtikize: + port: 30008 + exposedPort: 30008 + protocol: UDP + expose: + default: true + rtc-tcp-t0rt1k: + port: 30007 + exposedPort: 30007 + protocol: TCP + expose: + default: true + rtc-udp-t0rt1k: + port: 30006 + exposedPort: 30006 + protocol: UDP + expose: + default: true + rtc-tcp-roglog: + port: 30005 + exposedPort: 30005 + protocol: TCP + expose: + default: true + rtc-udp-roglog: + port: 30004 + exposedPort: 30004 + protocol: UDP + expose: + default: true metrics: port: 9100 - expose: false + expose: + default: false traefik: port: 9000 - expose: false + expose: + default: false service: type: LoadBalancer @@ -54,10 +113,6 @@ spec: service.beta.kubernetes.io/yandex-load-balancer-specification: '{"type": "network-load-balancer"}' service.beta.kubernetes.io/yandex-load-balancer-type: external - podAnnotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9100" - resources: requests: cpu: 100m diff --git a/bootstrap/argocd/install.sh b/bootstrap/argocd/install.sh index fed1032..c3bfedc 100755 --- a/bootstrap/argocd/install.sh +++ b/bootstrap/argocd/install.sh @@ -1,13 +1,12 @@ #!/bin/bash set -e -KUBECONFIG="/home/mrt0rtikize/infra/yandex-prod/kubeconfig" -KCTL="kubectl --kubeconfig ${KUBECONFIG}" +export KUBECONFIG="$(dirname "$(realpath "$0")")/../../kubeconfig" echo "=== Installing ArgoCD ===" helm repo add argo https://argoproj.github.io/argo-helm 2>/dev/null || true -helm repo update +helm repo update argo helm upgrade --install argocd argo/argo-cd \ --namespace argocd \ @@ -20,10 +19,10 @@ echo "" echo "=== ArgoCD installed ===" echo "" echo "To access ArgoCD UI:" -echo " kubectl --kubeconfig ${KUBECONFIG} port-forward svc/argocd-server -n argocd 8080:80" +echo " kubectl port-forward svc/argocd-server -n argocd 8080:80" echo "" echo "Admin password:" -kubectl --kubeconfig ${KUBECONFIG} -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d +kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d echo "" echo "" echo "Login with username: admin" diff --git a/kubeconfig b/kubeconfig deleted file mode 100644 index 7e7625e..0000000 --- a/kubeconfig +++ /dev/null @@ -1,5 +0,0 @@ -# Placeholder for Yandex Cloud Managed Kubernetes kubeconfig. -# Copy your kubeconfig here before running bootstrap. -# -# Example: -# yc managed-kubernetes cluster get-credentials --id --external --kubeconfig kubeconfig diff --git a/manifests/cnpg/cluster.yaml b/manifests/cnpg/cluster.yaml index cbee07c..4e837dd 100644 --- a/manifests/cnpg/cluster.yaml +++ b/manifests/cnpg/cluster.yaml @@ -7,6 +7,14 @@ spec: instances: 3 imageName: ghcr.io/cloudnative-pg/postgresql:16 + resources: + requests: + memory: 1Gi + cpu: 250m + limits: + memory: 2Gi + cpu: 1000m + storage: size: 20Gi storageClass: yc-network-ssd @@ -21,10 +29,13 @@ spec: postgresql: parameters: - shared_buffers: "256MB" - effective_cache_size: "768MB" + shared_buffers: "512MB" + effective_cache_size: "1536MB" maintenance_work_mem: "64MB" max_connections: "200" + work_mem: "16MB" + random_page_cost: "1.1" + effective_io_concurrency: "200" monitoring: enablePodMonitor: true diff --git a/manifests/cnpg/databases.yaml b/manifests/cnpg/databases.yaml index 596f199..1a26bbc 100644 --- a/manifests/cnpg/databases.yaml +++ b/manifests/cnpg/databases.yaml @@ -6,7 +6,7 @@ metadata: spec: name: synapse_mrt0rtikize owner: synapse_mrt0rtikize - clusterRef: + cluster: name: shared-pg --- apiVersion: postgresql.cnpg.io/v1 @@ -17,7 +17,7 @@ metadata: spec: name: mas_mrt0rtikize owner: mas_mrt0rtikize - clusterRef: + cluster: name: shared-pg --- apiVersion: postgresql.cnpg.io/v1 @@ -28,7 +28,7 @@ metadata: spec: name: synapse_t0rt1k owner: synapse_t0rt1k - clusterRef: + cluster: name: shared-pg --- apiVersion: postgresql.cnpg.io/v1 @@ -39,7 +39,7 @@ metadata: spec: name: mas_t0rt1k owner: mas_t0rt1k - clusterRef: + cluster: name: shared-pg --- apiVersion: postgresql.cnpg.io/v1 @@ -50,7 +50,7 @@ metadata: spec: name: synapse_roglog owner: synapse_roglog - clusterRef: + cluster: name: shared-pg --- apiVersion: postgresql.cnpg.io/v1 @@ -61,7 +61,7 @@ metadata: spec: name: mas_roglog owner: mas_roglog - clusterRef: + cluster: name: shared-pg --- apiVersion: postgresql.cnpg.io/v1 @@ -72,7 +72,7 @@ metadata: spec: name: synapse_uretra owner: synapse_uretra - clusterRef: + cluster: name: shared-pg --- apiVersion: postgresql.cnpg.io/v1 @@ -83,5 +83,5 @@ metadata: spec: name: mas_uretra owner: mas_uretra - clusterRef: + cluster: name: shared-pg diff --git a/manifests/cnpg/secrets.yaml b/manifests/cnpg/secrets.yaml deleted file mode 100644 index 00fa72d..0000000 --- a/manifests/cnpg/secrets.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: mrt0rtikize-pg-creds - namespace: cnpg - labels: - cnpg.io/reload: "" -type: kubernetes.io/basic-auth -stringData: - synapse: | - username: synapse_mrt0rtikize - password: change-me-synapse - mas: | - username: mas_mrt0rtikize - password: change-me-mas ---- -apiVersion: v1 -kind: Secret -metadata: - name: t0rt1k-pg-creds - namespace: cnpg - labels: - cnpg.io/reload: "" -type: kubernetes.io/basic-auth -stringData: - synapse: | - username: synapse_t0rt1k - password: change-me-synapse - mas: | - username: mas_t0rt1k - password: change-me-mas ---- -apiVersion: v1 -kind: Secret -metadata: - name: roglog-pg-creds - namespace: cnpg - labels: - cnpg.io/reload: "" -type: kubernetes.io/basic-auth -stringData: - synapse: | - username: synapse_roglog - password: change-me-synapse - mas: | - username: mas_roglog - password: change-me-mas ---- -apiVersion: v1 -kind: Secret -metadata: - name: uretra-pg-creds - namespace: cnpg - labels: - cnpg.io/reload: "" -type: kubernetes.io/basic-auth -stringData: - synapse: | - username: synapse_uretra - password: change-me-synapse - mas: | - username: mas_uretra - password: change-me-mas diff --git a/manifests/gitea/ingressroute-ssh.yaml b/manifests/gitea/ingressroute-ssh.yaml new file mode 100644 index 0000000..01bcb9f --- /dev/null +++ b/manifests/gitea/ingressroute-ssh.yaml @@ -0,0 +1,13 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteTCP +metadata: + name: gitea-ssh + namespace: gitea +spec: + entryPoints: + - ssh + routes: + - match: HostSNI(`*`) + services: + - name: gitea + port: 22 diff --git a/manifests/matrix-mrt0rtikize/ingressroute-rtc-tcp.yaml b/manifests/matrix-mrt0rtikize/ingressroute-rtc-tcp.yaml new file mode 100644 index 0000000..2fcdc09 --- /dev/null +++ b/manifests/matrix-mrt0rtikize/ingressroute-rtc-tcp.yaml @@ -0,0 +1,13 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteTCP +metadata: + name: rtc-tcp + namespace: matrix-mrt0rtikize +spec: + entryPoints: + - rtc-tcp-mrt0rtikize + routes: + - match: HostSNI(`*`) + services: + - name: matrix-mrt0rtikize-matrix-rtc-sfu-tcp + port: 30009 diff --git a/manifests/matrix-mrt0rtikize/ingressroute-rtc-udp.yaml b/manifests/matrix-mrt0rtikize/ingressroute-rtc-udp.yaml new file mode 100644 index 0000000..1bb0b7d --- /dev/null +++ b/manifests/matrix-mrt0rtikize/ingressroute-rtc-udp.yaml @@ -0,0 +1,12 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteUDP +metadata: + name: rtc-udp + namespace: matrix-mrt0rtikize +spec: + entryPoints: + - rtc-udp-mrt0rtikize + routes: + - services: + - name: matrix-mrt0rtikize-matrix-rtc-sfu-muxed-udp + port: 30008 diff --git a/manifests/matrix-roglog/ingressroute-rtc-tcp.yaml b/manifests/matrix-roglog/ingressroute-rtc-tcp.yaml new file mode 100644 index 0000000..5a800d8 --- /dev/null +++ b/manifests/matrix-roglog/ingressroute-rtc-tcp.yaml @@ -0,0 +1,13 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteTCP +metadata: + name: rtc-tcp + namespace: matrix-roglog +spec: + entryPoints: + - rtc-tcp-roglog + routes: + - match: HostSNI(`*`) + services: + - name: matrix-roglog-matrix-rtc-sfu-tcp + port: 30005 diff --git a/manifests/matrix-roglog/ingressroute-rtc-udp.yaml b/manifests/matrix-roglog/ingressroute-rtc-udp.yaml new file mode 100644 index 0000000..1379a1a --- /dev/null +++ b/manifests/matrix-roglog/ingressroute-rtc-udp.yaml @@ -0,0 +1,12 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteUDP +metadata: + name: rtc-udp + namespace: matrix-roglog +spec: + entryPoints: + - rtc-udp-roglog + routes: + - services: + - name: matrix-roglog-matrix-rtc-sfu-muxed-udp + port: 30004 diff --git a/manifests/matrix-t0rt1k/ingressroute-rtc-tcp.yaml b/manifests/matrix-t0rt1k/ingressroute-rtc-tcp.yaml new file mode 100644 index 0000000..550e807 --- /dev/null +++ b/manifests/matrix-t0rt1k/ingressroute-rtc-tcp.yaml @@ -0,0 +1,13 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteTCP +metadata: + name: rtc-tcp + namespace: matrix-t0rt1k +spec: + entryPoints: + - rtc-tcp-t0rt1k + routes: + - match: HostSNI(`*`) + services: + - name: matrix-t0rt1k-matrix-rtc-sfu-tcp + port: 30007 diff --git a/manifests/matrix-t0rt1k/ingressroute-rtc-udp.yaml b/manifests/matrix-t0rt1k/ingressroute-rtc-udp.yaml new file mode 100644 index 0000000..20dace5 --- /dev/null +++ b/manifests/matrix-t0rt1k/ingressroute-rtc-udp.yaml @@ -0,0 +1,12 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteUDP +metadata: + name: rtc-udp + namespace: matrix-t0rt1k +spec: + entryPoints: + - rtc-udp-t0rt1k + routes: + - services: + - name: matrix-t0rt1k-matrix-rtc-sfu-muxed-udp + port: 30006 diff --git a/manifests/matrix-t0rt1k/vmpodscrape-synapse.yaml b/manifests/matrix-t0rt1k/vmpodscrape-synapse.yaml new file mode 100644 index 0000000..3ff1608 --- /dev/null +++ b/manifests/matrix-t0rt1k/vmpodscrape-synapse.yaml @@ -0,0 +1,15 @@ +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: synapse + namespace: matrix-t0rt1k +spec: + namespaceSelector: + matchNames: + - matrix-t0rt1k + podMetricsEndpoints: + - port: synapse-metrics + path: /metrics + selector: + matchLabels: + app.kubernetes.io/component: matrix-server diff --git a/manifests/metrics/grafana/ingress.yaml b/manifests/metrics/grafana/ingress.yaml index 32ec704..f063112 100644 --- a/manifests/metrics/grafana/ingress.yaml +++ b/manifests/metrics/grafana/ingress.yaml @@ -21,6 +21,6 @@ spec: pathType: Prefix backend: service: - name: vm-k8s-stack-grafana + name: monitoring-grafana port: number: 80 diff --git a/manifests/metrics/grafana/loki-datasource.yaml b/manifests/metrics/grafana/loki-datasource.yaml new file mode 100644 index 0000000..3ad5eb0 --- /dev/null +++ b/manifests/metrics/grafana/loki-datasource.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: loki-grafana-ds + namespace: metrics + labels: + grafana_datasource: "1" +data: + datasource.yaml: | + apiVersion: 1 + datasources: + - access: proxy + isDefault: false + name: Loki + type: loki + uid: Loki + url: http://loki.metrics.svc.cluster.local:3100 diff --git a/manifests/metrics/namespace.yaml b/manifests/metrics/grafana/namespace.yaml similarity index 100% rename from manifests/metrics/namespace.yaml rename to manifests/metrics/grafana/namespace.yaml