diff --git a/k8s/monitoring/README.md b/k8s/monitoring/README.md new file mode 100644 index 0000000..a4a31e9 --- /dev/null +++ b/k8s/monitoring/README.md @@ -0,0 +1,72 @@ +# Monitoring stack on k8s (VM: 178.18.243.51) + +Migrated from docker-compose (`/root/monitoring`, `/root/CI-CD/sonarqube` on +the VM — left `docker compose stop`'d, not removed, as a rollback path) into +the VM's own k3s cluster, with real data migrated (not a fresh start): +Prometheus TSDB, Grafana dashboards/DB, SonarQube data+DB (confirmed intact: +the real `management-platform` SonarQube project and Grafana's existing +admin password both survived). + +## Apply order +``` +kubectl apply -f monitoring-pvcs.yaml +# migrate data into the PVCs via a loader pod per volume (see chat history — +# not scripted, was a one-time manual migration) +kubectl apply -f monitoring-apps.yaml +kubectl apply -f monitoring-ingress.yaml +``` + +## Domains — READ THIS BEFORE TOUCHING DNS +- `grafanna.nav.ovh` (note: **two Ns** — `grafana.nav.ovh` without the typo + is a DIFFERENT, unrelated, pre-existing server at 109.199.127.74. Do not + point anything at plain `grafana.nav.ovh` or touch that record.) +- `sonar.nav.ovh` +- `prom.nav.ovh` + +All three already existed as A records pointing at 178.18.243.51 before this +work; only `nginx-vhost.conf` + a Let's Encrypt cert (via certbot, HTTP-01, +auto-renews) make them actually resolve to something. + +## Why nginx is in front of Traefik, not Traefik directly on 80/443 +This VM already runs an unrelated app ("nqks") on nginx, owning ports 80/443. +k3s's bundled Traefik defaults to a LoadBalancer Service, which on a +single-node k3s claims host 80/443 via iptables DNAT (klipper-lb) — +this silently hijacked nqks's traffic the first time (found and fixed +during this session). Traefik's Service is patched to ClusterIP-only via +a HelmChartConfig (`kubectl get helmchartconfig -n kube-system traefik`); +nginx reverse-proxies the 3 domains to Traefik's ClusterIP instead, and +handles TLS + BasicAuth itself. + +## Access control +- Browser access to all 3 domains: HTTP Basic Auth (`/etc/nginx/.htpasswd-monitoring` + on the VM, user `admin`) — ask whoever ran this migration for the password, + it's not in git. +- **SonarQube is also reachable directly at `178.18.243.51:9000`, no auth** — + this is deliberate, not an oversight. Jenkins' SonarQube server config + (main cluster, `hudson.plugins.sonar.SonarGlobalConfiguration.xml`) is + hardcoded to that exact address and authenticates via its own token, not + BasicAuth (which would break the scanner). `sonarqube-jenkins-proxy.service` + (systemd, socat) forwards host:9000 -> the sonarqube Service's ClusterIP; + ufw restricts port 9000 to the main server's IP only, not the open + internet. Do not add BasicAuth in front of this specific path. + - Fragility note: the socat unit hardcodes the Service's ClusterIP. If the + `sonarqube` Service in the `monitoring` namespace is ever deleted and + recreated (not just the pod — pod recreates keep the same ClusterIP), + update the `TCP::9000` target in + `/etc/systemd/system/sonarqube-jenkins-proxy.service` on the VM to match. + +## Gotchas hit building this (context for next time) +- Don't add a k8s Service of type LoadBalancer on this cluster without first + patching it to ClusterIP/setting a HelmChartConfig — same hijack risk as + Traefik's default. +- Don't add `hostPort` to a pod that also needs normal in-cluster (Service/ + Ingress) traffic to reach it — it breaks pod-to-pod routing to that pod + (confirmed directly: Traefik -> sonarqube hung/504'd the whole time + hostPort was present, while direct host-to-pod and host-to-ClusterIP both + worked fine — the asymmetry was the tell). Use a host-level proxy (socat) + or a NodePort in the 30000-32767 range instead. +- SonarQube's DB was mid-upgrade (`DB_MIGRATION_NEEDED`) after the version + jump — needed one `POST /api/system/migrate_db` (admin/admin was already + changed on the real instance; used the actual admin session) — and the + post-migration rule re-registration OOM'd on the compose file's original + 256MB web heap; bumped `SONAR_WEB_JAVAADDITIONALOPTS` to `-Xms512m -Xmx1536m`. diff --git a/k8s/monitoring/monitoring-apps.yaml b/k8s/monitoring/monitoring-apps.yaml new file mode 100644 index 0000000..e261518 --- /dev/null +++ b/k8s/monitoring/monitoring-apps.yaml @@ -0,0 +1,203 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config + namespace: monitoring +data: + prometheus.yml: | + global: + scrape_interval: 15s + evaluation_interval: 15s + + scrape_configs: + + - job_name: 'server1-node' + static_configs: + - targets: ['173.249.20.244:9100'] + labels: + server: 'server1' + + - job_name: 'my-cadvisor' + static_configs: + - targets: ['173.249.20.244:8094'] + labels: + server: 'server1' + owner: 'ameni' + + - job_name: 'server1-blackbox' + static_configs: + - targets: ['173.249.20.244:9115'] + labels: + server: 'server1' +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: {app: prometheus} + strategy: {type: Recreate} + template: + metadata: + labels: {app: prometheus} + spec: + securityContext: + fsGroup: 65534 + containers: + - name: prometheus + image: prom/prometheus:latest + args: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.retention.time=15d + - --storage.tsdb.path=/prometheus + ports: [{containerPort: 9090}] + resources: + limits: {memory: 512Mi} + volumeMounts: + - {name: config, mountPath: /etc/prometheus/prometheus.yml, subPath: prometheus.yml} + - {name: data, mountPath: /prometheus} + volumes: + - name: config + configMap: {name: prometheus-config} + - name: data + persistentVolumeClaim: {claimName: prometheus-data} +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: monitoring +spec: + selector: {app: prometheus} + ports: [{port: 9090, targetPort: 9090}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: {app: grafana} + strategy: {type: Recreate} + template: + metadata: + labels: {app: grafana} + spec: + securityContext: + fsGroup: 472 + containers: + - name: grafana + image: grafana/grafana:latest + env: + - {name: GF_SECURITY_ADMIN_PASSWORD, value: admin} + ports: [{containerPort: 3000}] + resources: + limits: {memory: 256Mi} + volumeMounts: + - {name: data, mountPath: /var/lib/grafana} + volumes: + - name: data + persistentVolumeClaim: {claimName: grafana-data} +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: monitoring +spec: + selector: {app: grafana} + ports: [{port: 3000, targetPort: 3000}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sonarqube-db + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: {app: sonarqube-db} + strategy: {type: Recreate} + template: + metadata: + labels: {app: sonarqube-db} + spec: + securityContext: + fsGroup: 999 + containers: + - name: sonarqube-db + image: postgres:15 + env: + - {name: POSTGRES_USER, value: sonar} + - {name: POSTGRES_PASSWORD, value: sonar} + - {name: POSTGRES_DB, value: sonar} + ports: [{containerPort: 5432}] + volumeMounts: + - {name: data, mountPath: /var/lib/postgresql/data} + volumes: + - name: data + persistentVolumeClaim: {claimName: sonarqube-db-data} +--- +apiVersion: v1 +kind: Service +metadata: + name: sonarqube-db + namespace: monitoring +spec: + selector: {app: sonarqube-db} + ports: [{port: 5432, targetPort: 5432}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sonarqube + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: {app: sonarqube} + strategy: {type: Recreate} + template: + metadata: + labels: {app: sonarqube} + spec: + securityContext: + fsGroup: 1000 + containers: + - name: sonarqube + image: sonarqube:community + env: + - {name: SONAR_ES_BOOTSTRAP_CHECKS_DISABLE, value: "true"} + - {name: SONAR_JDBC_URL, value: "jdbc:postgresql://sonarqube-db:5432/sonar"} + - {name: SONAR_JDBC_USERNAME, value: sonar} + - {name: SONAR_JDBC_PASSWORD, value: sonar} + - {name: SONAR_SEARCH_JAVAADDITIONALOPTS, value: "-Xms512m -Xmx512m"} + - {name: SONAR_WEB_JAVAADDITIONALOPTS, value: "-Xms512m -Xmx1536m"} + ports: [{containerPort: 9000}] + resources: + limits: {memory: 3Gi} + volumeMounts: + - {name: data, mountPath: /opt/sonarqube/data} + - {name: logs, mountPath: /opt/sonarqube/logs} + - {name: extensions, mountPath: /opt/sonarqube/extensions} + volumes: + - name: data + persistentVolumeClaim: {claimName: sonarqube-data} + - name: logs + persistentVolumeClaim: {claimName: sonarqube-logs} + - name: extensions + persistentVolumeClaim: {claimName: sonarqube-extensions} +--- +apiVersion: v1 +kind: Service +metadata: + name: sonarqube + namespace: monitoring +spec: + selector: {app: sonarqube} + ports: [{port: 9000, targetPort: 9000}] diff --git a/k8s/monitoring/monitoring-ingress.yaml b/k8s/monitoring/monitoring-ingress.yaml new file mode 100644 index 0000000..00a56ad --- /dev/null +++ b/k8s/monitoring/monitoring-ingress.yaml @@ -0,0 +1,53 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: grafana + namespace: monitoring +spec: + ingressClassName: traefik + rules: + - host: grafana.nav.ovh + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: grafana + port: {number: 3000} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: sonarqube + namespace: monitoring +spec: + ingressClassName: traefik + rules: + - host: sonar.nav.ovh + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: sonarqube + port: {number: 9000} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prometheus + namespace: monitoring +spec: + ingressClassName: traefik + rules: + - host: prom.nav.ovh + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: prometheus + port: {number: 9090} diff --git a/k8s/monitoring/monitoring-pvcs.yaml b/k8s/monitoring/monitoring-pvcs.yaml new file mode 100644 index 0000000..c23c61e --- /dev/null +++ b/k8s/monitoring/monitoring-pvcs.yaml @@ -0,0 +1,76 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: prometheus-data + namespace: monitoring +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 3Gi + storageClassName: local-path +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: grafana-data + namespace: monitoring +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi + storageClassName: local-path +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: sonarqube-data + namespace: monitoring +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 2Gi + storageClassName: local-path +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: sonarqube-logs + namespace: monitoring +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi + storageClassName: local-path +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: sonarqube-extensions + namespace: monitoring +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi + storageClassName: local-path +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: sonarqube-db-data + namespace: monitoring +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi + storageClassName: local-path diff --git a/k8s/monitoring/nginx-vhost.conf b/k8s/monitoring/nginx-vhost.conf new file mode 100644 index 0000000..9577a43 --- /dev/null +++ b/k8s/monitoring/nginx-vhost.conf @@ -0,0 +1,102 @@ +server { + server_name grafanna.nav.ovh; + location / { + auth_basic "Restricted"; + auth_basic_user_file /etc/nginx/.htpasswd-monitoring; + proxy_pass http://10.43.200.43; + proxy_set_header Host grafanna.nav.ovh; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + listen 443 ssl; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/grafanna.nav.ovh/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/grafanna.nav.ovh/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot + +} + +server { + server_name sonar.nav.ovh; + location / { + auth_basic "Restricted"; + auth_basic_user_file /etc/nginx/.htpasswd-monitoring; + proxy_pass http://10.43.200.43; + proxy_set_header Host sonar.nav.ovh; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_http_version 1.1; + } + + listen 443 ssl; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/grafanna.nav.ovh/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/grafanna.nav.ovh/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot + +} + +server { + server_name prom.nav.ovh; + location / { + auth_basic "Restricted"; + auth_basic_user_file /etc/nginx/.htpasswd-monitoring; + proxy_pass http://10.43.200.43; + proxy_set_header Host prom.nav.ovh; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_http_version 1.1; + } + + listen 443 ssl; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/grafanna.nav.ovh/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/grafanna.nav.ovh/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot + +} +server { + if ($host = grafanna.nav.ovh) { + return 301 https://$host$request_uri; + } # managed by Certbot + + + listen 80; + server_name grafanna.nav.ovh; + return 404; # managed by Certbot + + +} + +server { + if ($host = sonar.nav.ovh) { + return 301 https://$host$request_uri; + } # managed by Certbot + + + listen 80; + server_name sonar.nav.ovh; + return 404; # managed by Certbot + + +} + +server { + if ($host = prom.nav.ovh) { + return 301 https://$host$request_uri; + } # managed by Certbot + + + listen 80; + server_name prom.nav.ovh; + return 404; # managed by Certbot + + +} \ No newline at end of file diff --git a/k8s/monitoring/sonarqube-jenkins-proxy.service b/k8s/monitoring/sonarqube-jenkins-proxy.service new file mode 100644 index 0000000..fec953a --- /dev/null +++ b/k8s/monitoring/sonarqube-jenkins-proxy.service @@ -0,0 +1,13 @@ +[Unit] +Description=TCP proxy: host:9000 -> k8s sonarqube Service ClusterIP:9000 (for Jenkins, which expects this exact address) +After=k3s.service +Wants=k3s.service + +[Service] +Type=simple +ExecStart=/usr/bin/socat TCP-LISTEN:9000,fork,reuseaddr TCP:10.43.4.19:9000 +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target