Real data migrated, not a fresh start: 1.2GB Prometheus TSDB, Grafana's
existing dashboards/DB (its admin password was already changed from
default — confirmed real, in-use data), SonarQube's data/logs/extensions
and its own Postgres DB (confirmed: the actual `management-platform`
SonarQube project survived, matching the Jenkinsfile's own
-Dsonar.projectKey). Old docker-compose services stopped, not removed —
left as a rollback path.
Real problems hit and fixed along the way (see k8s/monitoring/README.md
for the full detail, worth reading before touching any of this again):
- k3s's bundled Traefik defaults to a LoadBalancer Service, which
immediately hijacked ports 80/443 via iptables DNAT (klipper-lb) out
from under this VM's existing unrelated nginx-served app ("nqks") the
moment k3s was installed. Patched Traefik to ClusterIP via a
HelmChartConfig; nginx now reverse-proxies the 3 new domains to
Traefik's ClusterIP instead of Traefik touching host ports at all.
- Caught my own mistake before it caused damage: "corrected" what I
assumed was a typo (grafanna.nav.ovh -> grafana.nav.ovh) — the two-N
version is what the user actually specified, and is what points at this
VM; plain grafana.nav.ovh is a different, unrelated, pre-existing live
server at 109.199.127.74. Renamed everything back before requesting any
cert or touching that domain.
- Adding hostPort to the SonarQube pod (for Jenkins' hardcoded
178.18.243.51:9000) broke pod-to-pod routing to that pod entirely —
Traefik couldn't reach it (hung/504) while direct host-to-pod and
host-to-ClusterIP both worked fine the whole time, which is what gave
it away. Replaced with a host-level socat proxy (systemd unit) to the
Service's ClusterIP instead, which touches nothing pod-level. Verified
Jenkins' exact address still works AND Traefik routing to the same pod
works simultaneously.
- SonarQube needed a DB migration after the version jump
(DB_MIGRATION_NEEDED), then OOM'd during post-migration rule
re-registration on the compose file's original 256MB web heap — bumped
SONAR_WEB_JAVAADDITIONALOPTS.
TLS: real Let's Encrypt certs via certbot (HTTP-01, nginx already owns
port 80 so no DNS-01/OVH-webhook replication needed on this second
cluster), auto-renewing. Access: HTTP Basic Auth in front of all three
domains for browser access; SonarQube also stays reachable unauthenticated
at 178.18.243.51:9000 specifically for Jenkins (which authenticates via
its own token, not BasicAuth — that path would break the scanner),
restricted to the main server's IP via ufw, not open to the internet.
Verified end-to-end: all three https://*.nav.ovh domains return 401
without credentials and load correctly with them; Jenkins' exact
SonarQube address (no auth) still returns 200; real data confirmed
present in Grafana, Prometheus, and SonarQube post-migration.
204 lines
5.1 KiB
YAML
204 lines
5.1 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: prometheus-config
|
|
namespace: monitoring
|
|
data:
|
|
prometheus.yml: |
|
|
global:
|
|
scrape_interval: 15s
|
|
evaluation_interval: 15s
|
|
|
|
scrape_configs:
|
|
|
|
- job_name: 'server1-node'
|
|
static_configs:
|
|
- targets: ['173.249.20.244:9100']
|
|
labels:
|
|
server: 'server1'
|
|
|
|
- job_name: 'my-cadvisor'
|
|
static_configs:
|
|
- targets: ['173.249.20.244:8094']
|
|
labels:
|
|
server: 'server1'
|
|
owner: 'ameni'
|
|
|
|
- job_name: 'server1-blackbox'
|
|
static_configs:
|
|
- targets: ['173.249.20.244:9115']
|
|
labels:
|
|
server: 'server1'
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: prometheus
|
|
namespace: monitoring
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels: {app: prometheus}
|
|
strategy: {type: Recreate}
|
|
template:
|
|
metadata:
|
|
labels: {app: prometheus}
|
|
spec:
|
|
securityContext:
|
|
fsGroup: 65534
|
|
containers:
|
|
- name: prometheus
|
|
image: prom/prometheus:latest
|
|
args:
|
|
- --config.file=/etc/prometheus/prometheus.yml
|
|
- --storage.tsdb.retention.time=15d
|
|
- --storage.tsdb.path=/prometheus
|
|
ports: [{containerPort: 9090}]
|
|
resources:
|
|
limits: {memory: 512Mi}
|
|
volumeMounts:
|
|
- {name: config, mountPath: /etc/prometheus/prometheus.yml, subPath: prometheus.yml}
|
|
- {name: data, mountPath: /prometheus}
|
|
volumes:
|
|
- name: config
|
|
configMap: {name: prometheus-config}
|
|
- name: data
|
|
persistentVolumeClaim: {claimName: prometheus-data}
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: prometheus
|
|
namespace: monitoring
|
|
spec:
|
|
selector: {app: prometheus}
|
|
ports: [{port: 9090, targetPort: 9090}]
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: grafana
|
|
namespace: monitoring
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels: {app: grafana}
|
|
strategy: {type: Recreate}
|
|
template:
|
|
metadata:
|
|
labels: {app: grafana}
|
|
spec:
|
|
securityContext:
|
|
fsGroup: 472
|
|
containers:
|
|
- name: grafana
|
|
image: grafana/grafana:latest
|
|
env:
|
|
- {name: GF_SECURITY_ADMIN_PASSWORD, value: admin}
|
|
ports: [{containerPort: 3000}]
|
|
resources:
|
|
limits: {memory: 256Mi}
|
|
volumeMounts:
|
|
- {name: data, mountPath: /var/lib/grafana}
|
|
volumes:
|
|
- name: data
|
|
persistentVolumeClaim: {claimName: grafana-data}
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: grafana
|
|
namespace: monitoring
|
|
spec:
|
|
selector: {app: grafana}
|
|
ports: [{port: 3000, targetPort: 3000}]
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: sonarqube-db
|
|
namespace: monitoring
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels: {app: sonarqube-db}
|
|
strategy: {type: Recreate}
|
|
template:
|
|
metadata:
|
|
labels: {app: sonarqube-db}
|
|
spec:
|
|
securityContext:
|
|
fsGroup: 999
|
|
containers:
|
|
- name: sonarqube-db
|
|
image: postgres:15
|
|
env:
|
|
- {name: POSTGRES_USER, value: sonar}
|
|
- {name: POSTGRES_PASSWORD, value: sonar}
|
|
- {name: POSTGRES_DB, value: sonar}
|
|
ports: [{containerPort: 5432}]
|
|
volumeMounts:
|
|
- {name: data, mountPath: /var/lib/postgresql/data}
|
|
volumes:
|
|
- name: data
|
|
persistentVolumeClaim: {claimName: sonarqube-db-data}
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: sonarqube-db
|
|
namespace: monitoring
|
|
spec:
|
|
selector: {app: sonarqube-db}
|
|
ports: [{port: 5432, targetPort: 5432}]
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: sonarqube
|
|
namespace: monitoring
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels: {app: sonarqube}
|
|
strategy: {type: Recreate}
|
|
template:
|
|
metadata:
|
|
labels: {app: sonarqube}
|
|
spec:
|
|
securityContext:
|
|
fsGroup: 1000
|
|
containers:
|
|
- name: sonarqube
|
|
image: sonarqube:community
|
|
env:
|
|
- {name: SONAR_ES_BOOTSTRAP_CHECKS_DISABLE, value: "true"}
|
|
- {name: SONAR_JDBC_URL, value: "jdbc:postgresql://sonarqube-db:5432/sonar"}
|
|
- {name: SONAR_JDBC_USERNAME, value: sonar}
|
|
- {name: SONAR_JDBC_PASSWORD, value: sonar}
|
|
- {name: SONAR_SEARCH_JAVAADDITIONALOPTS, value: "-Xms512m -Xmx512m"}
|
|
- {name: SONAR_WEB_JAVAADDITIONALOPTS, value: "-Xms512m -Xmx1536m"}
|
|
ports: [{containerPort: 9000}]
|
|
resources:
|
|
limits: {memory: 3Gi}
|
|
volumeMounts:
|
|
- {name: data, mountPath: /opt/sonarqube/data}
|
|
- {name: logs, mountPath: /opt/sonarqube/logs}
|
|
- {name: extensions, mountPath: /opt/sonarqube/extensions}
|
|
volumes:
|
|
- name: data
|
|
persistentVolumeClaim: {claimName: sonarqube-data}
|
|
- name: logs
|
|
persistentVolumeClaim: {claimName: sonarqube-logs}
|
|
- name: extensions
|
|
persistentVolumeClaim: {claimName: sonarqube-extensions}
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: sonarqube
|
|
namespace: monitoring
|
|
spec:
|
|
selector: {app: sonarqube}
|
|
ports: [{port: 9000, targetPort: 9000}]
|