Secure scrapes for online instances

provision-contest/ansible/roles/grafana/templates/prometheus.yml.j2

@@ -5,11 +5,17 @@ global:
 scrape_configs:
   - job_name: 'grafana'
     static_configs:
-      - targets: ['localhost:{{ grafana_port }}']
+      - targets: ['localhost:3000']
   - job_name: 'prometheus'
     static_configs:
       - targets: ['localhost:9090']
   - job_name: 'db'
+    basic_auth:
+      username: "prometheus"
+      password: "{{ PROMETHEUS_PASS }}"
+    tls_config:
+      insecure_skip_verify: true
+    scheme: https
     static_configs:
       - targets:
 {% for host in groups["domserver"] %}
@@ -137,6 +143,12 @@ scrape_configs:
         - {{ hostvars[host].ansible_host }}:9113
 {% endfor %}
   - job_name: 'web_fpm_domserver'
+    basic_auth:
+      username: "prometheus"
+      password: "{{ PROMETHEUS_PASS }}"
+    tls_config:
+      insecure_skip_verify: true
+    scheme: https
     static_configs:
       - targets:
 {% for host in groups["domserver"] %}

provision-contest/ansible/roles/prometheus_target_all/files/.gitignore

@@ -0,0 +1,2 @@
+*.key
+*.crt

provision-contest/ansible/roles/prometheus_target_all/tasks/main.yml

@@ -28,9 +28,18 @@
 
 - name: Get HTPassword
   delegate_to: localhost
+  become: false
   shell: "echo {{ PROMETHEUS_PASS }} | htpasswd -inBC 10 \"\" | tr -d ':\n'"
   register: htpassd_shell
 
+- name: Store HTPassword for nginx wrapper
+  copy:
+    content: "prometheus:{{ htpassd_shell.stdout }}"
+    dest: /etc/prometheus/.htpasswd
+    owner: root
+    group: root
+    mode: 0644
+
 - name: Set certificate to encrypt node_exporter traffic
   template:
     owner: prometheus

provision-contest/ansible/roles/prometheus_target_all/templates/web.yml.j2

@@ -0,0 +1,5 @@
+basic_auth_users:
+    prometheus: {{ htpassd_shell.stdout }}
+tls_server_config:
+    cert_file: /etc/prometheus/node_exporter.crt
+    key_file: /etc/prometheus/node_exporter.key

provision-contest/ansible/roles/prometheus_target_web/files/php-fpm-exporter.service

@@ -4,7 +4,7 @@ Documentation=https://github.com/hipages/php-fpm_exporter
 [Service]
 User=www-data
 Restart=always
-ExecStart=/usr/bin/php-fpm_exporter server --phpfpm.fix-process-count --phpfpm.scrape-uri unix:///var/run/php-fpm-domjudge.sock;/fpm_status
+ExecStart=/usr/bin/php-fpm_exporter server --web.listen-address :19253 --phpfpm.fix-process-count --phpfpm.scrape-uri unix:///var/run/php-fpm-domjudge.sock;/fpm_status
 ExecReload=/bin/kill -HUP $MAINPID
 TimeoutStopSec=20s
 SendSIGKILL=no

provision-contest/ansible/roles/prometheus_target_web/tasks/main.yml

@@ -18,11 +18,11 @@
 
 - name: Scrape mysql exporter with TLS encryption
   lineinfile:
-     dest: /etc/default/prometheus-mysqld-exporter
+    dest: /etc/default/prometheus-mysqld-exporter
     state: present
     regexp: '^ARGS=""'
-    line: 'ARGS="--web.config /etc/prometheus/prometheus-authentication.yml"'
-   notify: Restart mysqld-exporter
+    line: 'ARGS="--web.config.file /etc/prometheus/prometheus-authentication.yml"'
+  notify: Restart mysqld-exporter
 
 # Gather PHP-FPM statistics
 # The exporter from this is currently not in deb sources
@@ -66,9 +66,10 @@
 # Gather NGINX statistics,
 # Observe that we use the observed process itself in the monitoring
 - name: Get NGINX status
-  synchronize:
-    src: nginx-status.conf
+  template:
+    src: nginx-status.conf.j2
     dest: /etc/nginx/sites-enabled/nginx-status.conf
+    mode: 0644
   notify: Restart nginx
 
 # In the future add: --web.config /etc/prometheus/prometheus-authentication.yml"'
@@ -79,7 +80,7 @@
     dest: /etc/default/prometheus-nginx-exporter
     state: present
     regexp: '^ARGS=""'
-    line: 'ARGS="-nginx.scrape-uri=http://localhost:8787/basic_status"'
+    line: 'ARGS="-web.listen-address=127.0.0.1:19113 -nginx.scrape-uri=http://localhost:8787/basic_status"'
   notify: Restart nginx-exporter
 
 - name: Create storage dir for exporter settings

provision-contest/ansible/roles/prometheus_target_web/templates/nginx-status.conf.j2

@@ -0,0 +1,34 @@
+server {
+       listen 127.0.0.1:8787;
+       server_name _default_;
+
+       location = /basic_status {
+               stub_status;
+       }
+}
+
+server {
+       listen 0.0.0.0:9113 ssl;
+       ssl_certificate /etc/prometheus/node_exporter.crt;
+       ssl_certificate_key /etc/prometheus/node_exporter.key;
+       ssl_protocols TLSv1.3;
+
+       auth_basic "Prometheus scraping";
+       auth_basic_user_file /etc/prometheus/.htpasswd;
+       location / {
+               proxy_pass http://127.0.0.1:19113;
+       }
+}
+
+server {
+       listen 0.0.0.0:9253 ssl;
+       ssl_certificate /etc/prometheus/node_exporter.crt;
+       ssl_certificate_key /etc/prometheus/node_exporter.key;
+       ssl_protocols TLSv1.3;
+
+       auth_basic "Prometheus scraping";
+       auth_basic_user_file /etc/prometheus/.htpasswd;
+       location / {
+               proxy_pass http://127.0.0.1:19253;
+       }
+}