diff --git a/group_vars/bird.yml b/group_vars/bird.yml index 640499de501d78eb0b97038d7224ef8969afb331..e1d2e89132059592be9e2ae0d8aaded99215e5c6 100644 --- a/group_vars/bird.yml +++ b/group_vars/bird.yml @@ -1,2 +1,5 @@ --- glob_bird: {} + +glob_prometheus_bird_exporter: + listen_addr: "{{ query('ldap', 'ip', ansible_hostname, 'adm') | ansible.utils.ipv4 | first }}" diff --git a/group_vars/prometheus.yml b/group_vars/prometheus.yml index ccd9a040d0ce7bd1d6444e20f2a442960ccb9387..4979349e3f886a66291d0e45a1902406784bd3a1 100644 --- a/group_vars/prometheus.yml +++ b/group_vars/prometheus.yml @@ -1,9 +1,23 @@ --- glob_prometheus: {} +glob_service_prometheus_target: + git: + remote: https://gitlab.adm.crans.org/nounous/prometheus-target.git + version: main + name: prometheus-target + install_dir: /var/local/services/prometheus-target + generated: false + cron: + frequency: "*/10 * * * *" + options: "" + config: + ldap: + server: "ldaps://{{ query('ldap', 'ip', 'tealc', 'adm') | ansible.utils.ipv4 | first }}" + glob_ninjabot: config: - nick: monitoring + nick: fyre server: irc.adm.crans.org port: 6667 channel: "#monitoring" diff --git a/host_vars/monitoring.adm.crans.org.yml b/host_vars/fyre.adm.crans.org.yml similarity index 76% rename from host_vars/monitoring.adm.crans.org.yml rename to host_vars/fyre.adm.crans.org.yml index fc4f2a9b265bd97e0bb21d5f684b35d9ba55438c..63f5d00e51e1457b6bedd433c2e287578683afbc 100644 --- a/host_vars/monitoring.adm.crans.org.yml +++ b/host_vars/fyre.adm.crans.org.yml @@ -5,8 +5,6 @@ interfaces: loc_prometheus: node: - file: targets_node.json - targets: "{{ groups['server'] | select('match', '^.*\\.adm\\.crans\\.org$') | list | sort }}" config: - job_name: servers file_sd_configs: @@ -22,8 +20,6 @@ loc_prometheus: replacement: '$1:9100' nginx: - file: targets_nginx.json - targets: "{{ groups['nginx'] | select('match', '^.*\\.adm\\.crans\\.org$') | list | sort }}" config: - job_name: nginx file_sd_configs: @@ -77,11 +73,6 @@ loc_prometheus: replacement: 127.0.0.1:9115 blackbox_icmp: - file: targets_icmp.json - targets: - - karst.adm.crans.org - - horst.adm.crans.org - - rodney.adm.crans.org config: - job_name: blackbox_icmp file_sd_configs: @@ -98,24 +89,37 @@ loc_prometheus: - target_label: __address__ replacement: 127.0.0.1:9115 + bird: + config: + - job_name: bird + file_sd_configs: + - files: + - '/etc/prometheus/targets_bird.json' + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - source_labels: [__param_target] + target_label: __address__ + replacement: '$1:9324' + mtail: - file: targets_mtail.json - targets: - - tealc.adm.crans.org config: - job_name: mtail - static_configs: - - targets: ["tealc.adm.crans.org"] + file_sd_configs: + - files: + - '/etc/prometheus/targets_mtail.json' relabel_configs: - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] target_label: instance - - source_labels: [instance] + - source_labels: [__param_target] target_label: __address__ replacement: '$1:3903' ilo_snmp: - file: targets_ilo_snmp.json - targets: "{{ groups['ilo_snmp'] | select('match', '^.*\\.adm\\.crans\\.org$') | list | sort }}" config: - job_name: ilo_snmp file_sd_configs: @@ -133,9 +137,27 @@ loc_prometheus: - replacement: '127.0.0.1:9116' target_label: __address__ + ups_snmp: + config: + - job_name: ups_snmp + file_sd_configs: + - files: + - '/etc/prometheus/targets_ups_snmp.json' + metrics_path: '/snmp' + params: + module: + - eatonups + relabel_configs: + - source_labels: + - __address__ + target_label: __param_target + - source_labels: + - __param_target + target_label: instance + - replacement: 127.0.0.1:9116 + target_label: __address__ + printer_snmp: - file: targets_printer.json - targets: ["printer.lp.crans.org"] config: - job_name: printer_snmp static_configs: diff --git a/hosts b/hosts index 63e23322c975e75c0b0ade2a20fbae3659856377..92170eeaea34bf4f60531ef302d8b919a4794d7c 100644 --- a/hosts +++ b/hosts @@ -29,8 +29,7 @@ belenios.adm.crans.org routeurs_vm [blackbox] -fyre.cachan-adm.crans.org -monitoring.adm.crans.org +fyre.adm.crans.org [certbot] irc.adm.crans.org @@ -102,7 +101,7 @@ gitzly.adm.crans.org gitlab-ci.adm.crans.org [grafana] -monitoring.adm.crans.org +fyre.adm.crans.org [horde] horde.adm.crans.org @@ -185,10 +184,10 @@ routeur-jack.adm.crans.org helloworld.adm.crans.org [prometheus] -monitoring.adm.crans.org +fyre.adm.crans.org [prometheus_alertmanager] -monitoring.adm.crans.org +fyre.adm.crans.org [radvd:children] routeurs_vm @@ -225,7 +224,7 @@ eclat.adm.crans.org tealc.adm.crans.org [snmp] -monitoring.adm.crans.org +fyre.adm.crans.org helloworld.adm.crans.org [slapd] @@ -301,6 +300,7 @@ ethercalc.adm.crans.org en7.adm.crans.org flirt.adm.crans.org fluxx.adm.crans.org +fyre.adm.crans.org gitlab-ci.adm.crans.org gitzly.adm.crans.org helloworld.adm.crans.org @@ -312,7 +312,6 @@ kenobi.adm.crans.org kiwi.adm.crans.org linx.adm.crans.org mailman.adm.crans.org -monitoring.adm.crans.org neree.adm.crans.org netns.adm.crans.org owl.adm.crans.org diff --git a/plays/monitoring.yml b/plays/monitoring.yml index 7704fe8bb283aea4f8b3f426c356af5aba9a24af..b240734407998d550629ff97ec4bc4842b604107 100755 --- a/plays/monitoring.yml +++ b/plays/monitoring.yml @@ -4,8 +4,10 @@ - hosts: prometheus vars: prometheus: "{{ glob_prometheus | default({}) | combine(loc_prometheus | default({})) }}" + service: "{{ glob_service_prometheus_target | default({}) | combine(loc_service_prometheus_target | default({})) }}" roles: - prometheus + - service - hosts: prometheus_alertmanager vars: @@ -46,6 +48,13 @@ roles: - prometheus-nginx-exporter +# Monitor Bird metrics +- hosts: bird + vars: + prometheus_bird_exporter: "{{ glob_prometheus_bird_exporter | default({}) | combine(loc_prometheus_bird_exporter | default({})) }}" + roles: + - prometheus-bird-exporter + # Monitor mailq with a special text exporter # - hosts: redisdead.adm.crans.org # roles: ["prometheus-node-exporter-postfix"] diff --git a/roles/bird2/templates/bird/bird.conf.j2 b/roles/bird2/templates/bird/bird.conf.j2 index 56b2dc105738ffdbb7479976d3df2d41cf7a8578..963503ba01ae04de5208d51577d3611213fc3068 100644 --- a/roles/bird2/templates/bird/bird.conf.j2 +++ b/roles/bird2/templates/bird/bird.conf.j2 @@ -18,6 +18,10 @@ router id {{ bird.id }}; # Turn on global debugging of all protocols (all messages or just selected classes) # debug protocols all; +# To get meaningful uptime information in the Prometheus exporter, BIRD needs to be +# configured to use ISO-format timestamps +timeformat protocol iso long; + # +----------------------+ # | CONSTANT DEFINITIONS | # +----------------------+ diff --git a/roles/prometheus-bird-exporter/handlers/main.yml b/roles/prometheus-bird-exporter/handlers/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..b89125261336aa5941a32b8ec478608eb25acf37 --- /dev/null +++ b/roles/prometheus-bird-exporter/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart prometheus-bird-exporter + service: + name: prometheus-bird-exporter + state: restarted diff --git a/roles/prometheus-bird-exporter/tasks/main.yml b/roles/prometheus-bird-exporter/tasks/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..effc17d5100371252ed1c1f265767ac823910630 --- /dev/null +++ b/roles/prometheus-bird-exporter/tasks/main.yml @@ -0,0 +1,16 @@ +--- +- name: Install Prometheus bird-exporter + apt: + update_cache: true + name: prometheus-bird-exporter + register: apt_result + retries: 3 + until: apt_result is succeeded + +- name: Make Prometheus bird-exporter listen on adm only and use Bird v2 + lineinfile: + path: /etc/default/prometheus-bird-exporter + regexp: ^ARGS= + line: | + ARGS="-format.new -bird.v2 -web.listen-address={{ prometheus_bird_exporter.listen_addr }}:9324" + notify: Restart prometheus-bird-exporter diff --git a/roles/prometheus/tasks/main.yml b/roles/prometheus/tasks/main.yml index 3e62cf9ffee3f7ecd9b2318b21fd6b6cfe363566..0208135c933447e581e2674cca27715445cd8378 100644 --- a/roles/prometheus/tasks/main.yml +++ b/roles/prometheus/tasks/main.yml @@ -28,6 +28,7 @@ dest: /etc/prometheus/{{ item.value.file }} mode: 0644 loop: "{{ prometheus | dict2items }}" + when: "item.value.file is defined" - name: Activate prometheus service systemd: