diff --git a/roles/prometheus-alertmanager/templates/prometheus/alertmanager.yml.j2 b/roles/prometheus-alertmanager/templates/prometheus/alertmanager.yml.j2 index 2442108bc4eed67f329e17133d80b76c8e3f36ce..666c5ffcb20840106b3e79ebca42c98ba5873d3f 100644 --- a/roles/prometheus-alertmanager/templates/prometheus/alertmanager.yml.j2 +++ b/roles/prometheus-alertmanager/templates/prometheus/alertmanager.yml.j2 @@ -28,14 +28,15 @@ route: # When the first notification was sent, wait 'group_interval' to send a batch # of new alerts that started firing for that group. - group_interval: 5m + group_interval: 1m # If an alert has successfully been sent, wait 'repeat_interval' to # resend them. repeat_interval: 3h # A default receiver - receiver: team-roots-mails + #receiver: team-roots-mails + receiver: webhook-ninjabot # Inhibition rules allow to mute a set of alerts given that another alert is @@ -55,4 +56,7 @@ receivers: - name: 'team-roots-mails' email_configs: - to: 'roots@crans.org' - +- name: 'webhook-ninjabot' + webhook_configs: + - url: 'http://zamok.adm.crans.org:5000/' + send_resolved: true diff --git a/roles/prometheus/templates/prometheus/alert.rules.yml.j2 b/roles/prometheus/templates/prometheus/alert.rules.yml.j2 index cf24fd7a60a12b30f8bb35f48ba3c947e4cfed63..18923c4f15ea759c9a1fba60f940cabcfac431d3 100644 --- a/roles/prometheus/templates/prometheus/alert.rules.yml.j2 +++ b/roles/prometheus/templates/prometheus/alert.rules.yml.j2 @@ -59,6 +59,8 @@ groups: - alert: APIHighRequestLatency expr: api_http_request_latencies_second{quantile="0.5"} > 1 for: 10m + labels: + severity: warning annotations: summary: "Latence élevée sur {{ $labels.instance }}" description: "{{ $labels.instance }} a un temps de réponse médian au dessus d'une seconde (actuellement : {{ $value }}s)." @@ -77,7 +79,7 @@ groups: expr: node_ntp_drift_seconds > 0.01 for: 1m labels: - severity: moderate + severity: warning annotations: description: "Décalage NTP élevé sur {{ $labels.instance }}" summary: "Le décalage NTP est élevé ({{ $value }} > 0.01)"