PrometheusでサクッとサーバーのPING監視とDiscordへアラート通知

version: '3'
services:
  prometheus:
    image: prom/prometheus
    container_name: prometheus
    volumes:
      - ./prometheus:/etc/prometheus
    command: "--config.file=/etc/prometheus/prometheus.yaml"
    ports:
      - 9090:9090
    extra_hosts:
      - "host.docker.internal:host-gateway"
    restart: always

  alertmanager:
    image: prom/alertmanager
    container_name: alertmanager
    volumes:
      - ./alertmanager:/etc/alertmanager
    command: "--config.file=/etc/alertmanager/config.yaml"
    ports:
      - 9093:9093
    restart: always

  blackbox_exporter:
    image: prom/blackbox-exporter:latest
    container_name: blackbox_exporter
    volumes:
      - ./blackbox_exporter/config.yaml:/etc/blackbox_exporter/config.yaml
    ports:
      - 9115:9115
    restart: always

global:
  scrape_interval:     15s # 監視対象をスクレイプ（情報取得）する間隔
  evaluation_interval: 15s # アラートルールを評価する間隔

rule_files:
  - /etc/prometheus/alert.rules

alerting:
  alertmanagers:
    - static_configs:
      - targets:
        - host.docker.internal:9093

scrape_configs:
  - job_name: 'NAS_ping'
    metrics_path: /probe
    params:
      module: [icmp]
    static_configs:
      - targets:
        - 192.168.0.22 # ここに監視対象端末のIPアドレスを入れる
    scrape_timeout: 5s # スクレイプのタイムアウト
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: host.docker.internal:9115

groups:
  - name: NAS_node_alerts
    rules:
      - alert: NASNodeDown
        expr: probe_success{job="NAS_ping"} == 0
        for: 30s # 30秒以上DOWNならアラート
        labels:
          severity: critical
        annotations:
          summary: "NAS {{ $labels.instance }} is down"
          description: "NAS {{ $labels.instance }} のPING応答がありませんでした"

route:
    receiver: default-receiver
    group_by: ['alertname' , 'job']
    group_wait: 30s      # 最初のアラートは30秒待機後に送信
    group_interval: 5m   # 新しいアラートが同じグループに追加されても5分に1回送信
    repeat_interval: 1d  # 同じアラートが継続している場合は 1日に1回再通知
    routes:
    - receiver: discord
      match:
        severity: critical # alert.rulesでcriticalに指定されているアラートをdiscordに送る

receivers:
  - name: default-receiver
  - name: discord
    discord_configs:
      - webhook_url: https://discord.com/api/webhooks/****** # DiscordのWebhook URLを入力

modules:
  http_2xx:
    prober: http
    http:
  http_post_2xx:
    prober: http
    http:
      method: POST
  tcp_connect:
    prober: tcp
  pop3s_banner:
    prober: tcp
    tcp:
      query_response:
      - expect: "^+OK"
      tls: true
      tls_config:
        insecure_skip_verify: false
  ssh_banner:
    prober: tcp
    tcp:
      query_response:
      - expect: "^SSH-2.0-"
  irc_banner:
    prober: tcp
    tcp:
      query_response:
      - send: "NICK prober"
      - send: "USER prober prober prober :prober"
      - expect: "PING :([^ ]+)"
        send: "PONG ${1}"
      - expect: "^:[^ ]+ 001"
  icmp:
    prober: icmp
    timeout: 3s

# コンテナをバックグラウンドで起動
sudo docker-compose up -d

Creating prometheus        ... done
Creating alertmanager      ... done
Creating blackbox_exporter ... done


# コンテナ動作状況の確認
sudo docker-compose ps

      Name                     Command               State                    Ports
-----------------------------------------------------------------------------------------------------
alertmanager        /bin/alertmanager --config ...   Up      0.0.0.0:9093->9093/tcp,:::9093->9093/tcp
blackbox_exporter   /bin/blackbox_exporter --c ...   Up      0.0.0.0:9115->9115/tcp,:::9115->9115/tcp
prometheus          /bin/prometheus --config.f ...   Up      0.0.0.0:9090->9090/tcp,:::9090->9090/tcp

# コンテナ動作状況の確認
sudo docker-compose ps

      Name                     Command               State                     Ports
------------------------------------------------------------------------------------------------------
alertmanager        /bin/alertmanager --config ...   Up       0.0.0.0:9093->9093/tcp,:::9093->9093/tcp
blackbox_exporter   /bin/blackbox_exporter --c ...   Up       0.0.0.0:9115->9115/tcp,:::9115->9115/tcp
prometheus          /bin/prometheus --config.f ...   Restarting
# ↑ prometheusのコンテナ起動に失敗し、再起動ループに陥っている


# Docker composeのログから"ERROR"に該当する行を抽出する
sudo docker-compose logs | grep ERROR

prometheus           | time=2025-02-03T15:02:56.091Z level=ERROR source=main.go:593 msg="Error loading config (--config.file=/etc/prometheus/prometheus.yaml)" file=/etc/prometheus/prometheus.yaml err="parsing YAML file /etc/prometheus/prometheus.yaml: yaml: unmarshal errors:\n  line 21: cannot unmarshal !!str `192.168...` into struct { Targets []string \"yaml:\\\"targets\\\"\"; Labels model.LabelSet \"yaml:\\\"labels\\\"\" }"
# ↑ prometheus.ymalの21行目でunmarshalエラー発生。インデント誤りなどがないか確認しよう

probe_success{instance="192.168.22.22",job="NAS_ping"}

route:
    receiver: default-receiver
    group_by: ['alertname' , 'job']
    group_wait: 30s
    group_interval: 5m
    repeat_interval: 1d
    routes:
    - receiver: discord
      match:
        severity: critical
      mute_time_intervals:
      - out-of-NAS-Shutdown-hours

receivers:
  - name: default-receiver
  - name: discord
    discord_configs:
      - webhook_url: https://discord.com/api/webhooks/****** # DiscordのWebhookURL

mute_time_intervals:
  - name: out-of-NAS-Shutdown-hours
    time_intervals:
      - times:
         - start_time: 16:45 # ミュート開始時刻（UTCで）
           end_time: 23:15 # ミュート終了時刻（UTCで）