1、在k8s中安装blackbox

• blackbox.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: blackbox-config
  namespace: monitoring
data:
  blackbox.yml: |-
      modules:
        http_2xx:
          prober: http
          timeout: 5s
          http:
        http_post_2xx:
          prober: http
          http:
            method: POST
        tcp_connect:
          prober: tcp
        pop3s_banner:
          prober: tcp
          tcp:
            query_response:
            - expect: "^+OK"
            tls: true
            tls_config:
              insecure_skip_verify: false
        ssh_banner:
          prober: tcp
          tcp:
            query_response:
            - expect: "^SSH-2.0-"
        irc_banner:
          prober: tcp
          tcp:
            query_response:
            - send: "NICK prober"
            - send: "USER prober prober prober :prober"
            - expect: "PING :([^ ]+)"
              send: "PONG ${1}"
            - expect: "^:[^ ]+ 001"
        icmp:
          prober: icmp

---  
apiVersion: v1
kind: Service
metadata:
  labels:
    app: blackbox
    servicename: blackbox
  name: blackbox
  namespace: monitoring
spec:
  ports:
  - name: http
    port: 9115
    targetPort: http
  selector:
    app: blackbox

---    
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: blackbox
  name: blackbox
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: blackbox
  template:
    metadata:
      labels:
        app: blackbox
    spec:
      containers:
      - image: prom/blackbox-exporter
        name: blackbox
        args:
          - --config.file=/etc/blackbox_exporter/blackbox.yml
        ports:
        - containerPort: 9115
          name: http
        readinessProbe:
          httpGet:
            path: /probe?target=https://www.baidu.com/&module=http_2xx
            port: http
          initialDelaySeconds: 10
        resources:
          limits:
            cpu: 200m
            memory: 200Mi
          requests:
            cpu: 100m
            memory: 100Mi
        volumeMounts:
        - mountPath: /etc/blackbox_exporter/
          name: blackbox-config
          readOnly: false
      volumes:
      - configMap:
          name: blackbox-config
        name: blackbox-config

2、配置blockbox，对站点进行health
方法一：
在prometheus-operator中增加secret “prometheus-kube-prometheus-prometheus-scrape-confg”

• prometheus-additional.yaml

- job_name: 'blackbox'
  metrics_path: /probe
  params:
    module: [http_2xx]  # Look for a HTTP 200 response.
  static_configs:
    - targets:
      - http://prometheus.io    # Target to probe with http.
      - https://www.21cn.com   # Target to probe with https.
      - https://www.baidu.com # Target to probe with http on port 8080.
#   file_sd_configs:
#     - files:
#         - /prometheus/blackbox_exporter/*.yaml
       refresh_interval: 1m
  relabel_configs:
    - source_labels: [__address__]
      target_label: __param_target
    - source_labels: [__param_target]
      target_label: instance
    - target_label: __address__
      replacement: blackbox:9115  # The blackbox exporter's service name and port.

kubectl create secret generic prometheus-kube-prometheus-prometheus-scrape-confg --from-file=prometheus-additional.yaml --dry-run -oyaml -n monitoring > additional-scrape-configs.yaml

方法二：
或者在kube-prometheus-operator-stack的values.yaml中的prometheus节点增加下以配置

    additionalScrapeConfigs:
    - job_name: 'blackbox'
      metrics_path: /probe
      params:
        module: [http_2xx]  # Look for a HTTP 200 response.
      static_configs:
        - targets:
          #- http://prometheus.io    # Target to probe with http.
          #- https://prometheus.io   # Target to probe with https.
          - https://www.baidu.com # Target to probe with http on port 8080.
      relabel_configs:
        - source_labels: [__address__]
          target_label: __param_target
        - source_labels: [__param_target]
          target_label: instance
        - target_label: __address__
          replacement: blackbox:9115  # The blackbox exporter's real hostname:port.

执行helm upgrade，进行配置升级

helm upgrade prometheus kube-prometheus-stack-15.4.4.tgz --namespace monitoring -f kube-prometheus-stack/values.yaml 

添加后，promethues的config中会增加以下配置

3、配置警报

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    release: prometheus   #与prometheus-operator自动发现关联
  name: webmonitoring
  namespace: monitoring
spec:
  groups:
    - name: WebMonitoring
      rules:
      - alert: WebSite is not accessible now!
        annotations:
          message: Website - {{ $labels.instance }} is down
          summary:  WebSite is down
        expr: |
          sum(probe_success) by (instance) !=1
        for: 3m
        labels:
          severity: critical

grafana dashboard : 13659

black_box配置参考https://blog.csdn.net/qq_25934401/article/details/84325356