1.部署prometheus-webhook-dingtalk

kubectl apply -f configmap.yaml
kubectl apply -f deployment.yaml

configmap.yaml

apiVersion: v1
data:
  config.yml: |-
    targets:
      webhook1:
        #钉钉机器人webhook设置
        url: https://oapi.dingtalk.com/robot/send?access_token=XXXXXXXXXX
        # secret for signature
        secret: XXXXXXXXXXXXX
kind: ConfigMap
metadata:
  name: dingtalk-conf
  namespace: monitoring

deployment.yaml

apiVersion: v1
kind: Service
metadata:
  name: svc-webhook-dingtalk
  namespace: monitoring
spec:
  #type: NodePort
  type: ClusterIP
  ports:
  - port: 8060
    protocol: TCP
    targetPort: 8060
  selector:
    app: webhook-dingtalk
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: webhook-dingtalk
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: webhook-dingtalk
  template:
    metadata:
      labels:
        app: webhook-dingtalk
    spec:
      imagePullSecrets:
        - name: shu-cn
      containers:
      - image: shu.cn/prom/prometheus-webhook-dingtalk:v2.1.0
        name: webhook-dingtalk
        imagePullPolicy: IfNotPresent
        resources:
          requests:
            cpu: 50m
            memory: 128Mi
          limits:
            cpu: 100m
            memory: 512Mi
        ports:
        - containerPort: 8060
          name: tcp-8060
          protocol: TCP
        volumeMounts:
        - name: host-time
          mountPath: /etc/localtime
        - name: dingtalk-conf
          mountPath: /etc/prometheus-webhook-dingtalk/config.yml
          subPath: config.yml
      volumes:
        - name: host-time
          hostPath:
            path: /etc/localtime
        - name: dingtalk-conf
          configMap:
            name: dingtalk-conf

2.部署alertmanager

kubectl apply -f configmap.yaml
kubectl apply -f deployment.yaml

configmap.yaml

apiVersion: v1
data:
  alertmanager.yml: |-
    route:
      group_by: ['alertname'] #根据告警规则组名进行分组
      group_wait: 30s  #分组内第一个告警等待时间，若30s内触发同一个组的另一个告警则会合并发送
      group_interval: 5m #发送新告警时间间隔
      repeat_interval: 1h #重复告警间隔发送时间
      receiver: 'web.hook' #选择的告警接收者，与下面的receivers中定义的名称要一致
    receivers:
      - name: 'web.hook' #定义告警接收者
        webhook_configs:
          - url: 'http://svc-webhook-dingtalk.monitoring.svc:8060/dingtalk/webhook1/send'
    inhibit_rules:
      - source_match:
          severity: 'warning' #已经触发告警的规则标签
        target_match:
          severity: 'warning'  #准备发送告警的规则标签
        equal: ['pod', 'namespace'] #告警的标签是否匹配这三个标签，可修改
kind: ConfigMap
metadata:
  name: alertmanager-conf
  namespace: monitoring

deployment.yaml

apiVersion: v1
kind: Service
metadata:
  name: svc-alertmanager
  namespace: monitoring
spec:
  #type: NodePort
  type: ClusterIP
  ports:
  - port: 9093
    protocol: TCP
    targetPort: 9093
  selector:
    app: alertmanager
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: alertmanager
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: alertmanager
  template:
    metadata:
      labels:
        app: alertmanager
    spec:
      imagePullSecrets:
        - name: shu-cn
      containers:
      - image: shu.cn/prom/alertmanager:v0.25.0
        name: alertmanager
        imagePullPolicy: IfNotPresent
        resources:
          requests:
            cpu: 50m
            memory: 128Mi
          limits:
            cpu: 100m
            memory: 512Mi
        ports:
        - containerPort: 9093
          name: tcp-9093
          protocol: TCP
        volumeMounts:
        - name: host-time
          mountPath: /etc/localtime
        - name: alertmanager-conf
          mountPath: /etc/alertmanager/alertmanager.yml
          subPath: alertmanager.yml
      volumes:
        - name: host-time
          hostPath:
            path: /etc/localtime
        - name: alertmanager-conf
          configMap:
            name: alertmanager-conf

3.alertmanager内部访问地址

svc-alertmanager.monitoring.svc:9093

4.prometheus主配置文件中添加alertmanager配置

...
    # Alertmanager configuration
    alerting:
      alertmanagers:
        - static_configs:
            - targets:
               - svc-alertmanager.monitoring.svc:9093
...