本文只是简单记录一下配置文件
prometheus.yaml

# A scrape configuration for running Prometheus on a Kubernetes cluster.
# This uses separate scrape configs for cluster components (i.e. API server, node)
# and services to allow each to use different authentication configs.
#
# Kubernetes labels will be added as Prometheus labels on metrics via the
# `labelmap` relabeling action.
#
# If you are using Kubernetes 1.7.2 or earlier, please take note of the comments
# for the kubernetes-cadvisor job; you will need to edit or remove this job.

# Scrape config for API servers.
#
# Kubernetes exposes API servers as endpoints to the default/kubernetes
# service so this uses `endpoints` role and uses relabelling to only keep
# the endpoints associated with the default/kubernetes service using the
# default named port `https`. This works for single API server deployments as
# well as HA API server deployments.
rule_files: 
- /etc/prometheus_rules/*.rule
scrape_configs:
- job_name: 'kubernetes-apiservers'

  kubernetes_sd_configs:
  - role: endpoints

  # Default to scraping over https. If required, just disable this or change to
  # `http`.
  scheme: https

  # This TLS & bearer token file config is used to connect to the actual scrape
  # endpoints for cluster components. This is separate to discovery auth
  # configuration because discovery & scraping are two separate concerns in
  # Prometheus. The discovery auth config is automatic if Prometheus runs inside
  # the cluster. Otherwise, more config options have to be provided within the
  # <kubernetes_sd_config>.
  tls_config:
    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    # If your node certificates are self-signed or use a different CA to the
    # master CA, then disable certificate verification below. Note that
    # certificate verification is an integral part of a secure infrastructure
    # so this should only be disabled in a controlled environment. You can
    # disable certificate verification by uncommenting the line below.
    #
    # insecure_skip_verify: true
  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

  # Keep only the default/kubernetes service endpoints for the https port. This
  # will add targets for each API server which Kubernetes adds an endpoint to
  # the default/kubernetes service.
  relabel_configs:
  - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
    action: keep
    regex: default;kubernetes;https

# Scrape config for nodes (kubelet).
#
# Rather than connecting directly to the node, the scrape is proxied though the
# Kubernetes apiserver.  This means it will work if Prometheus is running out of
# cluster, or can't connect to nodes for some other reason (e.g. because of
# firewalling).
- job_name: 'kubernetes-nodes'

  # Default to scraping over https. If required, just disable this or change to
  # `http`.
  scheme: https

  # This TLS & bearer token file config is used to connect to the actual scrape
  # endpoints for cluster components. This is separate to discovery auth
  # configuration because discovery & scraping are two separate concerns in
  # Prometheus. The discovery auth config is automatic if Prometheus runs inside
  # the cluster. Otherwise, more config options have to be provided within the
  # <kubernetes_sd_config>.
  tls_config:
    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

  kubernetes_sd_configs:
  - role: node

  relabel_configs:
  - action: labelmap
    regex: __meta_kubernetes_node_label_(.+)
  - target_label: __address__
    replacement: kubernetes.default.svc:443
  - source_labels: [__meta_kubernetes_node_name]
    regex: (.+)
    target_label: __metrics_path__
    replacement: /api/v1/nodes/${1}/proxy/metrics

# Scrape config for Kubelet cAdvisor.
#
# This is required for Kubernetes 1.7.3 and later, where cAdvisor metrics
# (those whose names begin with 'container_') have been removed from the
# Kubelet metrics endpoint.  This job scrapes the cAdvisor endpoint to
# retrieve those metrics.
#
# In Kubernetes 1.7.0-1.7.2, these metrics are only exposed on the cAdvisor
# HTTP endpoint; use "replacement: /api/v1/nodes/${1}:4194/proxy/metrics"
# in that case (and ensure cAdvisor's HTTP server hasn't been disabled with
# the --cadvisor-port=0 Kubelet flag).
#
# This job is not necessary and should be removed in Kubernetes 1.6 and
# earlier versions, or it will cause the metrics to be scraped twice.
- job_name: 'kubernetes-cadvisor'

  # Default to scraping over https. If required, just disable this or change to
  # `http`.
  scheme: https

  # This TLS & bearer token file config is used to connect to the actual scrape
  # endpoints for cluster components. This is separate to discovery auth
  # configuration because discovery & scraping are two separate concerns in
  # Prometheus. The discovery auth config is automatic if Prometheus runs inside
  # the cluster. Otherwise, more config options have to be provided within the
  # <kubernetes_sd_config>.
  tls_config:
    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

  kubernetes_sd_configs:
  - role: node

  relabel_configs:
  - action: labelmap
    regex: __meta_kubernetes_node_label_(.+)
  - target_label: __address__
    replacement: kubernetes.default.svc:443
  - source_labels: [__meta_kubernetes_node_name]
    regex: (.+)
    target_label: __metrics_path__
    replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor

# Scrape config for service endpoints.
#
# The relabeling allows the actual service scrape endpoint to be configured
# via the following annotations:
#
# * `prometheus.io/scrape`: Only scrape services that have a value of `true`
# * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need
# to set this to `https` & most likely set the `tls_config` of the scrape config.
# * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
# * `prometheus.io/port`: If the metrics are exposed on a different port to the
# service then set this appropriately.
- job_name: 'kubernetes-service-endpoints'

  kubernetes_sd_configs:
  - role: endpoints

  relabel_configs:
  - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
    action: keep
    regex: true
  - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
    action: replace
    target_label: __scheme__
    regex: (https?)
  - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
    action: replace
    target_label: __metrics_path__
    regex: (.+)
  - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
    action: replace
    target_label: __address__
    regex: ([^:]+)(?::\d+)?;(\d+)
    replacement: $1:$2
  - action: labelmap
    regex: __meta_kubernetes_service_label_(.+)
  - source_labels: [__meta_kubernetes_namespace]
    action: replace
    target_label: kubernetes_namespace
  - source_labels: [__meta_kubernetes_service_name]
    action: replace
    target_label: kubernetes_name

# Example scrape config for probing services via the Blackbox Exporter.
#
# The relabeling allows the actual service scrape endpoint to be configured
# via the following annotations:
#
# * `prometheus.io/probe`: Only probe services that have a value of `true`
- job_name: 'kubernetes-services'

  metrics_path: /metrics

  kubernetes_sd_configs:
  - role: service

  relabel_configs:
  - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
    action: keep
    regex: true
  - source_labels: [__address__]
    target_label: __param_target
  - target_label: __address__
    replacement: blackbox-exporter.example.com:9115
  - source_labels: [__param_target]
    target_label: instance
  - action: labelmap
    regex: __meta_kubernetes_service_label_(.+)
  - source_labels: [__meta_kubernetes_namespace]
    target_label: kubernetes_namespace
  - source_labels: [__meta_kubernetes_service_name]
    target_label: kubernetes_name

# Example scrape config for probing ingresses via the Blackbox Exporter.
#
# The relabeling allows the actual ingress scrape endpoint to be configured
# via the following annotations:
#
# * `prometheus.io/probe`: Only probe services that have a value of `true`
- job_name: 'kubernetes-ingresses'

  metrics_path: /probe
  params:
    module: [http_2xx]

  kubernetes_sd_configs:
    - role: ingress

  relabel_configs:
    - source_labels: [__meta_kubernetes_ingress_annotation_prometheus_io_probe]
      action: keep
      regex: true
    - source_labels: [__meta_kubernetes_ingress_scheme,__address__,__meta_kubernetes_ingress_path]
      regex: (.+);(.+);(.+)
      replacement: ${1}://${2}${3}
      target_label: __param_target
    - target_label: __address__
      replacement: blackbox-exporter.example.com:9115
    - source_labels: [__param_target]
      target_label: instance
    - action: labelmap
      regex: __meta_kubernetes_ingress_label_(.+)
    - source_labels: [__meta_kubernetes_namespace]
      target_label: kubernetes_namespace
    - source_labels: [__meta_kubernetes_ingress_name]
      target_label: kubernetes_name

# Example scrape config for pods
#
# The relabeling allows the actual pod scrape endpoint to be configured via the
# following annotations:
#
# * `prometheus.io/scrape`: Only scrape pods that have a value of `true`
# * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
# * `prometheus.io/port`: Scrape the pod on the indicated port instead of the
# pod's declared ports (default is a port-free target if none are declared).
- job_name: 'kubernetes-pods'

  kubernetes_sd_configs:
  - role: pod

  relabel_configs:
  - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
    action: keep
    regex: true
  - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
    action: replace
    target_label: __metrics_path__
    regex: (.+)
  - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
    action: replace
    regex: ([^:]+)(?::\d+)?;(\d+)
    replacement: $1:$2
    target_label: __address__
  - action: labelmap
    regex: __meta_kubernetes_pod_label_(.+)
  - source_labels: [__meta_kubernetes_namespace]
    action: replace
    target_label: kubernetes_namespace
  - source_labels: [__meta_kubernetes_pod_name]
    action: replace
    target_label: kubernetes_pod_name

alerting:
  alertmanagers:
  - static_configs:
    - targets:
       - alertmanager:9093

规则文件

/etc/prometheus_rules $ ls -l
total 0
lrwxrwxrwx    1 root     root            17 Apr 12 13:50 admin.rule -> ..data/admin.rule
lrwxrwxrwx    1 root     root            20 Apr 12 13:50 automoto.rule -> ..data/automoto.rule
lrwxrwxrwx    1 root     root            25 Apr 12 13:50 ceres-dev.rule.yml -> ..data/ceres-dev.rule.yml
lrwxrwxrwx    1 root     root            19 Apr 12 13:50 cicd-17.rule -> ..data/cicd-17.rule
lrwxrwxrwx    1 root     root            19 Apr 17 10:18 default.rule -> ..data/default.rule
lrwxrwxrwx    1 root     root            18 Apr 12 13:50 devops.rule -> ..data/devops.rule
lrwxrwxrwx    1 root     root            20 Apr 12 13:50 dujingya.rule -> ..data/dujingya.rule
lrwxrwxrwx    1 root     root            19 Apr 12 13:50 earth-2.rule -> ..data/earth-2.rule
lrwxrwxrwx    1 root     root            19 Apr 12 13:50 earth-3.rule -> ..data/earth-3.rule
lrwxrwxrwx    1 root     root            17 Apr 12 13:50 earth.rule -> ..data/earth.rule
lrwxrwxrwx    1 root     root            20 Apr 12 13:50 guoyiqin.rule -> ..data/guoyiqin.rule
lrwxrwxrwx    1 root     root            25 Apr 12 13:50 lijiaob-space.rule -> ..data/lijiaob-space.rule
lrwxrwxrwx    1 root     root            19 Apr 12 13:50 newcicd.rule -> ..data/newcicd.rule
lrwxrwxrwx    1 root     root            19 Apr 12 13:50 newteam.rule -> ..data/newteam.rule
lrwxrwxrwx    1 root     root            18 Apr 12 13:50 qateam.rule -> ..data/qateam.rule
lrwxrwxrwx    1 root     root            22 Apr 12 13:50 taijianxin.rule -> ..data/taijianxin.rule
lrwxrwxrwx    1 root     root            23 Apr 12 13:50 weihongweic.rule -> ..data/weihongweic.rule
lrwxrwxrwx    1 root     root            19 Apr 12 13:50 yaoweig.rule -> ..data/yaoweig.rule
lrwxrwxrwx    1 root     root            23 Apr 12 13:50 zhanghongyi.rule -> ..data/zhanghongyi.rule
lrwxrwxrwx    1 root     root            20 Apr 12 13:50 zhanghub.rule -> ..data/zhanghub.rule
/etc/prometheus_rules $ pwd
/etc/prometheus_rules

/etc/prometheus_rules $ cat admin.rule 
groups:
- name: admin.rule
  rules:
  - alert: container_cpu_usage_seconds_total_gt_9000_788a0a004db0bdc447d8f0ed31125bb7
    expr: (ceil(sum(rate(container_cpu_usage_seconds_total{namespace="admin",pod_name=~"^fffff-[0-9]{5,15}-[0-9a-zA-Z]{5}$"}[5m]))
      * 100 * 100)/100) > 90
    labels:
      tenxClusterID: CID-ca4135da3326
      tenxNamespace: admin
      tenxStrategyID: STRAID-L3PPd77yxNR6
      tenxStrategyName: tsfffzz
      tenxTargetName: fffff
      tenxTargetType: service
    annotations:
      condition: CPU利用率 > 90%
      createTime: 2017-11-28T15:56:40+08:00
      currentValue: CPU利用率 {{ $value }}%
      tenxMetricType: cpu/usage_rate
      tokenMD5: 9f834d0df08163e97b1ebb8423003292
  - alert: container_cpu_usage_seconds_total_gt_9000_788a0a004db0bdc447d8f0ed31125bb7
    expr: (ceil(sum(rate(container_cpu_usage_seconds_total{namespace="admin",pod_name=~"^fffff-[0-9]{5,15}-[0-9a-zA-Z]{5}$"}[5m]))
      * 100 * 100)/100) > 90
    labels:
      tenxClusterID: CID-ca4135da3326
      tenxNamespace: admin
      tenxStrategyID: STRAID-L3PPd77yxNR6
      tenxStrategyName: tsfffzz
      tenxTargetName: fffff
      tenxTargetType: service
    annotations:
      condition: CPU利用率 > 90%
      createTime: 2017-11-28T15:56:12+08:00
      currentValue: CPU利用率 {{ $value }}%
      tenxMetricType: cpu/usage_rate
      tokenMD5: 9f834d0df08163e97b1ebb8423003292

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  labels:
    app: prometheus
    plugin: prometheus
  name: prometheus
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus
      plugin: prometheus
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: prometheus
        plugin: prometheus
    spec:
      containers:
      - args:
        - --config.file=/etc/prometheus/prometheus.yaml
        - --storage.tsdb.path=/prometheus
        - --storage.tsdb.retention=7d
        - --web.enable-lifecycle
        command:
        - /bin/prometheus
        image: harbor.enncloud.cn/enncloud/prometheus:2.0
        imagePullPolicy: IfNotPresent
        name: prometheus
        ports:
        - containerPort: 9090
          protocol: TCP
        resources:
          limits:
            cpu: "2"
            memory: 3000Mi
          requests:
            cpu: 10m
            memory: 10Mi
        volumeMounts:
        - mountPath: /prometheus
          name: data
        - mountPath: /etc/prometheus
          name: config-volume
        - mountPath: /etc/prometheus_rules
          name: rules-volume
      - args:
        - 'while inotifywait -qq -e modify,create,delete /etc/prometheus_rules/..data/;
          do sh -c  "curl -X POST http://localhost:9090/-/reload"; done; '
        image: harbor.enncloud.cn/tenx_containers/inotify:1.0
        imagePullPolicy: IfNotPresent
        name: notify
        volumeMounts:
        - mountPath: /etc/prometheus_rules
          name: rules-volume
      dnsPolicy: ClusterFirst
      nodeName: test-slave-116
      restartPolicy: Always
      volumes:
      - hostPath:
          path: /paas/prometheus_data
        name: data
      - configMap:
          name: prometheus-config
        name: config-volume
      - configMap:
          name: prometheus-rules
        name: rules-volume