1.k8s的监控指标

监控指标	具体实现	举例
Pod性能	cAdvisor	容器CPU，内存利用率
Node性能	node-exporter	节点CPU，内存利用率
K8S资源对象	kube-state-metrics	Pod/Deployment/Service

2.创建namespace、sa账号，在k8s集群的master节点操作

参考链接：
Kubernetes RBAC 详解
k8s部署Prometheus
如何监控k8s apiserver
#创建一个monitor-sa的名称空间

kubectl create ns monitor-sa 

#创建一个sa账号

kubectl create serviceaccount monitor -n monitor-sa  

#把sa账号monitor通过clusterrolebing绑定到clusterrole上

kubectl create clusterrolebinding monitor-clusterrolebinding -n monitor-sa --clusterrole=cluster-admin  --serviceaccount=monitor-sa:monitor

3.创建数据目录

#在k8s集群的任何一个node节点操作，因为我的k8s集群只有一个node节点node1，所以我在node1上操作如下命令：

mkdir /data/prometheus
chmod 777 /data/prometheus

4. kube-state-metric的部署

prometheus通过 sa,clusterrolebinding来解决token、证书挂载问题
sa等配置： prometheus yaml中需要配置对应的saserviceAccountName

kube-state-metrics github项目地址

.
├── cluster-role-binding.yaml
├── cluster-role.yaml
├── deployment.yaml
├── README.md
├── service-account.yaml
└── service.yaml

service-account.yaml文件

apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/version: v1.8.0
  name: kube-state-metrics
  namespace: monitor-sa

cluster-role.yaml 文件

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/version: v1.8.0
  name: kube-state-metrics
rules:
- apiGroups:
  - ""
  resources:
  - configmaps
  - secrets
  - nodes
  - pods
  - services
  - resourcequotas
  - replicationcontrollers
  - limitranges
  - persistentvolumeclaims
  - persistentvolumes
  - namespaces
  - endpoints
  verbs:
  - list
  - watch
- apiGroups:
  - extensions
  resources:
  - daemonsets
  - deployments
  - replicasets
  - ingresses
  verbs:
  - list
  - watch
- apiGroups:
  - apps
  resources:
  - statefulsets
  - daemonsets
  - deployments
  - replicasets
  verbs:
  - list
  - watch
- apiGroups:
  - batch
  resources:
  - cronjobs
  - jobs
  verbs:
  - list
  - watch
- apiGroups:
  - autoscaling
  resources:
  - horizontalpodautoscalers
  verbs:
  - list
  - watch
- apiGroups:
  - authentication.k8s.io
  resources:
  - tokenreviews
  verbs:
  - create
- apiGroups:
  - authorization.k8s.io
  resources:
  - subjectaccessreviews
  verbs:
  - create
- apiGroups:
  - policy
  resources:
  - poddisruptionbudgets
  verbs:
  - list
  - watch
- apiGroups:
  - certificates.k8s.io
  resources:
  - certificatesigningrequests
  verbs:
  - list
  - watch
- apiGroups:
  - storage.k8s.io
  resources:
  - storageclasses
  - volumeattachments
  verbs:
  - list
  - watch
- apiGroups:
  - admissionregistration.k8s.io
  resources:
  - mutatingwebhookconfigurations
  - validatingwebhookconfigurations
  verbs:
  - list
  - watch
- apiGroups:
  - networking.k8s.io
  resources:
  - networkpolicies
  verbs:
  - list
  - watch

cluster-role-binding.yaml

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  labels:
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/version: v1.8.0
  name: kube-state-metrics
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kube-state-metrics
subjects:
- kind: ServiceAccount
  name: kube-state-metrics
  namespace: monitor-sa

deployment.yaml 文件

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/version: v1.8.0
  name: kube-state-metrics
  namespace: monitor-sa
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: kube-state-metrics
  template:
    metadata:
      labels:
        app.kubernetes.io/name: kube-state-metrics
        app.kubernetes.io/version: v1.8.0
    spec:
      containers:
      - image: quay.io/coreos/kube-state-metrics:v1.8.0
        livenessProbe:
          httpGet:
            path: /healthz
            port: 8080
          initialDelaySeconds: 5
          timeoutSeconds: 5
        name: kube-state-metrics
        ports:
        - containerPort: 8080
          name: http-metrics
        - containerPort: 8081
          name: telemetry
        readinessProbe:
          httpGet:
            path: /
            port: 8081
          initialDelaySeconds: 5
          timeoutSeconds: 5
      nodeSelector:
        kubernetes.io/os: linux
      serviceAccountName: kube-state-metrics

service.yaml文件

apiVersion: v1
kind: Service
metadata:
  annotations:
    prometheus.io/scrape: 'true'
  name: kube-state-metrics
  namespace: monitor-sa
  labels:
    app: kube-state-metrics
spec:
    ports:
    - name: kube-state-metrics
      port: 8080
      protocol: TCP
    - name: telemetry
      port: 8081
      protocol: TCP
    selector:
      app.kubernetes.io/name: kube-state-metrics

5.安装prometheus，以下步骤均在在k8s集群的master节点操作

1）创建一个configmap存储卷，用来存放prometheus配置信息

kubectl get sa monitor  -n monitor-sa -o yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  creationTimestamp: "2021-05-23T14:18:14Z"
  name: monitor
  namespace: monitor-sa
  resourceVersion: "18761312"
  selfLink: /api/v1/namespaces/monitor-sa/serviceaccounts/monitor
  uid: 12ed67ab-dae8-4704-87b8-5a073a7047d2
secrets:
- name: monitor-token-p6wgp

kubectl describe sa monitor  -n monitor-sa 
Name:                monitor
Namespace:           monitor-sa
Labels:              <none>
Annotations:         <none>
Image pull secrets:  <none>
Mountable secrets:   monitor-token-p6wgp
Tokens:              monitor-token-p6wgp
Events:              <none>


kubectl describe secrets monitor-token-p6wgp -n monitor-sa 
Name:         monitor-token-p6wgp
Namespace:    monitor-sa
Labels:       <none>
Annotations:  kubernetes.io/service-account.name: monitor
              kubernetes.io/service-account.uid: 12ed67ab-dae8-4704-87b8-5a073a7047d2

Type:  kubernetes.io/service-account-token

Data
====
ca.crt:     1025 bytes
namespace:  10 bytes
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6IkY5eEJEUjZMRjNnejhxMVl6enJiYmVHX0RSOFYza1JfbVVpZmhCVXlucDQifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJtb25pdG9yLXNhIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZWNyZXQubmFtZSI6Im1vbml0b3ItdG9rZW4tcDZ3Z3AiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoibW9uaXRvciIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjEyZWQ2N2FiLWRhZTgtNDcwNC04N2I4LTVhMDczYTcwNDdkMiIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDptb25pdG9yLXNhOm1vbml0b3IifQ.j2hWAze7aOZgVDg0j4NKOhoMUktu7XIJ56kU_RCRbt7RCaXYd_A4ijg7IJqVUHBitQKfx-_ZzNXcOqMZt5nCN5dtToKGWRK_Du0eqepKNcsfj9dzVvebaEbd-4t7LyhHvEdf5M1CviD0wnrw1O_9nXl1COpm9IojJB9I8tIzs9Y3fiMVd2oTUL3ctFKRSkwM4CTAEIm5SZN0QRgld7Ol8W7F-m8jjOh3c7MMm9FnnAn_NkQ57XSKJovMy_AdMA55gwZaufCYA225tubG9KS0eUyF70wgGvAKOMFn6yGpRZjHj26JcBDhoEZkwzFrBM4-blnGl9pMHXtPztAPlw-xQQ

 cat    >prometheus-cfg.yaml <<EOF
---
kind: ConfigMap
apiVersion: v1
metadata:
  labels:
    app: prometheus
  name: prometheus-config
  namespace: monitor-sa
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s
      scrape_timeout: 10s
      external_labels:
         monitor: 'AIUI-ceshi-k8s'
      evaluation_interval: 1m

    scrape_configs:
    - job_name: kubernetes-node
      kubernetes_sd_configs:
      - role: node
      tls_config:
        insecure_skip_verify: true
      bearer_token_file: /opt/k8s/k8s.token
      relabel_configs:
      - source_labels: [__address__]
        regex: '(.*):10250'
        replacement: '${1}:9100'
        target_label: __address__
        action: replace
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - source_labels: [instance]
        regex: .*db002.*
        action: drop

    - job_name: 'kubernetes-apiservers'
      kubernetes_sd_configs:
      - role: endpoints
      scheme: https
      tls_config:
        insecure_skip_verify: true
      bearer_token_file: /opt/k8s/k8s.token
      relabel_configs:
      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
        action: keep
        regex: default;kubernetes;https

    - job_name: 'kube-state-metrics'
      static_configs:
      - targets: ['kube-state-metrics:8080']

    - job_name: 'kubernetes-node-cadvisor'
      kubernetes_sd_configs:
      - role:  node
      scheme: https
      tls_config:
        insecure_skip_verify: true
      bearer_token_file: /opt/k8s/k8s.token
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - target_label: __address__
        replacement: 172.16.154.13:6443
      - source_labels: [__meta_kubernetes_node_name]
        regex: (.+)
        target_label: __metrics_path__
        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
    
    - job_name: 'kubernetes-service-endpoints'
      scrape_timeout: 10s      
      kubernetes_sd_configs:
      - role: endpoints
      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
        action: replace
        target_label: __scheme__
        regex: (https?)
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: kubernetes_name
     # - source_labels: [__meta_kubernetes_pod_container_port_number]
     #   action: replace
     #   target_label: container_port
EOF

cat >prometheus-deploy.yaml    <<EOF
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus-server
  namespace: monitor-sa
  labels:
    app: prometheus
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus
      component: server
  template:
    metadata:
      labels:
        app: prometheus
        component: server
      annotations:
        prometheus.io/scrape: 'false'
    spec:
      serviceAccountName: monitor
      containers:
      - name: prometheus
        image: prom/prometheus:v2.2.1
        imagePullPolicy: IfNotPresent
        command:
          - 'prometheus'
          - '--config.file=/etc/prometheus/prometheus.yml'
          - '--storage.tsdb.path=/prometheus'
          - '--storage.tsdb.retention=720h'
          - '--web.enable-lifecycle'
        ports:
        - containerPort: 9090
          protocol: TCP
        volumeMounts:
        - mountPath: /etc/prometheus/prometheus.yml
          name: prometheus-config
          subPath: prometheus.yml
        - mountPath: /prometheus/
          name: prometheus-storage-volume
        - mountPath: /opt/k8s/k8s.token
          name: k8s-token
          subPath: k8s.token
      volumes:
        - name: prometheus-config
          configMap:
            name: prometheus-config
            items:
              - key: prometheus.yml
                path: prometheus.yml
                mode: 0644
        - name: prometheus-storage-volume
          hostPath:
           path: /data/prometheus
           type: Directory
        - name: k8s-token
          hostPath:
           path: /opt/k8s
           type: Directory

cat  > prometheus-svc.yaml  <<EOF
---
apiVersion: v1
kind: Service
metadata:
  annotations:
    prometheus.io/scrape: 'true'
  name: prometheus
  namespace: monitor-sa
  labels:
    app: prometheus
spec:
  type: NodePort
  ports:
    - port: 9090
      targetPort: 9090
      nodePort: 30090
      protocol: TCP
  selector:
    app: prometheus
    component: server
EOF