cAdvisor

cAdvisor默认是安装的，可以通过浏览器访问节点的4194端口进行测试。
cAdvisor会收集当前节点的信息和容器的信息，主要会收集网络和磁盘IO，文件系统大小、内存使用情况等。

但是，cAdvisor只会收集实时的数据，并不会做持久化存储

cat influxdb.yaml 
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: monitoring-influxdb
  namespace: kube-system
spec:
  replicas: 1
  template:
    metadata:
      labels:
        task: monitoring
        k8s-app: influxdb
    spec:
      containers:
      - name: influxdb
        image: registry.cn-hangzhou.aliyuncs.com/google-containers/heapster-influxdb-amd64:v1.1.1
        volumeMounts:
        - mountPath: /data
          name: influxdb-storage
      volumes:
      - name: influxdb-storage
        emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
  labels:
    task: monitoring
    kubernetes.io/cluster-service: 'true'
    kubernetes.io/name: monitoring-influxdb
  name: monitoring-influxdb
  namespace: kube-system
spec:
  ports:
  - port: 8086
    targetPort: 8086
  selector:
    k8s-app: influxdb

创建pod和service：

kubectl create -f influxdb.yaml

查看执行结果：

[root@devops-k8s-master1 ~]# kubectl get all -n kube-system
NAME                                        READY     STATUS    RESTARTS   AGE
pod/coredns-77c989547b-kn4sg                1/1       Running   2          12d
pod/coredns-77c989547b-ss8dr                1/1       Running   2          12d
pod/heapster-67b4f6cfdd-9pmws               1/1       Running   0          1h
pod/kubernetes-dashboard-66c9d98865-6p57d   1/1       Running   3          11d
pod/monitoring-grafana-544dd6cf85-hbdmc     1/1       Running   0          17m
pod/monitoring-influxdb-6cd98fd78f-jmtz9    1/1       Running   0          2h

NAME                           TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)         AGE
service/coredns                ClusterIP   10.1.0.2       <none>        53/UDP,53/TCP   12d
service/heapster               ClusterIP   10.1.9.81      <none>        80/TCP          1h
service/kubernetes-dashboard   ClusterIP   10.1.241.64    <none>        443/TCP         11d
service/monitoring-grafana     NodePort    10.1.67.82     <none>        80:34890/TCP    17m
service/monitoring-influxdb    ClusterIP   10.1.188.111   <none>        8086/TCP        2h

NAME                                   DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/coredns                2         2         2            2           12d
deployment.apps/heapster               1         1         1            1           1h
deployment.apps/kubernetes-dashboard   1         1         1            1           11d
deployment.apps/monitoring-grafana     1         1         1            1           17m
deployment.apps/monitoring-influxdb    1         1         1            1           2h

NAME                                              DESIRED   CURRENT   READY     AGE
replicaset.apps/coredns-77c989547b                2         2         2         12d
replicaset.apps/heapster-67b4f6cfdd               1         1         1         1h
replicaset.apps/kubernetes-dashboard-66c9d98865   1         1         1         11d
replicaset.apps/monitoring-grafana-544dd6cf85     1         1         1         17m
replicaset.apps/monitoring-influxdb-6cd98fd78f    1         1         1         2h

[root@devops-k8s-master1 ~]# kubectl describe pod -n kube-system monitoring-influxdb-6cd98fd78f-jmtz9 
Name:           monitoring-influxdb-6cd98fd78f-jmtz9
Namespace:      kube-system
Node:           172.16.8.85/172.16.8.85
Start Time:     Sun, 24 Jun 2018 21:56:03 +0800
Labels:         k8s-app=influxdb
                pod-template-hash=2785498349
                task=monitoring
Annotations:    <none>
Status:         Running
IP:             10.2.84.2
Controlled By:  ReplicaSet/monitoring-influxdb-6cd98fd78f
Containers:
  influxdb:
    Container ID:   docker://5afd5e6b962be5b15aa5c7f01be339c403318c4bdf1c0b07df561cac55cd9138
    Image:          registry.cn-hangzhou.aliyuncs.com/google-containers/heapster-influxdb-amd64:v1.1.1
    Image ID:       docker-pullable://registry.cn-hangzhou.aliyuncs.com/google-containers/heapster-influxdb-amd64@sha256:15d48e8a864d73e763f6eb7a010e93902cf625967062bdf25a055098a6aac634
    Port:           <none>
    Host Port:      <none>
    State:          Running
      Started:      Sun, 24 Jun 2018 21:58:05 +0800
    Ready:          True
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /data from influxdb-storage (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from default-token-wzlwp (ro)
Conditions:
  Type           Status
  Initialized    True 
  Ready          True 
  PodScheduled   True 
Volumes:
  influxdb-storage:
    Type:    EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:  
  default-token-wzlwp:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  default-token-wzlwp
    Optional:    false
QoS Class:       BestEffort
Node-Selectors:  <none>
Tolerations:     <none>
Events:          <none>

Heapster

cat heapster.yaml 
apiVersion: v1
kind: ServiceAccount
metadata:
  name: heapster
  namespace: kube-system

---

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: heapster
subjects:
  - kind: ServiceAccount
    name: heapster
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: cluster-admin
  apiGroup: rbac.authorization.k8s.io

---

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: heapster
  namespace: kube-system
spec:
  replicas: 1
  template:
    metadata:
      labels:
        task: monitoring
        k8s-app: heapster
    spec:
      serviceAccountName: heapster
      containers:
      - name: heapster
        image: registry.cn-hangzhou.aliyuncs.com/google-containers/heapster-amd64:v1.4.2
        imagePullPolicy: IfNotPresent
        command:
        - /heapster
        - --source=kubernetes:https://kubernetes.default
        - --sink=influxdb:http://monitoring-influxdb:8086

---

apiVersion: v1
kind: Service
metadata:
  labels:
    task: monitoring
    kubernetes.io/cluster-service: 'true'
    kubernetes.io/name: Heapster
  name: heapster
  namespace: kube-system
spec:
  ports:
  - port: 80
    targetPort: 8082
  selector:
    k8s-app: heapster

创建pod和service：

kubectl create -f heapster.yaml

Grafana

cat grafana.yaml 
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: monitoring-grafana
  namespace: kube-system
spec:
  replicas: 1
  template:
    metadata:
      labels:
        task: monitoring
        k8s-app: grafana
    spec:
      containers:
      - name: grafana
        image: registry.cn-hangzhou.aliyuncs.com/google-containers/heapster-grafana-amd64:v4.4.1
        ports:
          - containerPort: 3000
            protocol: TCP
        volumeMounts:
        - mountPath: /var
          name: grafana-storage
        env:
        - name: INFLUXDB_HOST
          value: monitoring-influxdb
        - name: GRAFANA_PORT
          value: "3000"
          # The following env variables are required to make Grafana accessible via
          # the kubernetes api-server proxy. On production clusters, we recommend
          # removing these env variables, setup auth for grafana, and expose the grafana
          # service using a LoadBalancer or a public IP.
        - name: GF_AUTH_BASIC_ENABLED
          value: "false"
        - name: GF_AUTH_ANONYMOUS_ENABLED
          value: "true"
        - name: GF_AUTH_ANONYMOUS_ORG_ROLE
          value: Admin
        - name: GF_SERVER_ROOT_URL
          # If you're only using the API Server proxy, set this value instead:
          #value: /api/v1/proxy/namespaces/kube-system/services/monitoring-grafana/
          #value: /
      volumes:
      - name: grafana-storage
        emptyDir: {}

---

apiVersion: v1
kind: Service
metadata:
  labels:
    # For use as a Cluster add-on (https://github.com/kubernetes/kubernetes/tree/master/cluster/addons)
    # If you are NOT using this as an addon, you should comment out this line.
    kubernetes.io/cluster-service: 'true'
    kubernetes.io/name: monitoring-grafana
  name: monitoring-grafana
  namespace: kube-system
spec:
  # In a production setup, we recommend accessing Grafana through an external Loadbalancer
  # or through a public IP.
  # type: LoadBalancer
  type: NodePort
  ports:
  - port : 80
    targetPort: 3000
  selector:
    k8s-app: grafana

创建pod和service：

kubectl create -f grafana.yaml

想要访问grafana有两种方式：第一种是通过apiserver的非安全端口进行访问，第二种是通过proxy代理的端口访问

[root@devops-k8s-master1 ~]# kubectl cluster-info
Kubernetes master is running at https://172.16.8.81:6443
CoreDNS is running at https://172.16.8.81:6443/api/v1/namespaces/kube-system/services/coredns:dns/proxy
Heapster is running at https://172.16.8.81:6443/api/v1/namespaces/kube-system/services/heapster/proxy
kubernetes-dashboard is running at https://172.16.8.81:6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy
monitoring-grafana is running at https://172.16.8.81:6443/api/v1/namespaces/kube-system/services/monitoring-grafana/proxy
monitoring-influxdb is running at https://172.16.8.81:6443/api/v1/namespaces/kube-system/services/monitoring-influxdb/proxy

To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.
[root@devops-k8s-master1 ~]# kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}')
Name:         admin-user-token-9fgdm
Namespace:    kube-system
Labels:       <none>
Annotations:  kubernetes.io/service-account.name=admin-user
              kubernetes.io/service-account.uid=e7b6f3b7-6e5d-11e8-af2b-000c29ee0a78

Type:  kubernetes.io/service-account-token

Data
====
ca.crt:     1359 bytes
namespace:  11 bytes
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi11c2VyLXRva2VuLTlmZ2RtIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImFkbWluLXVzZXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiJlN2I2ZjNiNy02ZTVkLTExZTgtYWYyYi0wMDBjMjllZTBhNzgiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZS1zeXN0ZW06YWRtaW4tdXNlciJ9.DS4WilJF59Fg2yjxPNRXM0JAC5UzGHASpI_0_ftKmyEC_nt1a-EFsUeu7Az5O9TRj4R2zDNOFFXtf8zQbeskUpxKeed8nqRpPoU-_Xs2Q5dTyp6p4cNxG7e6auiH--0omZVCTI2iHg8etK4JWhwSdTMHXOY9mXefVKK_VdNFBZbC1959HuvjlNcK-r0Euv-FKyYdjCSNnLQ7Ky3NcBhZxBmyP-1yWmUBVWViQuimUSkxP3fezGCSmgGonOwSHbVhVNp5IJhowH7Z6B7X6P1x-sIENeBy9P1L6TUmzXGARnix7ms2YLKJ1bhodZ0-iAX0YYFo-wlk9S5BegmQc9wt9g