管理k8s组件日志

• K8s系统的组件日志

  # 除了kubelet外，其他组件都容器化了，可以使用journalctl 来查看systemd管理的组件kubelet
  # journalctl  -u kubelet | grep error
  11月 01 20:26:00 k8s-master kubelet[985]: E1101 20:26:00.402299     985 controller.go:144] failed to ensure lease exists, will retry in 800ms, error: Get "https://10.0.0.100:6443/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/k8s-master?timeout=10s": dial tcp 10.0.0.100:6443: connect: connection refused
  ​
  

• K8s Cluster里面部署的应用程序日志

  • 标准输出

    # 容器中的组件用kubectl logs podName查看，输出到控制台(被docker接管)
    # kubectl logs my-dep-5b7868d854-z9k5r  -f
    192.168.36.64 - - [01/Nov/2021:12:50:31 +0000] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36" "-"
    # 日志获取流程
    kubectl logs -->apiserver -->kubelet-->container
    ​
    # 查看proxy组件的日志
    # kubectl logs kube-proxy-7rxb6 -n kube-system
    ​
    # 容器中的日志在本地宿主机的位置：
    /var/lib/docker/containers/<container-id>/<container-id>-json.log
    ​
    1. 查找过程：
    先确定pod在那个node上，通过 -o wide 可以确定
    #  kubectl get pods kube-proxy-9dch7  -n kube-system -o wide
    NAME               READY   STATUS    RESTARTS      AGE    IP           NODE        NOMINATED NODE   READINESS GATES
    kube-proxy-9dch7   1/1     Running   4 (21h ago)   2d7h   10.0.0.101   k8s-node1   <none>           <none>
    2. 进入对应节点：查找container-id
    [root@k8s-node1 ~]# docker ps | grep kube-proxy-9dch7
    aefd56e468d3   6120bd723dce                                        "/usr/local/bin/kube…"   33 minutes ago   Up 33 minutes             k8s_kube-proxy_kube-proxy-9dch7_kube-system_5552bec6-86e9-4d46-ad7d-63da81a27058_4
    f0218b79a478   registry.aliyuncs.com/google_containers/pause:3.5   "/pause"                 33 minutes ago   Up 33 minutes             k8s_POD_kube-proxy-9dch7_kube-system_5552bec6-86e9-4d46-ad7d-63da81a27058_4
    container-id = aefd56e468d3
    3. 根据container-id查找宿主机上对应容器的日志，宿主机中日志为json格式
     cat /var/lib/docker/containers/aefd56e468d3<tab>/aefd56e468d3<tab>-json.log 
      ....
     {"log":"I1101 12:26:58.206292       1 shared_informer.go:247] Caches are synced for endpoint slice config \n","stream":"stderr","time":"2021-11-01T12:26:58.206436402Z"}
     4. 查看容器日志是否一致（master 节点执行）
     # kubectl logs kube-proxy-9dch7 -n kube-system
     ...
     I1101 12:26:58.206292       1 shared_informer.go:247] Caches are synced for endpoint slice config 

    • 日志文件

      # cat two-files.yaml 
      apiVersion: v1
      kind: Pod
      metadata:
        name: counter
      spec:
        containers:
        - name: count
          image: busybox
          args:
          - /bin/sh
          - -c
          - >
            i=0;
            while true;
            do
              echo "$i: $(date)" >> /var/log/1.log;
              echo "$(date) INFO $i" >> /var/log/2.log;
              i=$((i+1));
              sleep 1;
            done      
          volumeMounts:
          - name: varlog
            mountPath: /var/log
        volumes:
        - name: varlog
          emptyDir: {}
      ​
      # kubectl apply -f two-files.yaml
      pod/counter created
      ​
      # kubectl get pod -o wide
      NAME                      READY   STATUS    RESTARTS      AGE   IP                NODE        NOMINATED NODE   READINESS GATES
      counter                   1/1     Running   0             81s   192.168.36.75     k8s-node1   <none>           <none>
      ​
      ​
      ​
      # 文件日志在宿主机中的位置：/var/lib/kubelet/pods/<pod-id>/volumes/kubernetes.io~empty-dir/
      获取Pod-id（到对应node上）
      ]# docker ps | grep counter
      ed231e225433   busybox                                             "/bin/sh -c 'i=0; wh…"   About a minute ago   Up About a minute             k8s_count_counter_default_0d70c728-eb23-4c15-a2ea-68743d856023_0
      9123abb4fa34   registry.aliyuncs.com/google_containers/pause:3.5   "/pause"                 2 minutes ago        Up 2 minutes                  k8s_POD_counter_default_0d70c728-eb23-4c15-a2ea-68743d856023_0
      ​
       根据pod-id在宿主机上查找日志
      # # cat /var/lib/kubelet/pods/0d70c728-eb23-4c15-a2ea-68743d856023/volumes/kubernetes.io~empty-dir/varlog/1.log 
      ​
      ​
      ​

      边车模式

      apiVersion: v1
      kind: Pod
      metadata:
        name: counter1
      spec:
        containers:
        - name: count1
          image: busybox
          args:
          - /bin/sh
          - -c
          - >
            i=0;
            while true;
            do
              echo "$i: $(date)" >> /var/log/1.log;
              echo "$(date) INFO $i" >> /var/log/2.log;
              i=$((i+1));
              sleep 1;
            done      
          volumeMounts:
          - name: varlog
            mountPath: /var/log
            
        - name: count1-log
          image: busybox
          args: [/bin/sh, -c, 'tail -n+1 -f /var/log/1.log']
          volumeMounts:
          - name: varlog
            mountPath: /var/log
        - name: count2-log
          image: busybox
          args: [/bin/sh, -c, 'tail -n+1 -f /var/log/2.log']
          volumeMounts:
          - name: varlog
            mountPath: /var/log
      
        volumes:
        - name: varlog
          emptyDir: {}
      
      ​
      ​

      1. 查看日志
      # kubectl logs counter1 count1-log
      2. 宿主机上查看
      ​
      [root@k8s-node2 web1-log]# docker ps | grep counter1
      7a92417f9aa3   busybox                                             "/bin/sh -c 'tail -n…"   2 minutes ago   Up 2 minutes             k8s_count1-log_counter1_default_c95a2ad0-3dd6-43fe-962a-6eb82284a418_0
      5e204be05e75   busybox                                             "/bin/sh -c 'i=0; wh…"   2 minutes ago   Up 2 minutes             k8s_count1_counter1_default_c95a2ad0-3dd6-43fe-962a-6eb82284a418_0
      6577994c54ac   registry.aliyuncs.com/google_containers/pause:3.5   "/pause"                 3 minutes ago   Up 3 minutes             k8s_POD_counter1_default_c95a2ad0-3dd6-43fe-962a-6eb82284a418_0
      ​
      # cat /var/lib/kubelet/pods/c95a2ad0-3dd6-43fe-962a-6eb82284a418/volumes/kubernetes.io~empty-dir/varlog/2.log 
      ​