清理节点中孤儿pod

原因

• 公司一直使用的1.13版本k8s，经常会出现pod删除后，目录未清理的情况产生日志报错
• 孤儿pod越来越多，浪费的磁盘空间无法释放，且报警日志error数过多

解决方案

编写了以下脚本并添加crontab实现自动解决该问题。

#!/bin/bash
#worker节点孤儿pod处理
>/tmp/orphanedMount.txt && >/tmp/orphanedPod.txt

IFS=$'\n'    #设置默认分隔符

echo `date` >>/data/logs/orphanedPod.log;echo '异常挂载目录：' >>/data/logs/orphanedPod.log
#获取异常挂载目录
for i in `tail -n10 /data/logs/kubernetes/kubelet/kubelet.log`;do
  if [[ $i =~ "transport endpoint is not connected occurred during checking mounted volumes from disk" ]];then
    echo `echo $i | awk -F : '{print $(NF -1)}'|awk '{print $NF}'` >>/tmp/orphanedMount.txt;
  fi;
done

#umount异常挂载目录
for i in `cat /tmp/orphanedMount.txt|uniq`;do
  echo $i "is error mount" >>/data/logs/orphanedPod.log;
  umount $i;
  echo $i "deleted" >>/data/logs/orphanedPod.log;
done

echo '等待60s后查询孤儿pod日志' >>/data/logs/orphanedPod.log
sleep 60

echo '孤儿pod报错：' >>/data/logs/orphanedPod.log
#获取孤儿pod报错
for i in `tail -n10 /data/logs/kubernetes/kubelet/kubelet.log`;do
  if [[ $i =~ "Orphaned pod" ]];then
    echo `echo $i| awk -F '\"' '{print $2}'` >>/tmp/orphanedPod.txt;
  fi;
done

#删除孤儿pod目录
for i in `cat /tmp/orphanedPod.txt|uniq`;do
  echo $i "is orphaned pod" >>/data/logs/orphanedPod.log;
  mv /data/kube/kubelet/pods/$i /tmp;
  echo $i "deleted" >>/data/logs/orphanedPod.log;
done

INF=' '        #设置默认分隔符