一 基础优化

 # 关闭防火墙
systemctl stop firewalld
systemctl disable firewalld
 
# 关闭selinux
sed -i 's/enforcing/disabled/' /etc/selinux/config  # 永久
setenforce 0  # 临时
 
# 关闭swap
swapoff -a  # 临时
sed -ri 's/.*swap.*/#&/' /etc/fstab    # 永久
 
cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
sysctl --system  # 生效
 

yum install -y bash-completion supervisor.noarch nc iftop telnet
yum install -y conntrack ntpdate ntp ipvsadm ipset jq iptables curl sysstat libseccomp wget vim net-tools git bridge-utils dstat tree
yum install -y yum-utils device-mapper-persistent-data lvm2 unzip

二 etcd apiserver证书签发

mkdir -p ~/TLS/etcd
cd ~/TLS/etcd
 
cat > ca-config.json << EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "www": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF
 
cat > ca-csr.json << EOF
{
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing"
        }
    ]
}
EOF
 
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
 
cat > server-csr.json << EOF
{
    "CN": "etcd",
    "hosts": [
    "${ETCD[0]}",
    "${ETCD[1]}",
    "${ETCD[2]}"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing"
        }
    ]
}
EOF
 
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server

mkdir -p ~/TLS/k8s
cd ~/TLS/k8s
 
cat > ca-config.json << EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF
cat > ca-csr.json << EOF
{
    "CN": "kubernetes",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF
 
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
 
cat > server-csr.json << EOF
{
    "CN": "kubernetes",
    "hosts": [
      "10.0.0.1",
      "10.71.0.1",
      "127.0.0.1",
      "${Master[0]}",
      "${Master[1]}",
      "${Master[2]}",
      "${SLBIP}",
      "kubernetes",
      "kubernetes.default",
      "kubernetes.default.svc",
      "kubernetes.default.svc.cluster",
      "kubernetes.default.svc.cluster.local"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF
 
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server

三 ETCD安装

mkdir /home/etcd/{bin,config,ssl} -p 
cp ~/TLS/etcd/{ca-key.pem,ca.pem,server-key.pem,server.pem}  /home/etcd/ssl/
cp k8s/etcd/bin/etcd /home/etcd/bin/

cat >  /home/etcd/config/etcd.yml << EOF
name: "etcd01"
data-dir: "/docker-data/etcd-data/default.etcd"
listen-peer-urls: "https://ip01:2380"
listen-client-urls: "https://ip01:2379"

quota-backend-bytes: 8589934592
auto-compaction-retention: "3"
initial-cluster: "etcd01=https://ip01:2380,etcd02=https://ip02:2380,etcd03=https://ip03:2380"
initial-cluster-token: 'etcd-cluster'
initial-cluster-state: 'new'
initial-advertise-peer-urls: "https://ip01:2380"
advertise-client-urls: "https://ip01:2379"

client-transport-security:
  cert-file: "/home/etcd/ssl/server.pem"
  key-file: "/home/etcd/ssl/server-key.pem"
  client-cert-auth: true
  trusted-ca-file: "/home/etcd/ssl/ca.pem"
peer-transport-security:
  cert-file: "/home/etcd/ssl/server.pem"
  key-file: "/home/etcd/ssl/server-key.pem"
  client-cert-auth: true
  trusted-ca-file: "/home/etcd/ssl/ca.pem"
logger: zap
EOF

cat >  /usr/lib/systemd/system/etcd.service << EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
ExecStart=/home/etcd/bin/etcd --config-file=/home/etcd/config/etcd.yml
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
#配置完成一台etcd之后可以直接scp至其他节点然后修改ip配置启动etcd
#这里需要注意etcd name 01 02 03 都是具体的标示 scp到别的机器上之后需要修改
systemctl start etcd
systemctl enable etcd

ETCDCTL_API=3 etcdctl --cacert=/home/etcd/ssl/ca.pem --cert=/home/etcd/ssl/server.pem --key=/home/etcd/ssl/server-key.pem --endpoints="https://ip01:2379,https://ip02:2379,https://ip03:2379" endpoint health

四 Docker安装

yum install -y yum-utils
yum-config-manager     --add-repo     https://download.docker.com/linux/centos/docker-ce.repo
#如果安装高版本docker  containerd会依赖高版本 需要下载rpm安装新版本
#https://download.docker.com/linux/centos/7/x86_64/stable/Packages/
wget https://download.docker.com/linux/centos/7/x86_64/stable/Packages/containerd.io-1.4.9-3.1.el7.x86_64.rpm
yum localinstall containerd.io-1.4.9-3.1.el7.x86_64.rpm -y
#安装指定版本docker
yum install docker-ce-20.10.8 -y
#创建docker目录并且挂盘
mkdir /docker-data
mkfs.xfs /dev/vdb
mount /dev/vdb /docker-data
#需要在docker service文件添加--graph /docker-data/docker --bip=10.10.201.1/24 启动参数
mkdir -p  /etc/docker
cat > /etc/docker/daemon.json << EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "bip": "10.10.201.1/24",
  "data-root": "/docker-data/docker"
}
EOF
systemctl start docker
systemctl enable  docker

五 安装Containerd(这里需要注意你是否是用docker还是用containerd来作为cri)

#安装crictl 如果使用containerd 本身自带的ctl并不好用
cat /etc/crictl.yaml
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false

cat /etc/containerd/config.toml
version = 2
root = "/docker-data/containerd-data"
#root = "/var/lib/containerd"
#state = "/run/containerd"
state = "/docker-data/containerd-state"
plugin_dir = ""
disabled_plugins = []
required_plugins = []
oom_score = 0

[grpc]
  address = "/run/containerd/containerd.sock"
  tcp_address = ""
  tcp_tls_cert = ""
  tcp_tls_key = ""
  uid = 0
  gid = 0
  max_recv_message_size = 16777216
  max_send_message_size = 16777216

[ttrpc]
  address = ""
  uid = 0
  gid = 0

[debug]
  address = ""
  uid = 0
  gid = 0
  level = ""

[metrics]
  address = ""
  grpc_histogram = false

[cgroup]
  path = ""

[timeouts]
  "io.containerd.timeout.shim.cleanup" = "5s"
  "io.containerd.timeout.shim.load" = "5s"
  "io.containerd.timeout.shim.shutdown" = "3s"
  "io.containerd.timeout.task.state" = "2s"

[plugins]
  [plugins."io.containerd.gc.v1.scheduler"]
    pause_threshold = 0.02
    deletion_threshold = 0
    mutation_threshold = 100
    schedule_delay = "0s"
    startup_delay = "100ms"
  [plugins."io.containerd.grpc.v1.cri"]
    disable_tcp_service = true
    stream_server_address = "127.0.0.1"
    stream_server_port = "0"
    stream_idle_timeout = "4h0m0s"
    enable_selinux = false
    selinux_category_range = 1024
    sandbox_image = "harbor.hualala.com/base/pause:3.1"
    stats_collect_period = 10
    systemd_cgroup = false
    enable_tls_streaming = false
    max_container_log_line_size = 16384
    disable_cgroup = false
    disable_apparmor = false
    restrict_oom_score_adj = false
    max_concurrent_downloads = 3
    disable_proc_mount = false
    unset_seccomp_profile = ""
    tolerate_missing_hugetlb_controller = true
    disable_hugetlb_controller = true
    ignore_image_defined_volumes = false
    [plugins."io.containerd.grpc.v1.cri".containerd]
      snapshotter = "overlayfs"
      default_runtime_name = "runc"
      no_pivot = false
      disable_snapshot_annotations = true
      discard_unpacked_layers = false
      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
        runtime_type = ""
        runtime_engine = ""
        runtime_root = ""
        privileged_without_host_devices = false
        base_runtime_spec = ""
      [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
        runtime_type = ""
        runtime_engine = ""
        runtime_root = ""
        privileged_without_host_devices = false
        base_runtime_spec = ""
      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
          runtime_type = "io.containerd.runc.v2"
          runtime_engine = ""
          runtime_root = ""
          privileged_without_host_devices = false
          base_runtime_spec = ""
          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
            SystemdCgroup = true
    [plugins."io.containerd.grpc.v1.cri".cni]
      bin_dir = "/opt/cni/bin"
      conf_dir = "/etc/cni/net.d"
      max_conf_num = 1
      conf_template = ""
    [plugins."io.containerd.grpc.v1.cri".registry]
      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
          endpoint = ["https://registry-1.docker.io"]
    [plugins."io.containerd.grpc.v1.cri".image_decryption]
      key_model = ""
    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
      tls_cert_file = ""
      tls_key_file = ""
  [plugins."io.containerd.internal.v1.opt"]
    path = "/docker-data/containerd/"
  [plugins."io.containerd.internal.v1.restart"]
    interval = "10s"
  [plugins."io.containerd.metadata.v1.bolt"]
    content_sharing_policy = "shared"
  [plugins."io.containerd.monitor.v1.cgroups"]
    no_prometheus = false
  [plugins."io.containerd.runtime.v1.linux"]
    shim = "containerd-shim"
    runtime = "runc"
    runtime_root = ""
    no_shim = false
    shim_debug = false
  [plugins."io.containerd.runtime.v2.task"]
    platforms = ["linux/amd64"]
  [plugins."io.containerd.service.v1.diff-service"]
    default = ["walking"]
  [plugins."io.containerd.snapshotter.v1.devmapper"]
    root_path = ""
    pool_name = ""
    base_image_size = ""
    async_remove = false

六 Master安装

1.apiserver

mkdir -p /home/kubernetes/{bin,config,ssl}
mkdir -p /www-volume/logs/{kube-apiserver,kube-controller,kube-scheduler,kubelet,kube-proxy}
cp TLS/k8s/{ca.pem,ca-key.pem,server-key.pem,server.pem}  /home/kubernetes/ssl/
cp -r k8s/kubernetes/bin/   /home/kubernetes/bin/
cp -r  k8s/cni /opt/
  
cat > /home/kubernetes/config/token.csv << EOF
e537d04b49b03df79cf29c2b53bfcfae,kubelet-bootstrap,10001,"system:node-bootstrapper"
EOF
head -c 16 /dev/urandom | od -An -t x | tr -d ' '

cat > /home/kubernetes/config/kube-api-auditlog.yaml << EOF
apiVersion: audit.k8s.io/v1beta1
kind: Policy
rules:
  - level: None
    resources:
    - group: ""
      resources: ["configmaps"]
      resourceNames: ["controller-leader"]
  - level: None
    users: ["system:kube-proxy"]
    verbs: ["watch"]
    resources:
    - group: "" # core API group
      resources: ["endpoints", "services"]
  - level: None
    userGroups: ["system:authenticated"]
    nonResourceURLs:
    - "/api*" # Wildcard matching.
    - "/version"
  - level: Metadata
EOF

cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes

[Service]
ExecStart=/home/kubernetes/bin/kube-apiserver --logtostderr=false \
--v=2 \
--default-not-ready-toleration-seconds=60 \
--default-unreachable-toleration-seconds=60 \
--log-dir=/www-volume/k8s-logs/kube-apiserver \
--log-file=/www-volume/k8s-logs/kube-apiserver/kube-apiserver.log \
--log-file-max-size=1024 \
--etcd-servers=https://ip01:2379,https://ip02:2379,https://ip03:2379 \
--bind-address=ip01 \
--secure-port=6443 \
--advertise-address=ip01 \
--allow-privileged=true \
--service-cluster-ip-range=10.5.0.0/16 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction,DefaultTolerationSeconds \
--authorization-mode=RBAC,Node \
--api-audiences=api,istio-ca \
--enable-bootstrap-token-auth=true \
--token-auth-file=/home/kubernetes/config/token.csv \
--service-node-port-range=30000-32767 \
--kubelet-client-certificate=/home/kubernetes/ssl/server.pem \
--kubelet-client-key=/home/kubernetes/ssl/server-key.pem \
--tls-cert-file=/home/kubernetes/ssl/server.pem  \
--tls-private-key-file=/home/kubernetes/ssl/server-key.pem \
--client-ca-file=/home/kubernetes/ssl/ca.pem \
--service-account-key-file=/home/kubernetes/ssl/ca-key.pem \
--service-account-issuer=api,kubernetes.default.svc \
--service-account-signing-key-file=/home/kubernetes/ssl/server-key.pem \
--etcd-cafile=/home/etcd/ssl/ca.pem \
--etcd-certfile=/home/etcd/ssl/server.pem \
--etcd-keyfile=/home/etcd/ssl/server-key.pem \
--requestheader-client-ca-file=/home/kubernetes/ssl/ca.pem \
--proxy-client-cert-file=/home/kubernetes/ssl/server.pem \
--proxy-client-key-file=/home/kubernetes/ssl/server-key.pem \
--requestheader-allowed-names=kubernetes \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-group-headers=X-Remote-Group \
--requestheader-username-headers=X-Remote-User \
--enable-aggregator-routing=true \
--audit-log-maxage=15 \
--feature-gates=EphemeralContainers=true,RemoveSelfLink=false \
--audit-log-compress \
--audit-log-mode=batch \
--audit-policy-file=/home/kubernetes/config/kube-api-auditlog.yaml \
--audit-log-path=/www-volume/k8s-logs/kubernetes-audit/kuberntes-audit.log
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

2.kube-controller

cd ~/TLS/k8s
  
cat > kube-controller-manager-csr.json << EOF
{
  "CN": "system:kube-controller-manager",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager
  
KUBE_CONFIG="/home/kubernetes/config/kube-controller-manager.kubeconfig"
KUBE_APISERVER="https://ip01:6443"

kubectl config set-cluster kubernetes \
  --certificate-authority=/home/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-credentials kube-controller-manager \
  --client-certificate=./kube-controller-manager.pem \
  --client-key=./kube-controller-manager-key.pem \
  --embed-certs=true \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-controller-manager \
  --kubeconfig=${KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${KUBE_CONFIG}

cat > /usr/lib/systemd/system/kube-controller-manager.service << EOF
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes

[Service]
ExecStart=/home/kubernetes/bin/kube-controller-manager --logtostderr=false \
--v=2 \
--log-dir=/www-volume/k8s-logs/kube-controller \
--log-file=/www-volume/k8s-logs/kube-controller/kube-controller.log \
--log-file-max-size=100 \
--leader-elect=true \
--kubeconfig=/home/kubernetes/config/kube-controller-manager.kubeconfig \
--bind-address=0.0.0.0 \
--pod-eviction-timeout=30s \
--feature-gates=RotateKubeletServerCertificate=true \
--node-startup-grace-period=30s \
--node-monitor-grace-period=20s \
--node-monitor-period=5s \
--allocate-node-cidrs=true \
--cluster-cidr=10.6.0.0/16 \
--leader-elect-resource-lock=endpoints \
--service-cluster-ip-range=10.5.0.0/16 \
--cluster-signing-cert-file=/home/kubernetes/ssl/ca.pem \
--cluster-signing-key-file=/home/kubernetes/ssl/ca-key.pem  \
--root-ca-file=/home/kubernetes/ssl/ca.pem \
--service-account-private-key-file=/home/kubernetes/ssl/ca-key.pem \
--cluster-signing-duration=87600h0m0s
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl start kube-controller-manager
systemctl enable kube-controller-manager

3.kube-scheduler

cd ~/TLS/k8s
cat > kube-scheduler-csr.json << EOF
{
  "CN": "system:kube-scheduler",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}
EOF
 
# 生成证书
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler
  
KUBE_CONFIG="/home/kubernetes/config/kube-scheduler.kubeconfig"
KUBE_APISERVER="https://ip01:6443"

kubectl config set-cluster kubernetes \
  --certificate-authority=/home/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-credentials kube-scheduler \
  --client-certificate=./kube-scheduler.pem \
  --client-key=./kube-scheduler-key.pem \
  --embed-certs=true \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-scheduler \
  --kubeconfig=${KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${KUBE_CONFIG}

cd ~/TLS/k8s
cat > /usr/lib/systemd/system/kube-scheduler.service << EOF
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes

[Service]
ExecStart=/home/kubernetes/bin/kube-scheduler --logtostderr=false \
--v=2 \
--log-dir=/www-volume/k8s-logs/kube-scheduler \
--log-file=/www-volume/k8s-logs/kube-scheduler/kube-scheduler.log \
--log-file-max-size=100 \
--leader-elect=true \
--leader-elect-resource-lock=endpoints \
--kubeconfig=/home/kubernetes/config/kube-scheduler.kubeconfig \
--bind-address=0.0.0.0
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl start kube-scheduler
systemctl enable kube-scheduler

4.制作admin用户身份认证

cat > admin-csr.json <<EOF
{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}
EOF
 
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
生成kubeconfig文件：
mkdir /root/.kube
 
KUBE_CONFIG="/root/.kube/config"
KUBE_APISERVER="https://ip01:6443"
 
kubectl config set-cluster kubernetes \
  --certificate-authority=/home/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-credentials cluster-admin \
  --client-certificate=./admin.pem \
  --client-key=./admin-key.pem \
  --embed-certs=true \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-context default \
  --cluster=kubernetes \
  --user=cluster-admin \
  --kubeconfig=${KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${KUBE_CONFIG}
  
kubectl get cs
  
kubectl create clusterrolebinding kubelet-admin --clusterrole=system:kubelet-api-admin --user=kubernetes
  
kubectl create clusterrolebinding kubelet-bootstrap \
--clusterrole=system:node-bootstrapper \
--user=kubelet-bootstrap

七.Node安装

1.kubelet

KUBE_CONFIG="/home/kubernetes/config/bootstrap.kubeconfig"
KUBE_APISERVER="https://ip01:6443" # apiserver IP:PORT
TOKEN="e537d04b49b03df79cf29c2b53bfcfae" # 与token.csv里保持一致
# 生成 kubelet bootstrap kubeconfig 配置文件
kubectl config set-cluster kubernetes \
  --certificate-authority=/home/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-credentials "kubelet-bootstrap" \
  --token=${TOKEN} \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-context default \
  --cluster=kubernetes \
  --user="kubelet-bootstrap" \
  --kubeconfig=${KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${KUBE_CONFIG}

cat > /usr/lib/systemd/system/kubelet.service << EOF
[Unit]
Description=Kubernetes Kubelet
After=docker.service

[Service]
ExecStart=/home/kubernetes/bin/kubelet --logtostderr=false \
--v=2 \
--log-dir=/www-volume/k8s-logs/kubelet \
--log-file=/www-volume/k8s-logs/kubelet/kubelet.log \
--log-file-max-size=100 \
--hostname-override=k8s-master-0001 \
--network-plugin=cni \
--kubeconfig=/home/kubernetes/config/kubelet.kubeconfig \
--config=/home/kubernetes/config/kubelet-config.yml \
--bootstrap-kubeconfig=/home/kubernetes/config/bootstrap.kubeconfig \
--cert-dir=/home/kubernetes/ssl \
--pod-infra-container-image=pause:3.1
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

--container-runtime=remote \
--container-runtime-endpoint=unix:///run/containerd/containerd.sock \
这两个配置是修改为containerd才需要的

cat > /home/kubernetes/config/kubelet-config.yml << EOF
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 0.0.0.0
port: 10250
readOnlyPort: 10255
cgroupDriver: systemd
clusterDNS:
  - 10.5.0.2
clusterDomain: cluster.local
failSwapOn: false
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 2m0s
    enabled: true
  x509:
    clientCAFile: /home/kubernetes/ssl/ca.pem
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 5m0s
    cacheUnauthorizedTTL: 30s
evictionHard:
  imagefs.available: 15%
  memory.available: 8%
  nodefs.available: 10%
  nodefs.inodesFree: 5%
maxOpenFiles: 1000000
maxPods: 55
systemReserved:
  memory: 1G
kubeReserved:
  memory: 1G
podPidsLimit: 5000
staticPodPath: /etc/kubernetes/manifests/
EOF

systemctl daemon-reload
systemctl start kubelet
systemctl enable kubelet
kubectl get csr |  grep  Pending | awk '{print $1}' | xargs  kubectl certificate approve

2.kube-proxy

cd ~/TLS/k8s
 
# 创建证书请求文件
cat > kube-proxy-csr.json << EOF
{
  "CN": "system:kube-proxy",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF
 
# 生成证书
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
  
KUBE_CONFIG="/home/kubernetes/config/kube-proxy.kubeconfig"
KUBE_APISERVER="https://ip01:6443"

kubectl config set-cluster kubernetes \
  --certificate-authority=/home/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-credentials kube-proxy \
  --client-certificate=./kube-proxy.pem \
  --client-key=./kube-proxy-key.pem \
  --embed-certs=true \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-proxy \
  --kubeconfig=${KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${KUBE_CONFIG}

cat > /usr/lib/systemd/system/kube-proxy.service << EOF
[Unit]
Description=Kubernetes Proxy
After=network.target

[Service]
ExecStart=/home/kubernetes/bin/kube-proxy --logtostderr=false \
--v=2 \
--log-file=/www-volume/k8s-logs/kube-proxy/kube-proxy.log \
--log-file-max-size=100 \
--log-dir=/www-volume/k8s-logs/kube-proxy \
--config=/home/kubernetes/config/kube-proxy-config.yml
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF


cat >  /home/kubernetes/config/kube-proxy-config.yml << EOF
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
clientConnection:
  acceptContentTypes: ""
  burst: 10
  kubeconfig: /home/kubernetes/config/kube-proxy.kubeconfig
  qps: 5
clusterCIDR: 10.6.0.0/16
configSyncPeriod: 15m0s
conntrack:
  max: null
  maxPerCore: 32768
  min: 131072
  tcpCloseWaitTimeout: 1h0m0s
  tcpEstablishedTimeout: 24h0m0s
enableProfiling: false
healthzBindAddress: 0.0.0.0:10256
iptables:
  masqueradeAll: false
  masqueradeBit: 14
  minSyncPeriod: 0s
  syncPeriod: 30s
ipvs:
  excludeCIDRs: null
  minSyncPeriod: 0s
  scheduler: "rr"
  strictARP: false
  syncPeriod: 30s
kind: KubeProxyConfiguration
metricsBindAddress: 0.0.0.0:10249
mode: "ipvs"
nodePortAddresses: null
oomScoreAdj: -999
portRange: ""
resourceContainer: /kube-proxy
udpIdleTimeout: 250ms
winkernel:
  enableDSR: false
  networkName: ""
  sourceVip: ""
hostnameOverride: k8s-master-0001
EOF

systemctl daemon-reload
systemctl start kube-proxy
systemctl enable kube-proxy