[原创] k8s 部署文档
k8s note, support all cni plugin.master firstinstall dockercurl -fsSL https://get.docker.com -o get-docker.shbash get-docker.shsystemctl enable docker && systemctl start dockerdocker ps...
k8s note, support all cni plugin.
master first
install docker
curl -fsSL https://get.docker.com -o get-docker.sh
bash get-docker.sh
systemctl enable docker && systemctl start docker
docker ps
cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
},
"storage-driver": "overlay2"
}
EOF
# apt-get install -y libseccomp2 # if you nead
mkdir -p /etc/systemd/system/docker.service.d
systemctl daemon-reload
systemctl enable docker && systemctl restart docker
install CRI-O
# k8s new version has create CRI sockets, maybe you dont nead this
modprobe overlay
modprobe br_netfilter
cat > /etc/sysctl.d/99-kubernetes-cri.conf <<EOF
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF
sysctl --system
apt-get update
apt-get install software-properties-common
add-apt-repository ppa:projectatomic/ppa
apt-get update
apt-get install cri-o-1.11 -y
systemctl enable crio && systemctl start crio
install haproxy
# if you just one master node, ignore this
add-apt-repository ppa:vbernat/haproxy-1.9
apt update
apt install haproxy
# edit haproxy.cfg
cat >> /etc/haproxy/haproxy.cfg <<EOF
listen admin_stats
bind 0.0.0.0:10080
mode http
log 127.0.0.1 local0 err
stats refresh 30s
stats uri /status
stats realm welcome login\ Haproxy
stats auth admin_name:admin_passwd
stats hide-version
stats admin if TRUE
listen kube-master
bind 0.0.0.0:8443
mode tcp
option tcplog
balance source
server MASTER-01 MASTER-02-IP:6443 check inter 2000 fall 2 rise 2 weight 1
server MASTER-02 MASTER-02-IP:6443 check inter 2000 fall 2 rise 2 weight 1
EOF
systemctl enable haproxy && systemctl restart haproxy
install k8s and kubeadm
# install k8s
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list
apt update -y
apt-get install -y kubelet kubeadm kubectl
apt-mark hold kubelet kubeadm kubectl
cat > /etc/sysctl.d/99-k8s <<EOF
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF
systemctl daemon-reload
systemctl restart kubelet
systemctl enable kubelet && systemctl start kubelet
systemctl enable docker && systemctl start docker
install docker-compose
# apt install -y py-pip python-dev libffi-dev openssl-dev gcc libc-dev make
curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose
docker-compose --version
install kompose
covert docker-compose to k8s format, if you need
curl -L https://github.com/kubernetes/kompose/releases/download/v1.18.0/kompose-linux-amd64 -o kompose
chmod +x kompose
mv ./kompose /usr/local/bin/kompose
config and init k8s
cat >> kubeadm-config.yaml <<EOF
apiServer:
timeoutForControlPlane: 4m0s
extraArgs:
enable-admission-plugins: NamespaceLifecycle,LimitRanger,ServiceAccount,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,PersistentVolumeClaimResize,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota
runtime-config: "api/all=true"
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
kubernetesVersion: stable
controlPlaneEndpoint: "HAPROXY_IP:8443"
#controlPlaneEndpoint: "LOCAL_SUB_IP:6443"
dns:
type: "CoreDNS"
networking:
dnsDomain: cluster.local
podSubnet: "10.244.0.0/16"
EOF
# if not install or config haproxy server, controlPlaneEndpoint set local subnet ip, like this `192.168.8.181:6443`. don't change port 6443 if you not custom endpoint port
kubeadm reset --force
kubeadm init --config=kubeadm-config.yaml --experimental-upload-certs
# kubeadm init --config=kubeadm-config.yaml --experimental-upload-certs --ignore-preflight-errors=all
## here will print join k8s command for master(control-plane) node and worker node.
## copy command for master node, like this
## kubeadm join MASTER-01:8443 --token mepvx0.7db40f5sgr33af0s --discovery-token-ca-cert-hash sha256:0ba3d8aea484c03acb29c867a36e0d9ecbb281406db665fe47e0abd931f48718 --experimental-control-plane --certificate-key aba1fb9973c7122ed4bfb2c8dbf946494cc26dbca01444a58ac1b2c7b1384e84
## copy command for worker node, like this
## kubeadm join MASTER-01:8443 --token mepvx0.7db40f5sgr33af0s --discovery-token-ca-cert-hash sha256:0ba3d8aea484c03acb29c867a36e0d9ecbb281406db665fe47e0abd931f48718
# save kube admin config
mkdir -p $HOME/.kube
cp /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config
kubectl get nodes
# NAME STATUS ROLES AGE VERSION
# master-01 NotReady master 152m v1.14.1
docker registry server
# this is test, so run it on master
docker run -d --name registry --publish published=5000,target=5000 registry:2
init cni plugin
Not support bandwidth for now.
# use calico cni
kubectl apply -f https://docs.projectcalico.org/v3.7/manifests/canal.yaml
# 2019-06-24 v3.8 can use bandwidth plugin
kubectl apply -f https://docs.projectcalico.org/v3.8/manifests/canal.yaml
# use standard flannel
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/v0.11.0/Documentation/kube-flannel.yml
Support bandwidth
# clone project
git clone https://github.com/Ox0400/cni-plugin
cd cni-plugin
# build and push images
REGISTRY="YOUREGISTRYHOSTIP:5000"
sed -i -e "s?REGISTRYHOST?$REGISTRY?g" start.sh
bash start.sh
# apply cni plugin
sed -i -e "s?REGISTRY?$REGISTRY?g" example-cni-calico.yml
kubelet apply -f example-cni-calico.yml
2019-06-24 补充: v3.8 已经发布, 内置了 bandwidth https://docs.projectcalico.org/v3.8/getting-started/kubernetes/installation/flannel#installing-with-the-kubernetes-api-datastore-recommended
test apply app
# myweb.yaml
# ------------------- APP Deployment ------------------- #
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
app: myweb
name: myweb
spec:
replicas: 1
revisionHistoryLimit: 1
selector:
matchLabels:
app: myweb
template:
metadata:
labels:
app: myweb
#annotations:
# ingress-bandwidth and egress-bandwidth is difference with the test results. 10Mi just is 1kb. unit is bytes always? 10MB maybe set value is 10000Mi or 10Gi?
# kubernetes.io/ingress-bandwidth: 10Mi
# kubernetes.io/egress-bandwidth: 10Mi
spec:
hostAliases:
- ip: "127.0.0.1"
hostnames:
- "redis"
dnsConfig:
nameservers:
- 8.8.8.8
- 8.8.4.4
containers:
- name: api
image: nginx
imagePullPolicy: Always
ports:
- containerPort: 80
protocol: TCP
resources:
limits:
cpu: 0.5
memory: "256Mi"
ephemeral-storage: "100Mi"
requests:
cpu: 0.25
memory: "64Mi"
ephemeral-storage: "100Mi"
volumeMounts:
- mountPath: /etc/localtime
name: localtime
- name: redis
image: redis:alpine
ports:
- containerPort: 6379
protocol: TCP
resources:
limits:
cpu: 0.25
memory: "64Mi"
ephemeral-storage: "100Mi"
requests:
cpu: 0.20
memory: "32Mi"
ephemeral-storage: "100Mi"
volumeMounts:
- mountPath: /etc/localtime
name: localtime
volumes:
- name: localtime
hostPath:
path: /etc/localtime
type: FileOrCreate
---
# ------------------- APP Service ------------------- #
kind: Service
apiVersion: v1
metadata:
labels:
app: myweb
name: myweb
spec:
#type: ClusterIP
type: NodePort
ports:
- port: 80
targetPort: 80
selector:
app: myweb
# deploy app
kubectl apply -f myweb.yaml
# show app info
app=myweb
kubectl get pods,svc,deployments -o wide -l app=$app
# test connect nginx use HEAD
curl -I "`kubectl get pods -o wide -l app=$app | grep Running | awk '{print $7}'`:`kubectl get svc -o wide -l app=$app | grep NodePort | grep -P '\d\d\d\d+' -o`/"
# Spawns a shell within the container
kubectl exec `kubectl get pods -o wide -l app=$app | grep Running | awk '{print $1}'` -c api -it sh
## test storage limit
while true; do dd if=/dev/zero of=$(date '+%s').out count=1 bs=20MB 2> /dev/null; du -sh . ;sleep 1; done
## test bandwidth limit
pip install speedtest-cli && speedtest-cli
wget --output-document=/dev/null http://speedtest.wdc01.softlayer.com/downloads/test500.zip
master second
install docker
see master first
install cri
see master first
install haproxy
see master first
install k8s
see master first
install kompose
see master first
join k8s master node
# join master nodes use the master first init output command
kubeadm join MASTER-01-token mepvx0.7db40f5sgr33af0s --discovery-token-ca-cert-hash sha256:0ba3d8aea484c03acb29c867a36e0d9ecbb281406db665fe47e0abd931f48718 --experimental-control-plane --certificate-key aba1fb9973c7122ed4bfb2c8dbf946494cc26dbca01444a58ac1b2c7b1384e84
# save kube admin config
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config
worker
install docker
see master first
install cri
see master first
install k8s
see master first
join k8s worker node
kubeadm join 172.31.42.98:8443 --token mepvx0.7db40f5sgr33af0s --discovery-token-ca-cert-hash sha256:0ba3d8aea484c03acb29c867a36e0d9ecbb281406db665fe47e0abd931f48718
docker ps
QA
WARNING: Ignoring APKINDEX.xxxx.tar.gz: No such file or directory
set dns for docker daemon. /etc/docker/daemon.json
{
"dns": [own_dns,"8.8.8.8","8.8.8.4"]
}
Pod status: ImagePullBackOff
set insecure-registries for docker daemon
# show log
kubectl describe pods POD_NAME
{
"insecure-registries":["YOUREGISHOST:5000"]
}
kompose convert
mkdir k8s && mv *yaml k8s
kubectl apply -f k8s/
kubectl delete -f k8s/
kubectl get services gr1-api
Container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
- remove cni: /var/lib/kubelet/kubeadm-flags.env
- install cni: https://docs.docker.com/ee/ucp/kubernetes/install-cni-plugin/
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
NetworkPlugin cni failed to set up pod “v1-58cb8ccd5c-w2wlm_default” network: failed to set bridge addr: “cni0” already has an IP address different from 10.244.3.1/24"
## delete cni* interface
ifconfig cni0 down
ip link delete cni0
https://github.com/kubernetes/kubernetes/issues/39557#issuecomment-271944481
https://github.com/kubernetes/kubernetes/issues/39557#issuecomment-342604193
E0517 08:27:19.936024 1 main.go:289] Error registering network: failed to acquire lease: node “node-01” pod cidr not assigned
# edit /etc/kubernetes/manifests/kube-controller-manager.yaml
ifconfig cni0 down
ip link delete cni0
- --allocate-node-cidrs=true
- --cluster-cidr=10.244.0.0/16
if use kubeadm, please set podSubnet: “10.244.0.0/16”
https://github.com/coreos/flannel/blob/master/Documentation/kubernetes.md
kubelet version 1.9, yaml config selector
area must set matchLabels
Pod can’t connect internet
cat > kube-dns.yml <<EOF
apiVersion: v1
kind: ConfigMap
metadata:
name: kube-dns
namespace: kube-system
data:
upstreamNameservers: |
["8.8.8.8", "8.8.4.4"]
EOF
kubectl apply -f kube-dns.yml
kubectl get pod -n kube-system -o wide -l k8s-app=kube-dns
kubectl delete pod -n kube-system -l k8s-app=kube-dns
https://medium.com/@boruah.rajen/dns-resolution-works-in-host-but-not-from-kubernetes-pod-226dcbeccc13
error: unable to recognize “lxcfs-initializer.yaml”: no matches for kind “InitializerConfiguration” in version “admissionregistration.k8s.io/v1alpha1”
https://github.com/kubernetes/community/blob/b3349d5b1354df814b67bbdee6890477f3c250cb/contributors/design-proposals/api-machinery/admission-control-webhooks.md
https://zhuanlan.zhihu.com/p/27700061
https://docs.bitnami.com/kubernetes/how-to/configure-autoscaling-custom-metrics/#cluster-configuration
https://github.com/kubernetes/website/pull/8852
https://github.com/kubernetes/kubernetes/pull/58428
list all apiversion
kubectl api-versions
remove ip link datapath
curl -L git.io/weave -o /usr/local/bin/weave
chmod a+x /usr/local/bin/weave
systemctl restart kubelet && systemctl restart docker
/usr/local/bin/weave reset
ifconfig -a
set nameserver for pod
dnsConfig:
nameservers:
- 8.8.8.8
- 8.8.4.4
https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/
set hosts for pod
hostAliases:
- ip: "127.0.0.1"
hostnames:
- "redis"
limit bandwidth
see cni plugin
reset k8s cluters
## RESET k8s
kubeadm reset --force
systemctl stop kubelet && systemctl stop docker
iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X
# reset lxcfs
fusermount -u /var/lib/lxcfs
rm -rf /var/lib/lxcfs
# delete kubelet config
rm -rf /var/lib/kubelet/*
# reset cni
rm -rf /var/lib/cni/
rm -rf /etc/cni/
rm -rf /opt/cni
# reset interfaces
modprobe -r ipip
# system default interface is eth0 and lo, keep both.
ifconfig -a | grep "Link " |grep -e eth0 -e lo -v| awk '{print "ifconfig " $1 " down; ip link delete "$1}' | bash
# reset services
systemctl start kubelet && systemctl start docker
ifconfig -a
更多推荐
所有评论(0)