kubernetes配置案例
·
内存 硬盘 cpu
master 10G 100G 4
node 8G 100G 4
k8s
使用kubeeasy
搭建k8s
使用 kubeeasy 工具 完成 Kubernetes 1.22.1 集群的搭建
- 设置主机名master、node
- 关闭防火墙selinux
- 配置hosts文件
- 配置免密
master挂载镜像chinaskills_cloud_paas_v2.0.2.iso
mount /dev/sr0 /mnt -o loop
cp -rfv /mnt/* /opt/ # 替换base-rpms.tar.gz到/mnt/dependencies/里面
umount
master将kubeeasy复制到/usr/bin，安装kubeeasy
cp -rf /opt/kubeeasy /usr/bin/
kubeeasy install dependencies --host 192.168.100.10,192.168.100.11 --user root --password 000000 --offline-file /opt/dependencies/base-rpms.tar.gz
kubeeasy install kubernetes --master 192.168.100.10 --worker 192.168.100.11 --user root --password 000000 --version 1.22.1 --offline-file /opt/kubernetes.tar.gz
# 日志查看
tail -f /var/log/messages & kubeinstall.log
创建pod
然后使用 nginx 镜像在 default 命名空间下创建一个名为 exam 的 Pod，并为该 Pod 设置环境变量 exam，其值为 2022
kubectl run exam --image nginx:latest --env="exam=2022" --image-pull-policy='IfNotPresent'
部署 Istio 服务网格
在 Kubernetes 集群上完成 Istio 服务网格环境的安装，然后新建命名空间 exam，为该命 名空间开启自动注入 Sidecar
#网络安装
kubeeasy add --istio istion
#创建namespace
kubectl create namespace exam
#自动注入sidecar
kubectl label ns exam istio-injection=enabled
部署 KubeVirt 虚拟化
在 Kubernetes 集群上完成 KubeVirt 虚拟化环境的安装
kubeeasy add --virt kubevirt
部署 Harbor 仓库
安装部署
在 master 节点上完成 Harbor 镜像仓库及 Helm 包管理工具的部署。
kubeeasy add --registry harbor
使用deployment
然后使用 nginx 镜像 自定义一个 Chart，Deployment 名称为 nginx，副本数为 1，然后将该 Chart 部署到 default 命名空间下，Release 名称为 web
# 需要在工作目录有nginx.tar.gz
helm create nginx # 使用helm工具创建nginx项目
cd nginx
rm -rf templates/*
#创建deployment模板
kubectl create deployment nginx -o yaml --image=nginx:latest --replicas=1 --dry-run=client --port=80 > /root/nginx/templates/deployment.yaml
# 在image name后面添加查找镜像
vi /templates/deployment.yaml
imagePullPolicy: IfNotPresent
vi templates/NOTES.txt
name:nginx
cd ..
helm install web nginx
卸载 heml uninstall web
备份 ETCD 数据
Kubernetes 使用 ETCD 来存储集群的实时运行数据，为防止服务器宕机导致 Kubernetes 集群数据丢失，请将 Kubernetes 集群数据备份到/root/etcd.db 中
[root@k8s-master-node1 ~]# docker ps -a | grep etcd
9a8e4e589662 004811815584 "etcd --advertise-cl…" 47 minutes ago Up 47 minutes k8s_etcd_etcd-k8s-master-node1_kube-system_359e897a01149111969d3f5d475f8e96_0
39ae1627794c k8s.gcr.io/pause:3.5 "/pause" 47 minutes ago Up 47 minutes k8s_POD_etcd-k8s-master-node1_kube-system_359e897a01149111969d3f5d475f8e96_0
[root@k8s-master-node1 ~]# docker cp 9a8e4e589662:/usr/local/bin/etcdctl /usr/local/bin/etcdctl
CICD
安装 GitLab 环境
在 Kubernetes 集群中新建命名空间 gitlab-ci，将 GitLab 部署到该命名空间下，Deployment 和 Service 名称均为 gitlab，以 NodePort 方式将 80 端口对外暴露为 30880，设置 GitLab 服务 root 用户的密码为 admin@123，将项目包 demo-2048.tar.gz 导入到 GitLab 中并命名为 demo-2048。
apiVersion: apps/v1
kind: Deployment
metadata:
name: gitlab
namespace: gitlab-ci
labels:
name: gitlab
spec:
selector:
matchLabels:
name: gitlab
template:
metadata:
name: gitlab
labels:
name: gitlab
spec:
containers:
- image: gitlab/gitlab-ce:latest
name: gitlab
imagePullPolicy: IfNotPresent
env:
- name: GITLAB_ROOT_PASSWORD
value: admin@123
- name: GITLAB_HOST
value: 192.168.150.100
- name: GITLAB_PORT
value: "80"
ports:
- name: http
containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
name: gitlab
namespace: gitlab-ci
labels:
name: gitlab
spec:
type: NodePort
ports:
- name: http
port: 80
targetPort: http
nodePort: 30880
selector:
name: gitlab
安装jenkins
apiVersion: apps/v1
kind: Deployment
metadata:
name: jenkins
namespace: jenkins
labels:
name: jenkins
spec:
selector:
matchLabels:
name: jenkins
template:
metadata:
name: jenkins
labels:
name: jenkins
spec:
containers:
- name: jenkins
image: jenkins/jenkins:2.262-centos
ports:
- name: http
containerPort: 8080
- name: other
containerPort: 50000
---
apiVersion: v1
kind: Service
metadata:
name: jenkins
namespace: jenkins
labels:
name: jenkins
spec:
type: NodePort
ports:
- name: http
nodePort: 8080
targetPort: http
port: 8080
selector:
name: jenkins
容器云服务运维
时间同步
apiVersion: v1
kind: Pod
metadata:
name: time
spec:
containers:
- name: time
image: 192.168.59.22/library/nginx:latest
imagePullPolicy: IfNotPresent
volumeMounts:
- name: timefile
mountPath: /etc/localtime
volumes:
- name: timefile
hostPath:
path: /etc/localtime
secret
[root@master ~]# echo -n 'admin' | base64
YWRtaW4=
[root@master ~]# echo -n '1f2d1e2e67df' | base64
MWYyZDFlMmU2N2Rm
apiVersion: v1
kind: Secret
metadata:
name: secret
type: Opaque
data:
username: YWRtaW4=
password: MWYyZDFlMmU2N2Rm
资源限制
在 default 命名空间下使用 nginx:latest 镜像创建一个 QoS 类为 Guaranteed 的 Pod，名称 为 qos-demo。
kubectl create -f qos-demo.yaml
apiVersion: v1
kind: Pod
metadata:
labels:
run: pod
name: qos-demo
spec:
containers:
- image: nginx:latest
imagePullPolicy: IfNotPresent
name: pod
ports:
- containerPort: 80
resources:
limits:
cpu: "100m"
memory: "100Mi"
requests:
cpu: "100m"
memory: "100Mi"
安全上下文
使用 busybox 镜像启动一个名为 context-demo 的 Pod，为该 Pod 配置安全上下文，要求 容器内以用户 1000 和用户组 3000 来运行所有进程，并在启动时执行“sleep 1h”命令。
kubectl apply -f context-demo.yaml
apiVersion: v1
kind: Pod
metadata:
labels:
run: context-demo
name: context-demo
spec:
securityContext:
runAsUser: 1000
runAsGroup: 3000
containers:
- image: busybox:latest
name: context-demo
imagePullPolicy: IfNotPresent
command: [ "sh", "-c", "sleep 1h" ]
securityContext:
allowPrivilegeEscalation: false
nodeSelector:
kubernetes.io/hostname: k8s-master-node1
创建自定义资源类型
在 Kubernetes 集群中自定义一种资源类型 Student，API 为 stable.example.com/v1，单数 形式为 student，复数形式为 students，简写为 stu，作用域为命名空间级，然后在 default 命 名空间下创建一个名为 exam 的 Student 对象。
kubectl apply -f crd.yaml
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: students.stable.example.com
spec:
group: stable.example.com
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
properties:
spec:
type: object
tags:
type: string
scope: Namespaced
names:
plural: students
kind: Student
shortNames:
- stu
# kubectl get crd
NAME CREATED AT
students.stable.example.com 2023-02-27T01:47:08Z
kubectl apply -f stu.yaml
apiVersion: stable.example.com/v1
kind: Student
metadata:
name: exam
spec:
tags: test
添加主机别名到 Pod
使用 nginx 镜像在 default 命名空间下创建一个名为 nginx 的 Pod，并在 Pod 的/etc/hosts 中添加 IP 地址 127.0.0.1 与 chinaskills 的解析。
kubectl apply -f pod-host.yaml
apiVersion: v1
kind: Pod
metadata:
labels:
run: nginx
name: nginx
spec:
containers:
- image: nginx:latest
imagePullPolicy: IfNotPresent
name: nginx
ports:
- containerPort: 80
hostAliases:
- ip: 127.0.0.1
hostnames:
- chinaskills
kubectl exec -it nginx -- cat /etc/hosts
创建 HPA 规则
默认情况下 HPA 是无法调整伸缩灵敏度的，但不同的业务场景对伸缩灵敏度的要求不 一样。要求在 default 命名空间下使用 nginx 镜像创建一个名为 web 的 deployment，自定义 HPA 的伸缩灵敏度，为该 deployment 创建一个名为 web 的 HPA，扩容时立即新增当前 9 倍 数量的副本数，时间窗口为 5s，伸缩范围为 1–1000。例如一开始只有 1 个 Pod，当 CPU 使 用率超过 80%时，Pod 数量变化趋势为：1 → 10 → 100 → 1000
kubectl apply -f web-hpa.yaml
kubectl apply -f hpa.yaml
apiVersion: autoscaling/v2beta2
kind: HorizontalPodAutoscaler
metadata:
name: web
spec:
minReplicas: 1
maxReplicas: 1000
metrics:
- type: Resource
resource:
name: cpu
target :
type: Utilization
averageUtilization: 80
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: web
behavior:
scaleUp:
stabilizationWindowSeconds: 5
policies:
- type: Percent
value: 900
periodSeconds: 5
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: web
name: web
spec:
replicas: 1
selector:
matchLabels:
app: web
template:
metadata:
labels:
app: web
spec:
containers:
- image: nginx:latest
imagePullPolicy: IfNotPresent
name: nginx
ports:
- containerPort: 80
resources:
limits:
cpu: "10m"
memory: "20Mi"
requests:
cpu: "10m"
memory: "20Mi"
[root@k8s-master-node1 4]# kubectl describe hpa
Name: web
Namespace: default
Labels: <none>
Annotations: <none>
CreationTimestamp: Mon, 27 Feb 2023 10:07:25 +0800
Reference: Deployment/web
Metrics: ( current / target )
resource cpu on pods (as a percentage of request): <unknown> / 80%
Min replicas: 1
Max replicas: 1000
Behavior:
Scale Up:
Stabilization Window: 5 seconds
Select Policy: Max
Policies:
- Type: Percent Value: 900 Period: 5 seconds
Scale Down:
Select Policy: Max
Policies:
- Type: Percent Value: 100 Period: 15 seconds
Deployment pods: 0 current / 0 desired
Conditions:
Type Status Reason Message
---- ------ ------ -------
AbleToScale False FailedGetScale the HPA controller was unable to get the target's current scale: no matches for kind "Deployment" in group "app"
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Warning FailedGetScale 10s (x9 over 2m10s) horizontal-pod-autoscaler no matches for kind "Deployment" in group "app"
标签Pod
在 default 命名空间下使用 nginx 镜像运行一个 Pod，名称为 nginx，要求使用 requiredDuringSchedulingIgnoredDuringExecution 策略将 Pod 调度到具有“disktype=ssd”标 签的节点。
kubectl label node k8s-master-node1 disktype=ssd
apiVersion: v1
kind: Pod
metadata:
labels:
run: nginx
name: nginx
spec:
containers:
- image: nginx:latest
imagePullPolicy: IfNotPresent
name: nginx
ports:
- containerPort: 80
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: disktype
operator: In
values:
- ssd
[root@k8s-master-node1 4]# kubectl describe nodes k8s-master-node1 | grep ssd
disktype=ssd
创建 Pod 网络策略
创建一个网络策略 network-exam，要求只有 internal 命名空间下的 Pod 可以通过 TCP 协议的 8080 端口访问到 mysql 命名空间下的 Pod。
kubectl create ns mysql
kubectl create ns internal
kubectl apply -f network-exam.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: network-exam
namespace: internal
spec:
podSelector: {}
policyTypes:
- Ingress
ingress:
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: internal
ports:
- port: 8080
protocol: TCP
驱逐机制管理
设置 kubelet数据存储在/apps/data/kubelet目录下，并设置当 kubelet的存储空间不足 5%， 或者当容器运行时文件系统可用存储空间不足 5%时开始驱逐 Pod
vi /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS=--root-dir=/apps/data/kubelet --eviction-hard
=nodefs.available<5% --eviction-hard=imagefs.available<5%
[root@k8s-master-node1 ~]# systemctl daemon-reload && systemctl restart kubelet
创建 Ingress Gateway
使用提供的软件包 ServiceMesh.tar.gz 将 Bookinfo 应用部署到 default 命名空间下，使用 Istio Gateway 可 以实 现应 用程 序从 外部 访问， 请为 Bookinfo 应用创 建一 个名 为 bookinfo-gateway 的网关，指定所有 HTTP 流量通过 80 端口流入网格，然后将网关绑定到虚 拟服务 bookinfo 上
[root@k8s-master-node1 ~]# cd ServiceMesh/images/
[root@k8s-master-node1 images]# scp image.tar 192.168.59.23:/root/
[root@k8s-master-node1 images]# docker load -i image.tar
[root@k8s-worker-node1 ~]# docker load -i image.tar
[root@k8s-master-node1 ~]# kubectl label ns default istio-injection=enable
[root@k8s-master-node1 ~]# cd ServiceMesh/bookinfo/
[root@k8s-master-node1 bookinfo]# kubectl apply -f bookinfo.yaml
[root@k8s-master-node1 bookinfo]# kubectl apply -f ratings-v2.yaml
[root@k8s-master-node1 bookinfo]# kubectl apply -f reviews-v2.yaml
[root@k8s-master-node1 bookinfo]# kubectl apply -f reviews-v3.yaml
[root@k8s-master-node1 4]# kubectl apply -f bookinfo.yaml
apiVersion: networking.istio.io/v1beta1
kind: Gateway
metadata:
name: bookinfo-gateway
spec:
selector:
istio: ingressgateway
servers:
- port:
number: 80
name: http
protocol: HTTP
hosts:
- "*"
---
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
name: bookinfo
spec:
hosts:
- "*"
gateways:
- bookinfo-gateway
http:
- match:
- uri:
exact: /productpage
- uri:
prefix: /static
- uri:
exact: /login
- uri:
exact: /logout
- uri:
prefix: /api/v1/products
route:
- destination:
host: productpage
port:
number: 9080
创建基于用户身份的路由
创建一个名为 reviews 路由，要求来自名为 Jason 的用户的所有流量将被路由到服务 reviews:v2。
[root@k8s-master-node1 4]# kubectl apply -f review.yaml
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
name: reviews
spec:
host: reviews
subsets:
- name: v1
labels:
version: v1
- name: v2
labels:
version: v2
- name: v3
labels:
version: v3
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
name: reviews
spec:
hosts:
- reviews
http:
- match:
- headers:
end-user:
exact: jason
route:
- destination:
host: reviews
subset: v2
- route:
基于虚拟机的运维
创建 VM
使用镜像 fedora-virt:v1.0 在 default 命名空间下创建一台 vm，名称为 vm-fedora，内存为 1G。
[root@k8s-master-node1 ~]# docker load -i fedora-virt_v1.0.tar
[root@k8s-master-node1 5]# kubectl apply -f fedora-vm.yaml
apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
labels:
kubevirt.io/vm: vm-fedora
name: vm-fedora
spec:
runStrategy: Always
template:
metadata:
labels:
kubevirt.io/vm: vm-fedora
spec:
domain:
resources:
requests:
memory: 1Gi
devices:
disks:
- name: containerdisk
disk:
bus: virtio
volumes:
- name: containerdisk
containerDisk:
image: fedora-virt:v1.0
创建 emptyDisk 卷
使用镜像 fedora-virt:v1.0 在 default 命名空间下创建一台 vmi，名称为 vmi-fedora，并使 用 emptyDisk 卷为 vmi 挂载一块 2G 的磁盘
[root@k8s-master-node1 5]# kubectl apply -f vim-fedora.yaml
apiVersion: kubevirt.io/v1
kind: VirtualMachineInstance
metadata:
labels:
special: vmi-fedora
name: vmi-fedora
spec:
domain:
devices:
disks:
- disk:
bus: virtio
name: containerdisk
- name: emptydisk
disk :
bus: virtio
resources:
requests:
memory: 1Gi
terminationGracePeriodSeconds: 0
volumes:
- containerDisk:
image: fedora-virt:v1.0
name: containerdisk
- name: emptydisk
emptyDisk:
capacity: 2Gi
创建 VMI
将提供的镜像 exam.qcow2 转换为 docker 镜像 exam:v1.0，然后使用镜像 exam:v1.0 镜像 在 default 命名空间下创建一台 vmi，名称为 exam，将虚拟机的 80 端口以 NodePort 的方式 对外暴露为 30082，并使用数据源在启动时将 VM 的主机名初始化为 exam
docker build -t exam:v1.0 .
FROM scratch
ADD exam.qcow2 /disk/
kubectl apply -f exam.yaml
apiVersion: kubevirt.io/v1
kind: VirtualMachineInstance
metadata:
name: exam
labels:
app: exam
spec:
domain:
devices:
disks:
- name: containerdisk
disk:
bus: virtio
- name: cloudinitdisk
disk:
bus: virtio
resources:
requests:
memory: 512Mi
volumes:
- name: containerdisk
containerDisk:
image: exam:v1.0
imagePullPolicy: IfNotPresent
- name: cloudinitdisk
cloudInitNoCloud:
userData: |-
hostname: exam
---
apiVersion: v1
kind: Service
metadata:
labels:
app: exam
name: exam
spec:
ports:
- name: 80-80
nodePort: 30082
port: 80
protocol: TCP
targetPort: 80
selector:
app: exam
type: NodePort
更多推荐
7
0- 0
已为社区贡献4条内容
所有评论(0)