k8s部署mysql，ceph作为存储
·
服务器资源清单
服务器：OpenEuler22.04 ceph版本：ceph version 16.2.7 k8s版本：Client Version: v1.32.2
1、k8s服务器操作：
创建k8s与ceph的驱动
wget https://github.com/ceph/ceph-csi/archive/refs/tags/v3.15.0.zip
tar xf ceph-csi-3.15.0.tar.gz -C /usr/local/
/usr/local/ceph-csi-3.15.0/deploy/rbd/kubernetes
# 修改csi-config-map.yaml文件
# clusterID：通过ceph服务器，执行ceph -s查询
root@PS2022CVIYBBVU:/usr/local/ceph-csi-3.15.0/deploy/rbd/kubernetes# cat csi-config-map.yaml
#
# /!\ DO NOT MODIFY THIS FILE
#
# This file has been automatically generated by Ceph-CSI yamlgen.
# The source for the contents can be found in the api/deploy directory, make
# your modifications there.
#
---
apiVersion: v1
kind: ConfigMap
metadata:
name: "ceph-csi-config"
data:
config.json: |-
[
{
"clusterID": "65eac714-0bd5-11f1-9e23-000c298c1628",
"monitors": [
"192.168.198.22:6789","192.168.198.41:6789","192.168.198.43:6789"
]
}
]
修改csi-rbdplugin-provisioner.yaml
root@PS2022CVIYBBVU:/usr/local/ceph-csi-3.15.0/deploy/rbd/kubernetes# cat csi-rbdplugin-provisioner.yaml
---
kind: Service
apiVersion: v1
metadata:
name: csi-rbdplugin-provisioner
# replace with non-default namespace name
namespace: default
labels:
app: csi-metrics
spec:
selector:
app: csi-rbdplugin-provisioner
ports:
- name: http-metrics
port: 8080
protocol: TCP
targetPort: 8680
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: csi-rbdplugin-provisioner
# replace with non-default namespace name
namespace: default
spec:
replicas: 3
selector:
matchLabels:
app: csi-rbdplugin-provisioner
template:
metadata:
labels:
app: csi-rbdplugin-provisioner
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- csi-rbdplugin-provisioner
topologyKey: "kubernetes.io/hostname"
serviceAccountName: rbd-csi-provisioner
priorityClassName: system-cluster-critical
containers:
- name: csi-rbdplugin
image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/quay.io/cephcsi/cephcsi:v3.14.0
args:
- "--nodeid=$(NODE_ID)"
- "--type=rbd"
- "--controllerserver=true"
- "--endpoint=$(CSI_ENDPOINT)"
- "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)"
- "--v=5"
- "--drivername=rbd.csi.ceph.com"
- "--pidlimit=-1"
- "--rbdhardmaxclonedepth=8"
- "--rbdsoftmaxclonedepth=4"
- "--enableprofiling=false"
- "--setmetadata=true"
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
# - name: KMS_CONFIGMAP_NAME
# value: encryptionConfig
- name: CSI_ENDPOINT
value: unix:///csi/csi-provisioner.sock
- name: CSI_ADDONS_ENDPOINT
value: unix:///csi/csi-addons.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /csi
- mountPath: /dev
name: host-dev
- mountPath: /sys
name: host-sys
- mountPath: /lib/modules
name: lib-modules
readOnly: true
- name: ceph-csi-config
mountPath: /etc/ceph-csi-config/
# - name: ceph-csi-encryption-kms-config
# mountPath: /etc/ceph-csi-encryption-kms-config/
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
# - name: ceph-config
# mountPath: /etc/ceph/
- name: oidc-token
mountPath: /run/secrets/tokens
readOnly: true
- name: csi-provisioner
image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/sig-storage/csi-provisioner:v5.1.0
args:
- "--csi-address=$(ADDRESS)"
- "--v=1"
- "--timeout=150s"
- "--retry-interval-start=500ms"
- "--leader-election=true"
- "--feature-gates=HonorPVReclaimPolicy=true"
- "--prevent-volume-mode-conversion=true"
# if fstype is not specified in storageclass, ext4 is default
- "--default-fstype=ext4"
- "--extra-create-metadata=true"
- "--immediate-topology=false"
- "--http-endpoint=$(POD_IP):8090"
env:
- name: ADDRESS
value: unix:///csi/csi-provisioner.sock
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
imagePullPolicy: "IfNotPresent"
ports:
- containerPort: 8090
name: provisioner
protocol: TCP
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: csi-snapshotter
image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/sig-storage/csi-snapshotter:v8.2.0
args:
- "--csi-address=$(ADDRESS)"
- "--v=1"
- "--timeout=150s"
- "--leader-election=true"
- "--extra-create-metadata=true"
- "--feature-gates=CSIVolumeGroupSnapshot=true"
- "--http-endpoint=$(POD_IP):8092"
env:
- name: ADDRESS
value: unix:///csi/csi-provisioner.sock
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
imagePullPolicy: "IfNotPresent"
ports:
- containerPort: 8092
name: snapshotter
protocol: TCP
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: csi-attacher
image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/sig-storage/csi-attacher:v4.8.0
args:
- "--v=1"
- "--csi-address=$(ADDRESS)"
- "--leader-election=true"
- "--retry-interval-start=500ms"
- "--default-fstype=ext4"
- "--http-endpoint=$(POD_IP):8093"
env:
- name: ADDRESS
value: /csi/csi-provisioner.sock
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
imagePullPolicy: "IfNotPresent"
ports:
- containerPort: 8093
name: attacher
protocol: TCP
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: csi-resizer
image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/longhornio/csi-resizer:v1.13.1
args:
- "--csi-address=$(ADDRESS)"
- "--v=1"
- "--timeout=150s"
- "--leader-election"
- "--retry-interval-start=500ms"
- "--handle-volume-inuse-error=false"
- "--feature-gates=RecoverVolumeExpansionFailure=true"
- "--http-endpoint=$(POD_IP):8091"
env:
- name: ADDRESS
value: unix:///csi/csi-provisioner.sock
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
imagePullPolicy: "IfNotPresent"
ports:
- containerPort: 8091
name: resizer
protocol: TCP
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: csi-rbdplugin-controller
image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/quay.io/cephcsi/cephcsi:v3.14.0
args:
- "--type=controller"
- "--v=5"
- "--drivername=rbd.csi.ceph.com"
- "--drivernamespace=$(DRIVER_NAMESPACE)"
- "--setmetadata=true"
env:
- name: DRIVER_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: ceph-csi-config
mountPath: /etc/ceph-csi-config/
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
# - name: ceph-config
# mountPath: /etc/ceph/
- name: liveness-prometheus
image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/quay.io/cephcsi/cephcsi:v3.14.0
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--metricsport=8680"
- "--metricspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi-provisioner.sock
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
ports:
- containerPort: 8680
name: http-metrics
protocol: TCP
volumeMounts:
- name: socket-dir
mountPath: /csi
imagePullPolicy: "IfNotPresent"
volumes:
- name: host-dev
hostPath:
path: /dev
- name: host-sys
hostPath:
path: /sys
- name: lib-modules
hostPath:
path: /lib/modules
- name: socket-dir
emptyDir: {
medium: "Memory"
}
# - name: ceph-config
# configMap:
# name: ceph-config
- name: ceph-csi-config
configMap:
name: ceph-csi-config
# - name: ceph-csi-encryption-kms-config
# configMap:
# name: ceph-csi-encryption-kms-config
- name: keys-tmp-dir
emptyDir: {
medium: "Memory"
}
- name: oidc-token
projected:
sources:
- serviceAccountToken:
path: oidc-token
expirationSeconds: 3600
audience: ceph-csi-kms
修改csi-rbdplugin.yaml
root@PS2022CVIYBBVU:/usr/local/ceph-csi-3.15.0/deploy/rbd/kubernetes# cat csi-rbdplugin.yaml
---
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: csi-rbdplugin
# replace with non-default namespace name
namespace: default
spec:
selector:
matchLabels:
app: csi-rbdplugin
template:
metadata:
labels:
app: csi-rbdplugin
spec:
serviceAccountName: rbd-csi-nodeplugin
hostNetwork: true
hostPID: true
priorityClassName: system-node-critical
# to use e.g. Rook orchestrated cluster, and mons' FQDN is
# resolved through k8s service, set dns policy to cluster first
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: csi-rbdplugin
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/quay.io/cephcsi/cephcsi:v3.14.0
args:
- "--nodeid=$(NODE_ID)"
- "--pluginpath=/var/lib/kubelet/plugins"
- "--stagingpath=/var/lib/kubelet/plugins/kubernetes.io/csi/"
- "--type=rbd"
- "--nodeserver=true"
- "--endpoint=$(CSI_ENDPOINT)"
- "--csi-addons-endpoint=$(CSI_ADDONS_ENDPOINT)"
- "--v=5"
- "--drivername=rbd.csi.ceph.com"
- "--enableprofiling=false"
# If topology based provisioning is desired, configure required
# node labels representing the nodes topology domain
# and pass the label names below, for CSI to consume and advertise
# its equivalent topology domain
# - "--domainlabels=failure-domain/region,failure-domain/zone"
#
# Options to enable read affinity.
# If enabled Ceph CSI will fetch labels from kubernetes node and
# pass `read_from_replica=localize,crush_location=type:value` during
# rbd map command. refer:
# https://docs.ceph.com/en/latest/man/8/rbd/#kernel-rbd-krbd-options
# for more details.
# - "--enable-read-affinity=true"
# - "--crush-location-labels=topology.io/zone,topology.io/rack"
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
# - name: KMS_CONFIGMAP_NAME
# value: encryptionConfig
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
- name: CSI_ADDONS_ENDPOINT
value: unix:///csi/csi-addons.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /csi
- mountPath: /dev
name: host-dev
- mountPath: /sys
name: host-sys
- mountPath: /run/mount
name: host-mount
- mountPath: /etc/selinux
name: etc-selinux
readOnly: true
- mountPath: /lib/modules
name: lib-modules
readOnly: true
- name: ceph-csi-config
mountPath: /etc/ceph-csi-config/
# - name: ceph-csi-encryption-kms-config
# mountPath: /etc/ceph-csi-encryption-kms-config/
- name: plugin-dir
mountPath: /var/lib/kubelet/plugins
mountPropagation: "Bidirectional"
- name: mountpoint-dir
mountPath: /var/lib/kubelet/pods
mountPropagation: "Bidirectional"
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
- name: ceph-logdir
mountPath: /var/log/ceph
# - name: ceph-config
# mountPath: /etc/ceph/
- name: oidc-token
mountPath: /run/secrets/tokens
readOnly: true
- name: driver-registrar
# This is necessary only for systems with SELinux, where
# non-privileged sidecar containers cannot access unix domain socket
# created by privileged CSI driver container.
securityContext:
privileged: true
allowPrivilegeEscalation: true
image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0
args:
- "--v=1"
- "--csi-address=/csi/csi.sock"
- "--kubelet-registration-path=/var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock"
env:
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: registration-dir
mountPath: /registration
- name: liveness-prometheus
securityContext:
privileged: true
allowPrivilegeEscalation: true
image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/quay.io/cephcsi/cephcsi:v3.14.0
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--metricsport=8680"
- "--metricspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
volumeMounts:
- name: socket-dir
mountPath: /csi
imagePullPolicy: "IfNotPresent"
volumes:
- name: socket-dir
hostPath:
path: /var/lib/kubelet/plugins/rbd.csi.ceph.com
type: DirectoryOrCreate
- name: plugin-dir
hostPath:
path: /var/lib/kubelet/plugins
type: Directory
- name: mountpoint-dir
hostPath:
path: /var/lib/kubelet/pods
type: DirectoryOrCreate
- name: ceph-logdir
hostPath:
path: /var/log/ceph
type: DirectoryOrCreate
- name: registration-dir
hostPath:
path: /var/lib/kubelet/plugins_registry/
type: Directory
- name: host-dev
hostPath:
path: /dev
- name: host-sys
hostPath:
path: /sys
- name: etc-selinux
hostPath:
path: /etc/selinux
- name: host-mount
hostPath:
path: /run/mount
- name: lib-modules
hostPath:
path: /lib/modules
# - name: ceph-config
# configMap:
# name: ceph-config
- name: ceph-csi-config
configMap:
name: ceph-csi-config
# - name: ceph-csi-encryption-kms-config
# configMap:
# name: ceph-csi-encryption-kms-config
- name: keys-tmp-dir
emptyDir: {
medium: "Memory"
}
- name: oidc-token
projected:
sources:
- serviceAccountToken:
path: oidc-token
expirationSeconds: 3600
audience: ceph-csi-kms
---
# This is a service to expose the liveness metrics
apiVersion: v1
kind: Service
metadata:
name: csi-metrics-rbdplugin
# replace with non-default namespace name
namespace: default
labels:
app: csi-metrics
spec:
ports:
- name: http-metrics
port: 8080
protocol: TCP
targetPort: 8680
selector:
app: csi-rbdplugin
部署Ceph CSI
cd /usr/local/ceph-csi-3.15.0/deploy/rbd/kubernetes kubectl apply -f csi-config-map.yaml kubectl apply -f csi-provisioner-rbac.yaml kubectl apply -f csi-nodeplugin-rbac.yaml kubectl apply -f csi-rbdplugin-provisioner.yaml kubectl apply -f csi-rbdplugin.yaml
2、ceph服务器操作
ceph创建pool池，并创建权限
# 1、创建池：k8s_mysql_pool
ceph osd pool create k8s_mysql_pool replicated
# 2、创建对应的操作权限：k8s_mysql_pool_user
ceph auth get-or-create client.k8s_mysql_pool_user mon 'allow r' osd 'allow rw pool=k8s_mysql_pool' -o ceph.client.k8s_mysql_pool_user.keyring
# 3、查看授权信息
ceph auth get client.k8s_mysql_pool_user
[client.k8s_mysql_pool_user]
key = AQDveaJpjt5DIRAA3FhB6UBN9jzRs72gKwN3Qw==
[root@ceph1 ceph]# ceph auth get client.k8s_mysql_pool_user
[client.k8s_mysql_pool_user]
key = AQDveaJpjt5DIRAA3FhB6UBN9jzRs72gKwN3Qw==
caps mon = "allow r"
caps osd = "allow rw pool=pool01"
exported keyring for client.k8s_mysql_pool_user
# 4、将key转成base64，
# ceph的userID
echo -n "k8s_mysql_pool_user" | base64
azhzX215c3FsX3Bvb2xfdXNlcg==
# ceph的userKey
echo -n "AQDveaJpjt5DIRAA3FhB6UBN9jzRs72gKwN3Qw==" | base64
QVFEdmVhSnBqdDVESVJBQTNGaEI2VUJOOWp6UnM3MmdLd04zUXc9PQ==
3、k8s服务器创建mysql资源清单
创建mysql的资源清单以及ceph配置
# 5、创建 ceph-secret.yaml cat <<EOF | kubectl apply -f - root@PS2022CVIYBBVU:/opt/k8s/ceph/mysql# cat ceph-secret.yaml apiVersion: v1 kind: Secret metadata: name: ceph-secret namespace: default type: "kubernetes.io/rbd" data: userID: azhzX215c3FsX3Bvb2xfdXNlcg== # ceph中，ceph的userID，用户k8s_mysql_pool_user经过base64转换后 userKey: QVFEdmVhSnBqdDVEQVJBQTNGaEI2VUJOOWp6UnM3MmdLd04zUXc9PQ== EOF # 6、生成base64格式的mysql用户名：root root@PS2022CVIYBBVU:/opt/k8s/ceph/mysql# echo -n "root" | base64 cm9vdA== # 7、生成base64格式的mysql密码：Admin1234 root@PS2022CVIYBBVU:/opt/k8s/ceph/mysql# echo -n "Admin1234" | base64 QWRtaW4xMjM0 # 7、创建mysql-secret cat > mysql-secret.yaml <<-'EOF' apiVersion: v1 kind: Secret metadata: name: mysql-password namespace: deploy-mysql type: Opaque data: # 用户名: root (base64: cm9vdA==) username: cm9vdA== # 密码: Admin1234 (base64: QWRtaW4xMjM0) password: QWRtaW4xMjM0 # MySQL 8.0 常用的环境变量key mysql-root-password: QWRtaW4xMjM0 # MYSQL_ROOT_PASSWORD mysql-password: QWRtaW4xMjM0 # MYSQL_PASSWORD mysql-database: bXlkYg== # mydb 的base64 EOF # 8、创建ceph-storageclass.yaml动态存储 root@PS2022CVIYBBVU:/opt/k8s/ceph/mysql# cat ceph-storageclass.yaml apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ceph-rbd provisioner: rbd.csi.ceph.com parameters: # 集群ID - 可选，如果只有一个Ceph集群可以注释掉 clusterID: 65eac714-0bd5-11f1-9e23-000c298c1628 # 指定存储池 pool: k8s_mysql_pool # Ceph Monitors 地址 monitors: 192.168.198.22:6789,192.168.198.41:6789,192.168.198.43:6789 # --- 关键修正：CSI 专用的 Secret 参数 --- # 用于 provisioning 和 controller expand 的 Secret csi.storage.k8s.io/provisioner-secret-name: ceph-secret csi.storage.k8s.io/provisioner-secret-namespace: default csi.storage.k8s.io/controller-expand-secret-name: ceph-secret csi.storage.k8s.io/controller-expand-secret-namespace: default # 用于 node stage (在节点上挂载) 的 Secret csi.storage.k8s.io/node-stage-secret-name: ceph-secret csi.storage.k8s.io/node-stage-secret-namespace: default # 文件系统类型 fsType: ext4 # RBD 镜像格式和特性 imageFormat: "2" imageFeatures: layering reclaimPolicy: Retain volumeBindingMode: Immediate --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: mysql-pvc spec: storageClassName: ceph-rbd accessModes: - ReadWriteOnce resources: requests: storage: 1Gi # 9、部署mysql服务 root@PS2022CVIYBBVU:/opt/k8s/ceph/mysql# cat mysql-statefulset.yaml # mysql-statefulset.yaml (核心部分) apiVersion: apps/v1 kind: StatefulSet metadata: name: mysql spec: serviceName: mysql replicas: 1 selector: matchLabels: app: mysql template: metadata: labels: app: mysql spec: containers: - name: mysql image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/mysql:8.0.45 env: - name: MYSQL_ROOT_PASSWORD valueFrom: secretKeyRef: name: mysql-secret # 需要先创建secret存储密码 key: root-password ports: - containerPort: 3306 volumeMounts: - name: data mountPath: /var/lib/mysql volumeClaimTemplates: - metadata: name: data spec: accessModes: ["ReadWriteOnce"] storageClassName: ceph-rbd resources: requests: storage: 1Gi # 启动服务 # kubectl apply -f ceph-secret.yaml # kubectl apply -f ceph-storageclass.yaml # kubectl apply -f mysql-secret.yaml # kubectl apply -f mysql-statefulset.yaml
更多推荐
13
0- 0
已为社区贡献1条内容
所有评论(0)