二进制k8s集群安装nfs storageclass

环境准备

##/etc/hosts
192.168.48.101 master01
192.168.48.102 master02
192.168.48.103 master03
192.168.48.201 node01
192.168.48.202 node02
192.168.48.54  nfs

## keepalived的vip
192.168.48.66

IP	Hostname	CPU	Memory
192.168.48.101	master01	2	4G
192.168.48.102	master02	2	4G
192.168.48.103	master03	2	4G
192.168.48.201	node01	2	4G
192.168.48.202	node02	2	4G
192.168.48.54	nfs	2	4G

软件	版本
kubernetes	1.15.2
docker-ce	19.03
calico	3.8
etcd	3.3.13
CNI	0.8.1
coredns	1.4.0
metrics-server	0.3.3
ingress-controller	0.25.0
dashboard	1.10.1
Weave Scope	1.11.4
nfs	v4

安装

所有节点先安装nfs

yum -y install nfs-utils

nfs服务器建立共享挂载目录

[root@nfs ~]# mkdir /data/k8s/sc  -pv
mkdir: created directory ‘/data’
mkdir: created directory ‘/data/k8s’
mkdir: created directory ‘/data/k8s/sc’

[root@nfs ~]# vim /etc/exports

/data/k8s/sc   192.168.48.0/24(insecure,rw,async,no_root_squash)

启动nfs

[root@nfs ~]# systemctl enable nfs
Created symlink from /etc/systemd/system/multi-user.target.wants/nfs-server.service to /usr/lib/systemd/system/nfs-server.service.
[root@nfs ~]# systemctl start nfs

编辑NFS-Client provisioner的yaml

vim nfs-client.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-client-provisioner

---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: nfs-client-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["list", "watch", "create", "update", "patch"]
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]

---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    namespace: default
roleRef:
  kind: ClusterRole
  name: nfs-client-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
  name: nfs-client-provisioner
spec:
  replicas: 1
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      serviceAccountName: nfs-client-provisioner
      containers:
        - name: nfs-client-provisioner
          image: quay.io/external_storage/nfs-client-provisioner:latest
          imagePullPolicy: IfNotPresent
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: nfs-k8s
            - name: NFS_SERVER
              value: 192.168.48.54
            - name: NFS_PATH
              value: /data/k8s/sc
      volumes:
        - name: nfs-client-root
          nfs:
            server: 192.168.48.54
            path: /data/k8s/sc

镜像下载

quay.io/external_storage/nfs-client-provisioner:latest

[root@master01 ~]# docker pull quay.azk8s.cn/external_storage/nfs-client-provisioner:latest

[root@master01 ~]# docker tag quay.azk8s.cn/external_storage/nfs-client-provisioner:latest  quay.io/external_storage/nfs-client-provisioner:latest

下载地址

链接: https://pan.baidu.com/s/1Nb_u_mFwUeYSTk80hLVTxg 提取码: x49v

docker load -i nfs-client-provisioner.tar_20190814_155233.gz

创建NFS-Client provisioner

[root@master01 ~]# kubectl apply -f nfs-client.yaml 
serviceaccount/nfs-client-provisioner created
clusterrole.rbac.authorization.k8s.io/nfs-client-provisioner-runner created
clusterrolebinding.rbac.authorization.k8s.io/run-nfs-client-provisioner created
deployment.extensions/nfs-client-provisioner created

编写nfs-storageclass的yaml

[root@master01 ~]# vim nfs-client-class.yaml

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: k8s-nfs-storage
provisioner: nfs-k8s

创建nfs-storageclass

[root@master01 ~]# kubectl apply -f  nfs-client-class.yaml
storageclass.storage.k8s.io/k8s-nfs-storage created

[root@master01 ~]# kubectl get pods
NAME                                    READY   STATUS    RESTARTS   AGE
nfs-client-provisioner-74c85598-x6djg   1/1     Running   0          2m46s
nginx-7bffc778db-97vxv                  1/1     Running   0          5h27m

[root@master01 ~]# kubectl get storageclasses.storage.k8s.io 
NAME              PROVISIONER   AGE
k8s-nfs-storage   nfs-k8s       98s

测试

创建pvc

[root@master01 ~]# vim test-pvc.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: test-pvc
  annotations:
    volume.beta.kubernetes.io/storage-class: "k8s-nfs-storage"
spec:
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 1Mi

[root@master01 ~]# kubectl apply -f test-pvc.yaml 
persistentvolumeclaim/test-pvc created

[root@master01 ~]# kubectl get pvc
NAME       STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS      AGE
test-pvc   Bound    pvc-aa4ecdd4-d9b3-4337-9e3b-d99e98ee7f24   1Mi        RWX            k8s-nfs-storage   34s

创建pod

[root@master01 ~]# vim test-pod.yaml

apiVersion: v1
kind: Pod
metadata:
  name: test-pod
  namespace: default
  labels:
    app: myapp
    type: pod
spec:
  containers:
  - name: myapp
    image: busybox:1.28
    command: ["/bin/sh","-c","touch /data/test.txt;sleep 3600"]
    volumeMounts:
    - name: html
      mountPath: /data
  volumes:
  - name: html
    persistentVolumeClaim:
      claimName: test-pvc

[root@master01 ~]# kubectl apply -f test-pod.yaml 
pod/test-pod created

nfs目录查看

[root@nfs ~]# cd /data/k8s/sc/default-test-pvc-pvc-aa4ecdd4-d9b3-4337-9e3b-d99e98ee7f24/
[root@nfs default-test-pvc-pvc-aa4ecdd4-d9b3-4337-9e3b-d99e98ee7f24]# ll
total 0
-rw-r--r-- 1 root root 0 Aug 14 16:13 test.txt