k8s部署MinIO Operator

基于K8s部署minio

子子子文

1029人浏览 · 2022-06-24 10:29:46

子子子文 · 2022-06-24 10:29:46 发布

Minio Operator

环境

[root@master-10 minio]# kubectl get no -o wide
NAME        STATUS   ROLES    AGE     VERSION   INTERNAL-IP     EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION           CONTAINER-RUNTIME
master-10   Ready    master   104d    v1.19.3   192.168.91.10   <none>        CentOS Linux 7 (Core)   3.10.0-1160.el7.x86_64   docker://19.3.15
node-11     Ready    node     44m     v1.19.3   192.168.91.11   <none>        CentOS Linux 7 (Core)   3.10.0-1160.el7.x86_64   docker://19.3.15
node-12     Ready    node     104d    v1.19.3   192.168.91.12   <none>        CentOS Linux 7 (Core)   3.10.0-1160.el7.x86_64   docker://19.3.15
node-13     Ready    node     3d16h   v1.19.3   192.168.91.13   <none>        CentOS Linux 7 (Core)   3.10.0-1160.el7.x86_64   docker://19.3.15
node-14     Ready    node     3d16h   v1.19.3   192.168.91.14   <none>        CentOS Linux 7 (Core)   3.10.0-1160.el7.x86_64   docker://19.3.15
[root@master-10 minio]# kubectl top no 
NAME        CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%   
master-10   183m         2%     2148Mi          27%       
node-11     74m          1%     807Mi           22%       
node-12     76m          1%     876Mi           23%       
node-13     86m          2%     874Mi           23%       
node-14     88m          2%     779Mi           21%

前期准备

创建目录

#db001
mkdir -p /home/data/local-volume/minio/v1/node-pv0
#master1
mkdir -p /home/data/local-volume/minio/v1/node-pv1
#node001
mkdir -p /home/data/local-volume/minio/v1/node-pv2
#node002
mkdir -p /home/data/local-volume/minio/v1/node-pv3

创建PV和SC

#v1-node-pv-sc.yaml
#静态存储
apiVersion: v1
kind: PersistentVolume
metadata:
   name: minio-node-v1-pv0
spec:
   capacity:
      storage: 10Gi  
   volumeMode: Filesystem
   accessModes:
   - ReadWriteOnce
   persistentVolumeReclaimPolicy: Retain
   storageClassName: node-local-storage
   local:
      path: /home/data/local-volume/minio/v1/node-pv0 
   nodeAffinity:
      required:
         nodeSelectorTerms:
         - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
              - db001
---
apiVersion: v1
kind: PersistentVolume
metadata:
   name: minio-node-v1-pv1
spec:
   capacity:
      storage: 10Gi
   volumeMode: Filesystem
   accessModes:
   - ReadWriteOnce
   persistentVolumeReclaimPolicy: Retain
   storageClassName: node-local-storage
   local:
      path: /home/data/local-volume/minio/v1/node-pv1
   nodeAffinity:
      required:
         nodeSelectorTerms:
         - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
              - master1

---
apiVersion: v1
kind: PersistentVolume
metadata:
   name: minio-node-v1-pv2
spec:
   capacity:
      storage: 10Gi
   volumeMode: Filesystem
   accessModes:
   - ReadWriteOnce
   persistentVolumeReclaimPolicy: Retain
   storageClassName: node-local-storage
   local:
      path: /home/data/local-volume/minio/v1/node-pv2
   nodeAffinity:
      required:
         nodeSelectorTerms:
         - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
              - node001
---
apiVersion: v1
kind: PersistentVolume
metadata:
   name: minio-node-v1-pv3
spec:
   capacity:
      storage: 10Gi
   volumeMode: Filesystem
   accessModes:
   - ReadWriteOnce
   persistentVolumeReclaimPolicy: Retain
   storageClassName: node-local-storage
   local:
      path: /home/data/local-volume/minio/v1/node-pv3 
   nodeAffinity:
      required:
         nodeSelectorTerms:
         - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
              - node002
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
    name: node-local-storage
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer

创建namespace

[root@master1 minio]# kubectl create namespace minio-node
namespace/minio-node created

检查

[root@master1 minio]# kubectl get sc
NAME                 PROVISIONER                    RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION   AGE
node-local-storage   kubernetes.io/no-provisioner   Delete          WaitForFirstConsumer   false                  40s

[root@master1 minio]# kubectl get pv,ns -A | grep minio
persistentvolume/minio-node-v1-pv0   10Gi       RWO            Retain           Available                         node-local-storage            10s
persistentvolume/minio-node-v1-pv1   10Gi       RWO            Retain           Available                         node-local-storage            10s
persistentvolume/minio-node-v1-pv2   10Gi       RWO            Retain           Available                         node-local-storage            10s
persistentvolume/minio-node-v1-pv3   10Gi       RWO            Retain           Available                         node-local-storage            10s
namespace/minio-node                Active   26s
namespace/minio-operator            Active   4d23h

安装MinIO Operator

kubecrew

Make sure that git is installed.

Run this command to download and install krew:

项目地址：https://github.com/kubernetes-sigs/krew/releases

wget https://github.com/kubernetes-sigs/krew/releases/download/v0.4.1/krew.tar.gz
wget https://github.com/kubernetes-sigs/krew/releases/download/v0.4.1/krew.yaml
tar -zxvf krew.tar.gz
./krew-linux_amd64 install --manifest=krew.yaml --archive=krew.tar.gz

Add the $HOME/.krew/bin directory to your PATH environment variable. To do this, update your .bashrc or .zshrc file and append the following line:
```
export PATH="${KREW_ROOT:-$HOME/.krew}/bin:$PATH"
echo 'export PATH="${KREW_ROOT:-$HOME/.krew}/bin:$PATH"' >> ~/.bashrc
source ~/.bashrc
```
and restart your shell.
Run kubectl krew to check the installation.
```
kubectl krew
```

#krew会依赖git，注意git是否正确安装且写入环境变量
[root@master-10 minio]# kubectl krew update
Updated the local copy of plugin index.
#被墙容易失败,多试几次
[root@master-10 minio]# kubectl krew install minio
F0310 11:03:25.508814   73405 root.go:58] failed to update the local index: fetch index at "/root/.krew/index" failed: command execution failure, output="fatal: unable to access 'https://github.com/kubernetes-sigs/krew-index.git/': TCP connection reset by peer\n": exit status 128
[root@master-10 minio]# kubectl krew install minio
Updated the local copy of plugin index.
Installing plugin: minio
Installed plugin: minio
\
 | Use this plugin:
 | 	kubectl minio
 | Documentation:
 | 	https://github.com/minio/operator/tree/master/kubectl-minio
 | Caveats:
 | \
 |  | * For resources that are not in default namespace, currently you must
 |  |   specify -n/--namespace explicitly (the current namespace setting is not
 |  |   yet used).
 | /
/
WARNING: You installed a plugin from the krew-index plugin repository.
   These plugins are not audited for security by the Krew maintainers.
   Run them at your own risk.
   
   
#######离线
https://github.com/minio/operator/releases
wget https://github.com/minio/operator/releases/download/v4.4.11/kubectl-minio_4.4.11_linux_amd64 -O kubectl-minio
chmod +x kubectl-minio
mv kubectl-minio /usr/local/bin/

#Run the following command to verify installation of the plugin:
[root@master-10 minio]# kubectl minio version
v4.4.11

#Run the following command to initialize the Operator:
[root@master-10 minio]# kubectl minio init --cluster-domain=cluster.saas
namespace/minio-operator created
serviceaccount/minio-operator created
clusterrole.rbac.authorization.k8s.io/minio-operator-role created
clusterrolebinding.rbac.authorization.k8s.io/minio-operator-binding created
customresourcedefinition.apiextensions.k8s.io/tenants.minio.min.io created
service/operator created
deployment.apps/minio-operator created
serviceaccount/console-sa created
clusterrole.rbac.authorization.k8s.io/console-sa-role created
clusterrolebinding.rbac.authorization.k8s.io/console-sa-binding created
configmap/console-env created
service/console created
deployment.apps/console created
-----------------

To open Operator UI, start a port forward using this command:

kubectl minio proxy -n minio-operator 
-----------------
[root@master-10 ~]# kubectl get pods -n minio-operator
NAME                              READY   STATUS    RESTARTS   AGE
console-58b7dc8856-xlcbf          1/1     Running   2          22h
minio-operator-849f8cf8b8-ldq9x   1/1     Running   2          21h
minio-operator-849f8cf8b8-wf4qw   1/1     Running   2          22h

进入控制台

#新建终端
[root@master-10 minio]# kubectl minio proxy -n minio-operator 
Starting port forward of the Console UI.

To connect open a browser and go to http://localhost:9090

Current JWT to login: eyJhbGciOiJSUzI1NiIsImtpZCI6InY4NkNBSHptSUZhbXIyS2VWMk4yc3Rua2M0bUQyc2tYZVVHVVVDWEE1Y3cifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJtaW5pby1vcGVyYXRvciIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJjb25zb2xlLXNhLXRva2VuLTkyODU3Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImNvbnNvbGUtc2EiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiI4OGI1MDhjMS01YmI0LTQ0MDAtOWIyZi05YTMxYzZmZWJiYmMiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6bWluaW8tb3BlcmF0b3I6Y29uc29sZS1zYSJ9.s0ycCfkyvGmhxKJULjTz56yipt0T_viP5GTiTfTS3yBawYpiL-d9usO-Ot9H-7gRTKT9euykENeQF3D4lhpSeVlATD0RjFwY4D-Ks7MT3v2UWJ5j5GItpY6qMHsS-cKuRVosSRhgyFf9Fc4He_CbY8e5GdbWTW-0P138FmEPcrogrTw37AjHySjKbXrLzGEL84rdGTL2lgRpmPwQWXYerchNaGNFO5ir3PJGf7PYkmKfDByq02P_yhT4U70JgLhNgPwT9i0MzHnSORUFKyuNRaIoSW6-tgh7Wk2eEUTZI79QgIsaJ6GH4-oDNZcSf6mHKNZbEkaUt6VgoBR0VYZhJg



Forwarding from 0.0.0.0:9090 -> 9090

#复制JWT进入控制台
http://192.168.91.10:9090/

创建租户

minio-operator

minio-node
UfrFvdv9Wturm5FO
A6UNtGUr3QESHLzkSTwqdaNUNiyNRpBt

命令行方式

[root@master-10 minio]# kubectl minio tenant create tenant1 --namespace minio-tenant-1 --storage-class local-storage --servers 4 --volumes 4 --capacity 40Gi

Tenant 'tenant1' created in 'minio-tenant-1' Namespace

  Username: admin 
  Password: 933c9ff5-3ff5-4331-b95c-477830cf8825 
  Note: Copy the credentials to a secure location. MinIO will not display these again.

+-------------+-----------------+----------------+--------------+--------------+
| APPLICATION | SERVICE NAME    | NAMESPACE      | SERVICE TYPE | SERVICE PORT |
+-------------+-----------------+----------------+--------------+--------------+
| MinIO       | minio           | minio-tenant-1 | ClusterIP    | 443          |
| Console     | tenant1-console | minio-tenant-1 | ClusterIP    | 9443         |
+-------------+-----------------+----------------+--------------+--------------+

检查

[root@master1 minio]# kubectl get all -n minio-node 
NAME                      READY   STATUS    RESTARTS   AGE
pod/minio-node-pool-0-0   1/1     Running   0          57s
pod/minio-node-pool-0-1   1/1     Running   0          56s
pod/minio-node-pool-0-2   1/1     Running   0          56s
pod/minio-node-pool-0-3   1/1     Running   0          56s

NAME                         TYPE           CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE
service/minio                LoadBalancer   10.233.31.36   <pending>     80:31819/TCP     59s
service/minio-node-console   LoadBalancer   10.233.38.9    <pending>     9090:30786/TCP   58s
service/minio-node-hl        ClusterIP      None           <none>        9000/TCP         57s

NAME                                 READY   AGE
statefulset.apps/minio-node-pool-0   4/4     57s

#######
The minio service corresponds to the MinIO Tenant service. Applications should use this service for performing operations against the MinIO Tenant.
The minio-tenant-1-console service corresponds to the MinIO Console. Administrators should use this service for accessing the MinIO Console and performing administrative operations on the MinIO Tenant.
The minio-tenant-1-hl corresponds to a headless service used to facilitate communication between Pods in the Tenant.

删掉重建

[root@master1 minio]# kubectl delete pvc --all -n minio-node
[root@master1 minio]# kubectl delete -f v1-node-pv-sc.yaml 

[root@master1 minio]# kubectl delete secrets -n minio-node  minio-node-secret
secret "minio-node-secret" deleted

#新
lmfmRBFopIQS0Pmn
ZP823zRd7cz85rkcHfhyEQDIaq2OOAav

创建ingress

openssl genrsa -out tls.key 2048

#创建密钥    域名为www.minio.com
openssl req -new -x509 -key tls.key -out tls.crt -subj /C=CN/ST=Guangdong/L=Guangzhou/O=devops/CN=www.minio.com
#创建secret
kubectl create secret tls minio-secret --cert=tls.crt --key=tls.key  -n minio-node

[root@master1 minio]# kubectl create secret tls minio-secret --cert=tls.crt --key=tls.key  -n minio-node
secret/minio-secret created

ingress.yaml

[root@master1 minio]# cat minio-node-ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    nginx.ingress.kubernetes.io/use-regex: "true"
  name: minio-ingress
  namespace: minio-node
spec:
#  tls:
#  - hosts: 
#    - www.minio.com
#    secretName: minio-secret
  rules:
  - host: www.minio.com
    http:
      paths:
      - path: /
        backend:
          serviceName: minio
          servicePort: 80

创建ingress

[root@master1 minio]# kubectl create -f minio-node-ingress.yaml 
Warning: extensions/v1beta1 Ingress is deprecated in v1.14+, unavailable in v1.22+; use networking.k8s.io/v1 Ingress
ingress.extensions/minio-ingress created

访问tenant

在windows端设置域名解析

192.168.0.84 www.minio.com

浏览器访问

Python SDK

Download using pip

pip3 install minio

Download source

git clone https://github.com/minio/minio-py.git
cd minio-py
python3 setup.py install  #linux

file_uploader.py

from minio import Minio
from minio.error import S3Error


def main():
    # Create a client with the MinIO server playground, its access key
    # and secret key.
    client = Minio(
        "www.minio.com",
        access_key="UfrFvdv9Wturm5FO",
        secret_key="A6UNtGUr3QESHLzkSTwqdaNUNiyNRpBt",
        secure="false",
    )

    # Make 'test1' bucket if not exist.
    found = client.bucket_exists("test1")
    if not found:
        client.make_bucket("test1")
    else:
        print("Bucket 'test1' already exists")

    # Upload '/root/kube-yaml/minio/v1-node-pv-sc.yaml' as object name
    # 'v1-node-pv-sc.yaml' to bucket 'test1'.
    client.fput_object(
        "test1", "v1-node-pv-sc.yaml", "/root/kube-yaml/minio/v1-node-pv-sc.yaml",
    )
    print(
        "'/root/kube-yaml/minio/v1-node-pv-sc.yaml' is successfully uploaded as "
        "object 'v1-node-pv-sc.yaml' to bucket 'test1'."
    )


if __name__ == "__main__":
    try:
        main()
    except S3Error as exc:
        print("error occurred.", exc)

Run File Uploader

$ python3 file_uploader.py
'/home/user/Photos/asiaphotos.zip' is successfully uploaded as object 'asiaphotos-2015.zip' to bucket 'asiatrip'.

$ mc ls play/test1/
[2016-06-02 18:10:29 PDT]  82KiB asiaphotos-2015.zip

K8S/Kubernetes

K8S/Kubernetes社区为您提供最前沿的新闻资讯和知识内容

更多推荐

【深度】阿里巴巴万级规模 K8s 集群全局高可用体系之美

作者 | 韩堂、柘远、沉醉来源 | 阿里巴巴云原生公众号前言台湾作家林清玄在接受记者采访的时候，如此评价自己 30 多年写作生涯：“第一个十年我才华横溢，‘贼光闪现’，令周边黯然失色；第二个十年，我终于‘宝光现形’，不再去抢风头，反而与身边的美丽相得益彰；进入第三个十年，繁华落尽见真醇，我进入了‘醇光初现’的阶段，真正体味到了境界之美”。长夜有穷，真水无香。领略过了 K8s“身在江

K8S/Kubernetes

如何基于 K8s 构建下一代 DevOps 平台？

作者 | 孙健波（天元）导读：当前云原生 DevOps 体系现状如何？面临哪些挑战？如何通过 OAM 解决云原生 DevOps 场景下的诸多问题？云原生开发应用模型 OAM(Open Application Model) 社区核心成员孙健波将为大家一一解答，并分享如何基于 OAM 和 Kubernetes 打造无限能力的下一代 DevOps 平台。什么是 DevOps？为什么基于 Kub