拉取doris的镜像:

docker pull apache/doris:2.0.0_alpha-be-x86_64

docker pull apache/doris:2.0.0_alpha-fe-x86_64

先创建doris_be.yaml的文件,如果部署的时候提示这个错误“error: error validating "doris_be.yml": error validating data: invalid object to validate; if you choose to ignore these errors, turn validation off with --validate=false”可以试用Validate YAML - Online YAML Tools这个来检查yaml文件的格式,看哪里需要修改。

apiVersion: v1
kind: Service
metadata:
  name: doris-be-cluster1
  labels:
    app: doris-be-cluster1
spec:
  ports:
  - port: 9060
    name: be-port
  - port: 8040
    name: webserver-port
  - port: 9050
    name: heartbeat-port #This name should be fixed. Doris will get the port information through this name
  - port: 8060
    name: brpc-port
  clusterIP: None
  selector:
    app: doris-be-cluster1
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: doris-be-cluster1
  labels:
    app: doris-be-cluster1
spec:
  selector:
    matchLabels:
      app: doris-be-cluster1
  serviceName: doris-be-cluster1
  replicas: 3
  template:
    metadata:
      name: doris-be-cluster1
      labels:
        app: doris-be-cluster1
    spec:
      nodeSelector:
        tag: bigdata
    	initContainers:
      - command:
        - sh
        - -c
        - sysctl -w vm.max_map_count=2000000
        image: apache/doris:2.0.0_alpha-be-x86_64
        imagePullPolicy: IfNotPresent
        name: sysctl
        resources: {}
        securityContext:
          privileged: true
      containers:
      - name: doris-be-cluster1
        #Need to change to real mirror information
        #image: apache-doris-be:test
        # 修改点1: 镜像地址修改为真实doris镜像,可在https://hub.docker.com/r/apache/doris/tags找到需要的镜像版本
        image: apache/doris:2.0.0_alpha-be-x86_64
        imagePullPolicy: IfNotPresent
        env:
        #Specify the startup type as k8s to bypass some restrictions of the official image initialization script
        - name: BUILD_TYPE
          value: "k8s"
          # 修改点2: 增加环境变量,写明FE的IP与端口
        - name: FE_MASTER_IP
          value: "doris-follower-cluster1-0.doris-follower-cluster1.doris.svc.cluster.local"
        - name: FE_MASTER_PORT
          value: "9030"
        ports: 
        - containerPort: 9060
          name: be-port
        - containerPort: 8040
          name: webserver-port
        - containerPort: 9050
          name: heartbeat-port
        - containerPort: 8060
          name: brpc-port
        volumeMounts:
        #Mount the configuration file in the way of configmap
        - name: conf
          mountPath: /opt/apache-doris/be/conf
          #Ifnot mounted, when enable_profile, error will be reported when querying the data from jdbc catalog
          #Error message: error setting certificate verify locations: CAfile:/etc/pki/tls/certs/ca-bundle.crt CApath: none
        - name: sys
          mountPath: /etc/pki
          # 修改点3: 挂载存储
          subPath: pki
          readOnly: true
          # 同修改点3
        - name: sys
          mountPath: /opt/apache-doris/be/storage
          subPath: storage
      volumes:
      - name: conf
        configMap:
          name: be-conf
      - name: sys
        # 修改点4:不使用hostpath,注释掉
            #hostPath:
            #path: /etc/pki
  # 修改点5: 增加存储配置,此处使用longhorn
  volumeClaimTemplates:
  - metadata:
      name: sys
    spec:
      storageClassName: local-path
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 50Gi 
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: be-conf
data:
  be.conf: |
    PPROF_TMPDIR="$DORIS_HOME/log/"
    sys_log_level = INFO

    be_port = 9060
    webserver_port = 8040
    heartbeat_service_port = 9050
    brpc_port = 8060

    # 修改点6: 修改网段为k8s使用网段,配置数据存储路径
    priority_networks = 10.42.0.0/16
    storage_root_path = /opt/apache-doris/be/storage

然后创建doris_follower.yml文件

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

apiVersion: v1
kind: Service
metadata:
  name: doris-follower-cluster1
  labels:
    app: doris-follower-cluster1
spec:
  ports:
    - port: 8030
      name: http-port
    - port: 9020
      name: rpc-port
    - port: 9030
      name: query-port
    - port: 9010
      name: edit-log-port #This name should be fixed. Doris will get the port information through this name
  clusterIP: None
  selector:
    app: doris-follower-cluster1
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: doris-follower-cluster1
  labels:
    app: doris-follower-cluster1
spec:
  selector:
    matchLabels:
      app: doris-follower-cluster1
  serviceName: doris-follower-cluster1
  # 修改点1: 修改fe副本为1
  replicas: 1
  template:
    metadata:
      name: doris-follower-cluster1
      labels:
        app: doris-follower-cluster1
    spec:
      nodeSelector:
        tag: bigdata
      containers:
        - name: doris-follower-cluster1
          #Need to change to real mirror information
          # 修改点2: 镜像地址修改为真实doris镜像,可在https://hub.docker.com/r/apache/doris/tags找到需要的镜像版本
          image: apache/doris:2.0.0_alpha-fe-x86_64
          imagePullPolicy: IfNotPresent
          env:
            # 修改点3: 增加了APP_NAMESPACE与FE_IPADDRESS环境变量
            - name: APP_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
            - name: FE_IPADDRESS
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
            #Specify the startup type as k8s to bypass some restrictions of the official image initialization script
            - name: BUILD_TYPE
              value: "k8s"
            #Initialize the fe of three nodes
            - name: FE_INIT_NUMBER
            # 修改点4: 将数量改为1
              value: "1"
            #ServiceName of bakend_cn node,(if do not have bakend_cn node,do not configure this environment variable)
            # 修改点5: 不使用cn节点,注释变量CN_SERVICE  CN_STATEFULSET
            #- name: CN_SERVICE
            #  value: "doris-cn-cluster1"
            #StatefulSetName of bakend_cn node,(if do not have bakend_cn node,do not configure this environment variable)
            #- name: CN_STATEFULSET
            #  value: "doris-cn-cluster1"
            #ServiceName of bakend node,(if do not have bakend node,do not configure this environment variable)
            - name: BE_SERVICE
              value: "doris-be-cluster1"
            #StatefulSetName of bakend node,(if do not have bakend node,do not configure this environment variable)
            - name: BE_STATEFULSET
              value: "doris-be-cluster1"
            #ServiceName of follower node,(if do not have follower node,do not configure this environment variable)
            - name: FE_SERVICE
              value: "doris-follower-cluster1"
            ##StatefulSetName of follower node,(if do not have follower node,do not configure this environment variable)
            - name: FE_STATEFULSET
              value: "doris-follower-cluster1"
          ports:
            - containerPort: 8030
              name: http-port
            - containerPort: 9020
              name: rpc-port
            - containerPort: 9030
              name: query-port
            - containerPort: 9010
              name: edit-log-port
          volumeMounts:
            #Mount the configuration file in the way of configmap
            - name: conf
              mountPath: /opt/apache-doris/fe/conf
              #In order to call the api of k8s
            - name: kube
            # 使用本地配置则为/root/.kube/config
              mountPath: /root/.kube
              readOnly: true
            # 修改点6: 配置存储,用于元数据持久化
            - name: metadata
              mountPath: /opt/apache-doris/fe/doris-meta
      volumes:
        - name: conf
          configMap:
            name: follower-conf
        - name: kube
        # 修改点7: 修改为使用configMap(此处可以不修改,使用本地配置)
          #hostPath:
            #path: /root/.kube/config
          configMap:
            name: kube-conf
  # 修改点8: 增加存储配置,此处使用longhorn
  volumeClaimTemplates:
  - metadata:
      name: metadata
    spec:
      storageClassName: local-path
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 10Gi
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: follower-conf
data:
  fe.conf: |
    # 修改点9: 修改网段为k8s使用网段
    priority_networks = 10.42.0.0/16
    #It can automatically maintain node information by getting the number of replicas of StatefulSet, similar to alter system add/drop back
    enable_deploy_manager = k8s
    #Automatically adjust the IP of the node according to the domain name (for example, after the pod is restarted, the domain name is still doris-be-cluster1-0-doris-be-cluster1.default.svc.cluster.local, but the IP may change from 172.16.0.9 to 172.16.0.10)
    enable_fqdn_mode = true
    LOG_DIR = ${DORIS_HOME}/log
    sys_log_level = INFO
    http_port = 8030
    rpc_port = 9020
    query_port = 9030
    edit_log_port = 9010
    #Doris needs to generate the log4j configuration file according to the fe.yml configuration information, which is written in the same directory as fe.yml by default, but the config we mount is readonly, so specify this configuration to write the log4j file to another location
    custom_config_dir = /opt/apache-doris/
    #when set to false, the backend will not be dropped and remaining in DECOMMISSION state
    drop_backend_after_decommission = false
    # 修改点10: 增加元数据、java等配置
    mysql_service_nio_enabled = true
    JAVA_OPTS = "-Xmx8192m -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$DATE"
    JAVA_OPTS_FOR_JDK_9 = "-Xmx8192m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParalle=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$DATE:time"
    meta_dir = /opt/apache-doris/fe/doris-meta
    #metadata_failure_recovery = true
---
# 修改点11: 增加kube config的configmap(此处可不修改,直接使用本地节点配置)
apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-conf
data:
  config: |
    # 此处内容复制配置文件内容,如果server地址指向的为127.0.0.1,注意修改为实际节点IP
    # 一般为 /root/.kube/config 内容, rancher安装的rke2环境默认为/etc/rancher/rke2/rke2.yaml

然后创建doris-svc.yaml文件

apiVersion: v1
kind: Service
metadata:
  name: doris-svc
  namespace: doris
spec:
  type: NodePort
  ports:
    - port: 8030
      nodePort: 30803
      name: "p8030"
    - port: 9030
      nodePort: 30903
      name: "p9030"
  selector:
    app: doris-follower-cluster1

最后部署

#先创建命名空间
kubectl create ns doris
#然后部署be三个副本
kubectl apply -f doris_be.yml -n doris
#再部署一个fe
kubectl apply -f doris_follower.yml -n doris
#最后创建一个外部访问的服务svc
kubectl apply -f doris-svc.yaml -n doris

如果需要删除这些,使用kubectl get deploy xxx -n doris是找不到的,因为kubernetes部署分为两种,一种是有状态的使用“sts”,无状态的使用“deployment”

所以查找doris的需要使用下面的语句才能查找到:

kubect get sts -n doris

删除的时候使用如下语句:

kubectl delete sts doris-follower-cluster1 -n doris

通过查看pod的日志: kubectl -n doris logs -f doris-follower-cluster1-0 ,提示需要证书certification

解决上面的问题需要在spec:containers:env下面加上

KUBERNETES_TRUST_CERTIFICATES:"true" ,

KUBERNETES_DISABLE_HOSTNAME_VERIFICATION:"true"

查看k8s上yaml的文件:

第一步先kubectl get po -n doris,查看pod名称

通过验证,最终正确的doris_be.yaml

apiVersion: v1
kind: Service
metadata:
  name: doris-be-cluster1
  labels:
    app: doris-be-cluster1
spec:
  ports:
  - port: 9060
    name: be-port
  - port: 8040
    name: webserver-port
  - port: 9050
    name: heartbeat-port #This name should be fixed. Doris will get the port information through this name
  - port: 8060
    name: brpc-port
  clusterIP: None
  selector:
    app: doris-be-cluster1
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: doris-be-cluster1
  labels:
    app: doris-be-cluster1
spec:
  selector:
    matchLabels:
      app: doris-be-cluster1
  serviceName: doris-be-cluster1
  replicas: 3
  template:
    metadata:
      name: doris-be-cluster1
      labels:
        app: doris-be-cluster1
    spec:
      nodeSelector:
        tag: bigdata
        initContainers:
      - command:
        - sh
        - -c
        - sysctl -w vm.max_map_count=2000000
        image: apache/doris:2.0.0_alpha-be-x86_64
        imagePullPolicy: IfNotPresent
        name: sysctl
        resources: {}
        securityContext:
          privileged: true
      containers:
      - name: doris-be-cluster1
        #Need to change to real mirror information
        #image: apache-doris-be:test
        # 修改点1: 镜像地址修改为真实doris镜像,可在https://hub.docker.com/r/apache/doris/tags找到需要的镜像版本
        image: apache/doris:2.0.0_alpha-be-x86_64
        imagePullPolicy: IfNotPresent
        env:
        #Specify the startup type as k8s to bypass some restrictions of the official image initialization script
        - name: BUILD_TYPE
          value: "k8s"
          # 修改点2: 增加环境变量,写明FE的IP与端口
        - name: FE_MASTER_IP
          value: "doris-follower-cluster1-0.doris-follower-cluster1.doris.svc.cluster.local"
        - name: FE_MASTER_PORT
          value: "9030"
        ports: 
        - containerPort: 9060
          name: be-port
        - containerPort: 8040
          name: webserver-port
        - containerPort: 9050
          name: heartbeat-port
        - containerPort: 8060
          name: brpc-port
        volumeMounts:
        #Mount the configuration file in the way of configmap
        - name: conf
          mountPath: /opt/apache-doris/be/conf
          #Ifnot mounted, when enable_profile, error will be reported when querying the data from jdbc catalog
          #Error message: error setting certificate verify locations: CAfile:/etc/pki/tls/certs/ca-bundle.crt CApath: none
        - name: sys
          mountPath: /etc/pki
          # 修改点3: 挂载存储
          subPath: pki
          readOnly: true
          # 同修改点3
        - name: sys
          mountPath: /opt/apache-doris/be/storage
          subPath: storage
      volumes:
      - name: conf
        configMap:
          name: be-conf
      - name: sys
        # 修改点4:不使用hostpath,注释掉
            #hostPath:
            #path: /etc/pki
  # 修改点5: 增加存储配置,此处使用longhorn
  volumeClaimTemplates:
  - metadata:
      name: sys
    spec:
      storageClassName: local-path
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 50Gi 
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: be-conf
data:
  be.conf: |
    PPROF_TMPDIR="$DORIS_HOME/log/"
    sys_log_level = INFO

    be_port = 9060
    webserver_port = 8040
    heartbeat_service_port = 9050
    brpc_port = 8060

    # 修改点6: 修改网段为k8s使用网段,配置数据存储路径
    priority_networks = 10.42.0.0/16
    storage_root_path = /opt/apache-doris/be/storage

最终正确的doris_follower.yaml。最后一段注释掉,因为使用了serviceaccount,所以访问k8s的时候直接使用serviceaccount的,不需要配置.kube/config

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

apiVersion: v1
kind: Service
metadata:
  name: doris-follower-cluster1
  labels:
    app: doris-follower-cluster1
spec:
  ports:
    - port: 8030
      name: http-port
    - port: 9020
      name: rpc-port
    - port: 9030
      name: query-port
    - port: 9010
      name: edit-log-port #This name should be fixed. Doris will get the port information through this name
  clusterIP: None
  selector:
    app: doris-follower-cluster1
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: doris-follower-cluster1
  labels:
    app: doris-follower-cluster1
spec:
  selector:
    matchLabels:
      app: doris-follower-cluster1
  serviceName: doris-follower-cluster1
  # 修改点1: 修改fe副本为1
  replicas: 1
  template:
    metadata:
      name: doris-follower-cluster1
      labels:
        app: doris-follower-cluster1
    spec:
      serviceAccountName: doris
      nodeSelector:
        tag: bigdata
      containers:
        - name: doris-follower-cluster1
          #Need to change to real mirror information
          # 修改点2: 镜像地址修改为真实doris镜像,可在https://hub.docker.com/r/apache/doris/tags找到需要的镜像版本
          image: apache/doris:2.0.0_alpha-fe-x86_64
          imagePullPolicy: IfNotPresent
          env:
            # 修改点3: 增加了APP_NAMESPACE与FE_IPADDRESS环境变量
            - name: APP_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
            - name: FE_IPADDRESS
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
            #Specify the startup type as k8s to bypass some restrictions of the official image initialization script
            - name: BUILD_TYPE
              value: "k8s"
            #Initialize the fe of three nodes
            - name: FE_INIT_NUMBER
            # 修改点4: 将数量改为1
              value: "1"
            #ServiceName of bakend_cn node,(if do not have bakend_cn node,do not configure this environment variable)
            # 修改点5: 不使用cn节点,注释变量CN_SERVICE  CN_STATEFULSET
            #- name: CN_SERVICE
            #  value: "doris-cn-cluster1"
            #StatefulSetName of bakend_cn node,(if do not have bakend_cn node,do not configure this environment variable)
            #- name: CN_STATEFULSET
            #  value: "doris-cn-cluster1"
            #ServiceName of bakend node,(if do not have bakend node,do not configure this environment variable)
            - name: BE_SERVICE
              value: "doris-be-cluster1"
            #StatefulSetName of bakend node,(if do not have bakend node,do not configure this environment variable)
            - name: BE_STATEFULSET
              value: "doris-be-cluster1"
            #ServiceName of follower node,(if do not have follower node,do not configure this environment variable)
            - name: FE_SERVICE
              value: "doris-follower-cluster1"
            ##StatefulSetName of follower node,(if do not have follower node,do not configure this environment variable)
            - name: FE_STATEFULSET
              value: "doris-follower-cluster1"
            - name: KUBERNETES_TRUST_CERTIFICATES
              value: "true"
            - name: KUBERNETES_DISABLE_HOSTNAME_VERIFICATION
              value: "true"
          ports:
            - containerPort: 8030
              name: http-port
            - containerPort: 9020
              name: rpc-port
            - containerPort: 9030
              name: query-port
            - containerPort: 9010
              name: edit-log-port
          volumeMounts:
            #Mount the configuration file in the way of configmap
            - name: conf
              mountPath: /opt/apache-doris/fe/conf
              #In order to call the api of k8s
            # 修改点6: 配置存储,用于元数据持久化
            - name: metadata
              mountPath: /opt/apache-doris/fe/doris-meta
      volumes:
        - name: conf
          configMap:
            name: follower-conf
  # 修改点8: 增加存储配置,此处使用longhorn
  volumeClaimTemplates:
  - metadata:
      name: metadata
    spec:
      storageClassName: csi-disk
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 10Gi
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: follower-conf
data:
  fe.conf: |
    # 修改点9: 修改网段为k8s使用网段
    priority_networks = 172.31.0.0/16
    #It can automatically maintain node information by getting the number of replicas of StatefulSet, similar to alter system add/drop back
    enable_deploy_manager = k8s
    #Automatically adjust the IP of the node according to the domain name (for example, after the pod is restarted, the domain name is still doris-be-cluster1-0-doris-be-cluster1.default.svc.cluster.local, but the IP may change from 172.16.0.9 to 172.16.0.10)
    enable_fqdn_mode = true
    LOG_DIR = ${DORIS_HOME}/log
    sys_log_level = INFO
    http_port = 8030
    rpc_port = 9020
    query_port = 9030
    edit_log_port = 9010
    #Doris needs to generate the log4j configuration file according to the fe.yml configuration information, which is written in the same directory as fe.yml by default, but the config we mount is readonly, so specify this configuration to write the log4j file to another location
    custom_config_dir = /opt/apache-doris/
    #when set to false, the backend will not be dropped and remaining in DECOMMISSION state
    drop_backend_after_decommission = false
    # 修改点10: 增加元数据、java等配置
    mysql_service_nio_enabled = true
    JAVA_OPTS = "-Xmx8192m -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$DATE"
    JAVA_OPTS_FOR_JDK_9 = "-Xmx8192m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParalle=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$DATE:time"
    meta_dir = /opt/apache-doris/fe/doris-meta
    #metadata_failure_recovery = true
---
# 修改点11: 增加kube config的configmap(此处可不修改,直接使用本地节点配置)
#注释掉这段,因为使用了serviceaccount,所以访问k8s的时候直接使用serviceaccount的,
#不需要配置.kube/config
#apiVersion: v1
#kind: ConfigMap
#metadata:
#  name: kube-conf
#data:
#  config: |
    # 此处内容复制配置文件内容,如果server地址指向的为127.0.0.1,注意修改为实际节点IP
    # 一般为 /root/.kube/config 内容, rancher安装的rke2环境默认为/etc/rancher/rke2/rke2.yaml
 

doris能够正常访问,但是用代码跑的时候提示No doris FE is available

然后你通过网页登录没问题,通过postman请求也没有问题

需要定位问题,把doris的源码下载下来,找到RestService这个类

在这一行打上断点

发现是301重定向了。因为我们使用了APISIX作为边缘网关来跳转,这个时候需要再APISIX里面把https去掉就可以。

有问题加群聊:

Logo

K8S/Kubernetes社区为您提供最前沿的新闻资讯和知识内容

更多推荐