k8s之资源调度的方式

1. 容器资源的限制

// 表示最多使用多少cpu和内存资源

  • resources.limits.cpu
  • resources.limits.memory

// 表示最少使用的CPU和内存资源

  • resources.requests.cpu
  • resources.requests.memory
[root@master ~]# kubectl describe nodes node1.example.com  //节点的信息
省略N行
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  Resource           Requests   Limits
  --------           --------   ------
  cpu                100m (5%)  100m (5%)  //cpu使用率
  memory             50Mi (1%)  50Mi (1%)  //内存使用情况
  ephemeral-storage  0 (0%)     0 (0%)
  hugepages-1Gi      0 (0%)     0 (0%)
  hugepages-2Mi      0 (0%)     0 (0%)


[root@master test]# cat test.yml 
apiVersion: v1
kind: Pod
metadata:
  name: test
spec:
  containers:
  - name: b1
    image: busybox
    command: ["/bin/sh","-c","sleep 9000"]
    resources:
      requests:
        memory: 100Mi
        cpu: 0.01
    env:
    - name: HN
      valueFrom:
        fieldRef:
          fieldPath: status.podIPs

[root@master test]# kubectl apply -f test.yml  //创建一个pod进行测试
pod/test created

[root@master test]# kubectl get pod -o wide  //运行再node1上
NAME   READY   STATUS    RESTARTS   AGE   IP            NODE                NOMINATED NODE   READINESS GATES
test   1/1     Running   0          57s   10.244.1.60   node1.example.com   <none>           <none>

[root@master haproxy]# kubectl describe nodes node1.example.com
  Resource           Requests    Limits
  --------           --------    ------
  cpu                110m (5%)   100m (5%)  //cpu变成了110，这是再原来的基础上增加的，原来是100，而我们设置requests为0.01就是增加十，因为默认100才能让node1运行，所以设置最少0.01就是在此基础上加10
  memory             150Mi (5%)  50Mi (1%)// 内存变为150，同理
  ephemeral-storage  0 (0%)      0 (0%)
  hugepages-1Gi      0 (0%)      0 (0%)
  hugepages-2Mi      0 (0%)      0 (0%)


[root@master test]# cat test.yml 
---
apiVersion: v1
kind: Pod
metadata:
  name: test
spec:
  containers:
  - name: b1
    image: busybox
    command: ["/bin/sh","-c","sleep 9000"]
    resources:
      requests:
        memory: 30Mi
        cpu: 0.001
      limits:
        memory: 40Mi
        cpu: 0.01
    env:
    - name: HN
      valueFrom:
        fieldRef:
          fieldPath: status.podIPs

2. nodeSelector节点选择器

nodeSelector：用于将Pod调度到匹配Label的Node上，如果没有匹配的标签会调度失败。
作用：

  • 约束Pod到特定的节点运行
  • 完全匹配节点标签
    应用场景：
  • 专用节点：根据业务线将Node分组管理
  • 配备特殊硬件：部分Node配有SSD硬盘、GPU
// 查看所有节点的标签
[root@master test]# kubectl get nodes --show-labels
NAME                 STATUS   ROLES                  AGE    VERSION   LABELS
master.example.com   Ready    control-plane,master   6d3h   v1.20.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=master.example.com,kubernetes.io/os=linux,node-role.kubernetes.io/control-plane=,node-role.kubernetes.io/master=
node1.example.com    Ready    <none>                 6d2h   v1.20.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=node1.example.com,kubernetes.io/os=linux
node2.example.com    Ready    <none>                 6d2h   v1.20.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=node2.example.com,kubernetes.io/os=linux

// 线运行一个pod，查看

[root@master test]# kubectl apply -f test.yml 
pod/test created
[root@master test]# cat test.yml 
---
apiVersion: v1
kind: Pod
metadata:
  name: test
spec:
  containers:
  - name: b1
    image: busybox
    command: ["/bin/sh","-c","sleep 9000"]
    env:
    - name: HN
      valueFrom:
        fieldRef:
          fieldPath: status.podIPs


[root@master haproxy]# kubectl get pod -o wide
NAME                     READY   STATUS    RESTARTS   AGE   IP            NODE                NOMINATED NODE   READINESS GATES
test                     1/1     Running   0          23m   10.244.1.61   node1.example.com   <none>           <none>  //运行在node1
web01-6cd64b79f6-gfkjk   1/1     Running   0          15s   10.244.2.51   node2.example.com   <none>           <none>

[root@master test]# cat test.yml  
---
apiVersion: v1
kind: Pod
metadata:
  name: test
spec:
  containers:
  - name: b1
    image: busybox
    command: ["/bin/sh","-c","sleep 9000"]
    env:
    - name: HN
      valueFrom:
        fieldRef:
          fieldPath: status.podIPs
  nodeSelector:
    disktype: ssd //选择运行在有ssd的类型的标签的节点上

[root@master test]# kubectl label nodes node2.example.com disktype=ssd  //给node2打上此标签
node/node2.example.com labeled

[root@master test]# kubectl get nodes node2.example.com --show-labels  //查看标签
NAME                STATUS   ROLES    AGE    VERSION   LABELS
node2.example.com   Ready    <none>   6d3h   v1.20.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,disktype=ssd,kubernetes.io/arch=amd64,kubernetes.io/hostname=node2.example.com,kubernetes.io/os=linux

[root@master ~]# kubectl get pod -o wide  //此时只有一个
NAME                     READY   STATUS    RESTARTS   AGE   IP            NODE                NOMINATED NODE   READINESS GATES
web01-6cd64b79f6-gfkjk   1/1     Running   0          15m   10.244.2.51   node2.example.com   <none>           <none>

[root@master test]# kubectl apply -f test.yml 
pod/test created

[root@master ~]# kubectl get pod -o wide  // 发现都运行在node2上
NAME                     READY   STATUS    RESTARTS   AGE   IP            NODE                NOMINATED NODE   READINESS GATES
test                     1/1     Running   0          20s   10.244.2.52   node2.example.com   <none>           <none>
web01-6cd64b79f6-gfkjk   1/1     Running   0          16m   10.244.2.51   node2.example.com   <none>           <none>

[root@master test]# kubectl label nodes node2.example.com disktype-  //删除标签的方式

[root@master test]# kubectl apply -f test.yml  //再次创建，因为没有标签所以会处于等待状态
pod/test created

[root@master test]# kubectl get nodes node2.example.com --show-labels
NAME                STATUS   ROLES    AGE    VERSION   LABELS
node2.example.com   Ready    <none>   6d3h   v1.20.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=node2.example.com,kubernetes.io/os=linux

[root@master test]# kubectl get pod
NAME                     READY   STATUS    RESTARTS   AGE
test                     0/1     Pending   0          5s
web01-6cd64b79f6-gfkjk   1/1     Running   0          40m

[root@master test]# kubectl label nodes node2.example.com disktype=ssd //打上标签之后运行到node2
node/node2.example.com labeled  

[root@master test]# kubectl get pod -o wide
NAME                     READY   STATUS    RESTARTS   AGE     IP            NODE                NOMINATED NODE   READINESS GATES
test                     1/1     Running   0          2m51s   10.244.2.54   node2.example.com   <none>           <none>
web01-6cd64b79f6-gfkjk   1/1     Running   0          43m     10.244.2.51   node2.example.com   <none>           <none>

3. nodeAffinity节点亲和性

nodeAffinity：节点亲和性，与nodeSeletor作用一样，但相比更灵活，满足更多条件，如：

  • 匹配有更多的逻辑组合，不只是字符串的完全相等
  • 调度分为软策略和硬策略，而不是硬性要求
    • 硬(required)：必须满足
    • 软(preferred)：尝试满足，但是不保证
      操作符：In(key in value //表示key包含value)、Noth、Exists、DoesNotExist()、Gt(大于)、Lt(小于)
[root@master test]# cat test.yml //使用硬策略必须满足
---
apiVersion: v1
kind: Pod
metadata:
  name: test
spec:
  containers:
  - name: b1
    image: busybox
    command: ["/bin/sh","-c","sleep 9000"]
    env:
    - name: HN
      valueFrom:
        fieldRef:
          fieldPath: status.podIPs
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - key: disktype
            operator: In
            values: 
            - ssd

[root@master test]# kubectl get nodes --show-labels  //都没有distype=ssd的标签
NAME                 STATUS   ROLES                  AGE    VERSION   LABELS
master.example.com   Ready    control-plane,master   6d5h   v1.20.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=master.example.com,kubernetes.io/os=linux,node-role.kubernetes.io/control-plane=,node-role.kubernetes.io/master=
node1.example.com    Ready    <none>                 6d4h   v1.20.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=node1.example.com,kubernetes.io/os=linux
node2.example.com    Ready    <none>                 6d4h   v1.20.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=node2.example.com,kubernetes.io/os=linux

[root@master test]# kubectl apply -f test.yml   //所以没有在任何节点运行
pod/test created
[root@master test]# kubectl get pod -o wide
NAME                     READY   STATUS    RESTARTS   AGE   IP            NODE                NOMINATED NODE   READINESS GATES
test                     0/1     Pending   0          6s    <none>        <none>              <none>           <none>
web01-6cd64b79f6-gfkjk   1/1     Running   0          73m   10.244.2.51   node2.example.com   <none>           <none>

[root@master test]# cat test.yml //加一条软策略
---
apiVersion: v1
kind: Pod
metadata:
  name: test
spec:
  containers:
  - name: b1
    image: busybox
    command: ["/bin/sh","-c","sleep 9000"]
    env:
    - name: HN
      valueFrom:
        fieldRef:
          fieldPath: status.podIPs
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - key: disktype
            operator: In
            values: 
            - ssd
      preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 3
        preference:
          matchExpressions:
          - key: gpu
            operator: In
            values: 
          - nvdia

// disktype=ssd为硬策略
[root@master test]# kubectl label nodes node1.example.com disktype=ssd gpu=nvdia  //给node1打两个标签
node/node1.example.com labeled

[root@master test]# kubectl label nodes node2.example.com disktype=sshd //给node2打上一个标签
node/node2.example.com labeled

[root@master test]# kubectl apply -f test.yml //此时他将会运行到node1上
pod/test created

[root@master test]# kubectl label nodes node1.example.com gpu-  //删除node1上的gpu
node/node1.example.com labeled

[root@master test]# kubectl label nodes node2.example.com gpu=nvdia //添加一个标签
node/node2.example.com labeled

[root@master test]# kubectl apply -f test.yml  //此时pod将运行在node2上
pod/test created

[root@master test]# kubectl get pod -o wide
NAME                     READY   STATUS    RESTARTS   AGE   IP            NODE                NOMINATED NODE   READINESS GATES
test                     1/1     Running   0          28s   10.244.2.55   node2.example.com   <none>           <none>
web01-6cd64b79f6-gfkjk   1/1     Running   0          93m   10.244.2.51   node2.example.com   <none>           <none>

4. taint污点&tolrations污点容忍

taint：避免pod调度到特定的Node上
tolerations：允许Pod调度到持有taints的Node上
应用场景：

  • 专用节点：根据业务线将Node分组管理，希望在默认情况下不调度该节点，只有配置了污点容忍才允许分配
  • 配备特殊硬件：部分Node配有SSD硬盘、gpu，希望在默认情况下不调度该节点，只有配置了污点容忍才允许分配
  • 基于taint的驱逐

格式：kubectl taint node[node]key=value:[effect]
例如：kubectl taint node k8s-node1 gpu=yes：NoSchedule
验证：kubectl describe node k8s-node1 | grep Taint

其中[effect]可取的值
NoSchedule：一定不能被调度
PreferNcSohedule：尽量不要调度，非必须配置容忍
NoExecute：不仅不会调度，还会驱逐Node上已有的Pod

去掉污点：
kubectl taint node [node] key:[effect]-

[root@master test]# kubectl describe node node1.example.com | grep -i taint //查看节点的方式
Taints:             <none>

[root@master test]# kubectl taint node node1.example.com node1:NoSchedule //给node1添加一个污点
node/node1.example.com tainted

[root@master test]# kubectl describe node node1.example.com | grep -i taint 
Taints:             node1:NoSchedule

[root@master test]# kubectl get pod -o wide  // 在没污点前运行在node2
NAME                     READY   STATUS    RESTARTS   AGE    IP            NODE                NOMINATED NODE   READINESS GATES
test                     1/1     Running   0          19s    10.244.2.56   node2.example.com   <none>           <none>
web01-6cd64b79f6-gfkjk   1/1     Running   0          125m   10.244.2.51   node2.example.com   <none>           <none>

[root@master test]# kubectl taint node node2.example.com node2:NoSchedule //给node2加污点
node/node2.example.com tainted

[root@master test]# kubectl describe node node2.example.com | grep -i taint
Taints:             node1:NoSchedule

[root@master test]# kubectl apply -f test.yml 
pod/test created

[root@master test]# kubectl get pod -o wide  //发现在node1上去运行了，因为node2上有污点了
NAME                     READY   STATUS    RESTARTS   AGE    IP            NODE                NOMINATED NODE   READINESS GATES
test                     1/1     Running   0          17s    10.244.1.63   node1.example.com   <none>           <none>
web01-6cd64b79f6-gfkjk   1/1     Running   0          129m   10.244.2.51   node2.example.com   <none>           <none>

[root@master test]# kubectl taint node node2.example.coomnode1:NoSchedule-  //删除node2上的污点
node/node2.example.com untainted

[root@master test]# kubectl describe node node2.example.com | grep -i taint
Taints:             <none>

[root@master test]# kubectl apply -f test.yml //再次创建
pod/test created

[root@master test]# kubectl get pod -o wide  //发现又再node2上运行了
NAME                     READY   STATUS    RESTARTS   AGE    IP            NODE                NOMINATED NODE   READINESS GATES
test                     1/1     Running   0          29s    10.244.2.57   node2.example.com   <none>           <none>
web01-6cd64b79f6-gfkjk   1/1     Running   0          134m   10.244.2.51   node2.example.com   <none>           <none>


[root@master test]# kubectl taint node node2.example.com node2:PreferNoSchedule  //这个污点表示尽量不再node2上运行，但是可以在node2运行
node/node2.example.com tainted

[root@master test]# kubectl apply -f test.yml 
pod/test created

[root@master test]# kubectl get pod -o wide  //发现他运行再node1上
NAME                     READY   STATUS    RESTARTS   AGE    IP            NODE                NOMINATED NODE   READINESS GATES
test                     1/1     Running   0          23s    10.244.1.64   node1.example.com   <none>           <none>
web01-6cd64b79f6-gfkjk   1/1     Running   0          139m   10.244.2.51   node2.example.com   <none>           <none>

[root@master test]# kubectl label nodes node2.example.com disktype=ssd // 在
node/node2.example.com labeled

[root@master test]# kubectl get pod -o wide  //没打标签前再node2上运行
NAME                     READY   STATUS    RESTARTS   AGE    IP            NODE                NOMINATED NODE   READINESS GATES
test                     1/1     Running   0          11m    10.244.2.58   node2.example.com   <none>           <none>
web01-6cd64b79f6-gfkjk   1/1     Running   0          158m   10.244.2.51   node2.example.com   <none>           <none>

[root@master test]# kubectl taint node node2.example.com node2:NoExecute  //打上标签，不仅不会再node2上运行还会驱逐到node1上运行
node/node2.example.com tainted

[root@master test]# kubectl get pod -o wide  //因为我们定义我是自主式pod，再退出之后就会被删除，所以这里没有
NAME                     READY   STATUS    RESTARTS   AGE     IP            NODE                NOMINATED NODE   READINESS GATES
web01-6cd64b79f6-v6gxd   1/1     Running   0          3m30s   10.244.1.65   node1.example.com   <none>           <none>
# 容器 # linux # docker # kubernetes
Logo
K8S/Kubernetes

K8S/Kubernetes社区为您提供最前沿的新闻资讯和知识内容

更多推荐

【深度】阿里巴巴万级规模 K8s 集群全局高可用体系之美

作者 |  韩堂、柘远、沉醉来源 | 阿里巴巴云原生公众号​前言台湾作家林清玄在接受记者采访的时候，如此评价自己 30 多年写作生涯：“第一个十年我才华横溢，‘贼光闪现’，令周边黯然失色；第二个十年，我终于‘宝光现形’，不再去抢风头，反而与身边的美丽相得益彰；进入第三个十年，繁华落尽见真醇，我进入了‘醇光初现’的阶段，真正体味到了境界之美”。​长夜有穷，真水无香。领略过了 K8s“身在江

avatar K8S/Kubernetes

如何基于 K8s 构建下一代 DevOps 平台？

作者 | 孙健波（天元）导读：当前云原生 DevOps 体系现状如何？面临哪些挑战？如何通过 OAM 解决云原生 DevOps 场景下的诸多问题？云原生开发应用模型 OAM(Open Application Model) 社区核心成员孙健波将为大家一一解答，并分享如何基于 OAM 和 Kubernetes 打造无限能力的下一代 DevOps 平台。什么是 DevOps？为什么基于 Kub

avatar K8S/Kubernetes

k8s 火了！

2020，上云之年，产品云端化成为一种趋势。在一线城市，很多公司都已经构建了自己的私有云环境，比如阿里云、网易云、华为云等。而Kubernetes 作为基于容器编排领域的王者，具备扩展...

avatar K8S/Kubernetes