CCE使用node节点使用VIP

背景：想在节点上使用VIP，将nodeport服务做到高可用。启动VIP后发现访问失败

部署

! Configuration File for keepalived
global_defs {
router_id master-node
}

vrrp_instance VI_1 {
     state BACKUP
     interface eth0
     mcast_src_ip 10.1.0.60
     virtual_router_id 51
     priority 100
     advert_int 1
     authentication {
               auth_type PASS
               auth_pass 1111
               }
     unicast_src_ip 10.1.0.60
     virtual_ipaddress {
                    10.1.0.88/24
                    }
}

! Configuration File for keepalived
global_defs {
router_id master-node
}

vrrp_instance VI_1 {
                state MASTER
                interface eth0
                mcast_src_ip 10.1.0.175
                virtual_router_id 51
                priority 101
                advert_int 1
                authentication {
                              auth_type PASS
                              auth_pass 1111
                             }
                unicast_src_ip 10.1.0.175
                virtual_ipaddress {
                             10.1.0.88
                             }
}

[root@cce-node3-dev ~]# ip a show eth0
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether fa:16:3e:46:43:df brd ff:ff:ff:ff:ff:ff
    inet 10.1.0.60/24 brd 10.3.0.255 scope global dynamic noprefixroute eth0
       valid_lft 100122990sec preferred_lft 100122990sec
    inet 10.1.0.88/32 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:fe46:43df/64 scope link 
       valid_lft forever preferred_lft forever

启动服务

-- 30007集群nodeport服务，8080 NGINX，任意一个节点启动

# 测试结果如下
-- RIP
10.1.0.60:80 OK
10.1.0.60:30007 OK

-- VIP
10.1.0.88:80 OK
10.1.0.88:30007 NOT OK

检查发现node节点并没有监听30007端口，而是通过ipvs添加了被负载均衡的虚拟地址。

[root@recovery-test-28141 ~]# ipvsadm -Ln | grep 30007 -A3
TCP  10.1.0.60:30007   rr
  -> 10.244.0.89:80               Masq        1      0          0
  -> 10.244.0.128:80              Masq        1      0          0
TCP  10.1.0.60:30008   rr
[root@recovery-test-28141 ~]# netstat -npl | grep 30007

解决方法

方法一、使用pod.spec.hostNetwork

不要创建service，使用宿主机IP，节点可以看到端口监听

apiVersion: apps/v1
kind: Deployment
metadata:
  namespace: os
  labels:
    app: nginx-hostnetwork
  name: nginx-hostbnetwork
spec:
  replicas: 0
  selector:
    matchLabels:
      app: nginx-hostnetwork
  template:
    metadata:
      labels:
        app: nginx-hostnetwork
    spec:
      #nodeSelector:      # node节点选择器
      #  role: master     # node节点标签(Label)
      imagePullSecrets:
        - name: default-secret
      dnsPolicy: ClusterFirst
      hostNetwork: true
      containers:
      - image: nginx:1.21.4
        imagePullPolicy: IfNotPresent
        name: nginx
        ports:
        - containerPort: 80
        - containerPort: 443

验证

[root@cce-node3-dev ~]# ss -antlp|grep 80
LISTEN 0      511          0.0.0.0:80         0.0.0.0:*    users:(("nginx",pid=3607811,fd=7),("nginx",pid=3607810,fd=7),("nginx",pid=3607809,fd=7),("nginx",pid=3607808,fd=7),("nginx",pid=3607807,fd=7),("nginx",pid=3607806,fd=7),("nginx",pid=3607805,fd=7),("nginx",pid=3607804,fd=7),("nginx",pid=3607758,fd=7))

[root@cce-node3-dev ~]# curl localhost:80
Welcome to nginx!
[root@cce-node3-dev ~]# curl 10.1.0.88
Welcome to nginx!

user@z5ok45akqx9338m-machine:~$ kubectl get pod -owide -nos  #使用的是宿主机IP
NAME                              READY   STATUS    RESTARTS   AGE     IP             NODE         NOMINATED 
nginx-hostport-67d944c568-kxgpk   1/1     Running   0          6m30s   10.1.0.156     10.3.0.156   <none> 
web-v1-bc48844d8-kccsb            2/2     Running   0          1d     10.244.0.228   10.3.0.251    <none>   

方法二使用 pod.spec.containers.ports.hostPort

不需要创建service，使用的是容器网段IP，宿主机看不到端口监听，但是会转发这个端口的访问

apiVersion: apps/v1
kind: Deployment
metadata:
  namespace: os
  labels:
    app: nginx-hostport
  name: nginx-hostport
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx-hostport
  template:
    metadata:
      labels:
        app: nginx-hostport
    spec:
      #nodeSelector:      # node节点选择器
      #  role: master     # node节点标签(Label)
      imagePullSecrets:
        - name: default-secret
      dnsPolicy: ClusterFirst
      containers:
      - image: nginx:1.21.4
        imagePullPolicy: IfNotPresent
        name: nginx
        ports:
        - containerPort: 80
          hostPort: 80  #会映射pod的开放端口，可调整
          name: http
        - containerPort: 443
          hostPort: 443
          name: https

user@z5ok45akqx9338m-machine:~$ kubectl get pod -owide -nos  #使用的是宿主机IP
NAME                              READY   STATUS    RESTARTS   AGE     IP             NODE         NOMINATED 
nginx-hostport-67d944c568-kxgpk   1/1     Running   0          1m30s   10.244.0.222   10.3.0.156   <none> 
web-v1-bc48844d8-kccsb            2/2     Running   0          1d     10.244.0.228   10.3.0.251    <none> 

[root@cce-node3-dev ~]# ss -antlp|grep 80   #没有
[root@cce-node3-dev ~]# curl 10.1.0.88
Welcome to nginx!
[root@cce-node3-dev ~]# curl 10.1.0.60
Welcome to nginx!