nginx安装配置

1、通过docker-compose安装nginx,1.19的镜像支持stream模块，用来实现四层协议的转发、代理或者负载均衡，这边用来转发到k8s的nginx-ingress。

docker-compose.yaml

version: '3'
services:
  redis:
    image: "nginx:1.19"
    hostname: nginx
    container_name: nginx
    ports:
      - 80:80
      - 443:443
    volumes:
      - ./etc/nginx/nginx.conf:/etc/nginx/nginx.conf
    environment:
      - TZ=Asia/Shanghai
    restart: always

nginx.conf

error_log stderr notice;

worker_processes 2;
worker_rlimit_nofile 130048;
worker_shutdown_timeout 10s;

events {
  multi_accept on;
  use epoll;
  worker_connections 16384;
}

stream {
  upstream nginx_ingress {
    least_conn;
    server 172.21.74.5:80 max_fails=3 fail_timeout=5s;
    server 172.21.74.6:80 max_fails=3 fail_timeout=5s;
    server 172.21.74.7:89 max_fails=3 fail_timeout=5s;
  }
  upstream nginx_ingress_ssl {
    least_conn;
    server 172.21.74.5:443 max_fails=3 fail_timeout=5s;
    server 172.21.74.6:443 max_fails=3 fail_timeout=5s;
    server 172.21.74.7:443 max_fails=3 fail_timeout=5s;
  }
  upstream k8s {
    least_conn;
    server 172.21.74.2:6443 max_fails=3 fail_timeout=5s;
    server 172.21.74.3:6443 max_fails=3 fail_timeout=5s;
    server 172.21.74.4:6443 max_fails=3 fail_timeout=5s;
  }

  server {
    listen        80;
    proxy_pass    nginx_ingress;
    proxy_timeout 10m;
    proxy_connect_timeout 1s;
  }
  server {
    listen        443;
    proxy_pass    nginx_ingress_ssl;
    proxy_timeout 10m;
    proxy_connect_timeout 1s;
  }
  server {
    listen        6443;
    proxy_pass    k8s;
    proxy_timeout 10m;
    proxy_connect_timeout 1s;
  }
}

http {
  aio threads;
  aio_write on;
  tcp_nopush on;
  tcp_nodelay on;

  keepalive_timeout 5m;
  keepalive_requests 100;
  reset_timedout_connection on;
  server_tokens off;
  autoindex off;

  server {
    listen 18081;
    location /healthz {
      access_log off;
      return 200;
    }
    location /stub_status {
      stub_status on;
      access_log off;
    }
  }
}

安装keepalived
1、直接通过yum安装

yum -y install keepalived

2、配置

master

cat <<EOF > /etc/keepalived/keepalived.conf
#检测脚本
vrrp_script chk_http_port {
    script "/data/services/nginx/nginx.sh" #心跳执行的脚本，检测nginx是否启动
    interval 2                          #（检测脚本执行的间隔，单位是秒）
    weight 2                            #权重
}
#vrrp 实例定义部分
vrrp_instance VI_1 {
    state MASTER            # 指定keepalived的角色，MASTER为主，BACKUP为备
    interface ens160         # 当前进行vrrp通讯的网络接口卡(当前centos的网卡) 用ifconfig查看你具体的网卡
    virtual_router_id 66    # 虚拟路由编号，主从要一直
    priority 100            # 优先级，数值越大，获取处理请求的优先级越高
    advert_int 1            # 检查间隔，默认为1s(vrrp组播周期秒数)
    #授权访问
    authentication {
        auth_type PASS #设置验证类型和密码，MASTER和BACKUP必须使用相同的密码才能正常通信
        auth_pass 123456
    }
    track_script {
        chk_http_port            #（调用检测脚本）
    }
    virtual_ipaddress {
        172.21.74.20            # 定义虚拟ip(VIP)，可多设，每行一个
    }
}
EOF

backup

cat <<EOF > /etc/keepalived/keepalived.conf
#检测脚本
vrrp_script chk_http_port {
    script "/data/services/nginx/nginx.sh" #心跳执行的脚本，检测nginx是否启动
    interval 2                          #（检测脚本执行的间隔，单位是秒）
    weight 2                            #权重
}
#vrrp 实例定义部分
vrrp_instance VI_1 {
    state BACKUP            # 指定keepalived的角色，MASTER为主，BACKUP为备
    interface ens160         # 当前进行vrrp通讯的网络接口卡(当前centos的网卡) 用ifconfig查看你具体的网卡
    virtual_router_id 66    # 虚拟路由编号，主从要一致
    priority 99            # 优先级，数值越大，获取处理请求的优先级越高
    advert_int 1            # 检查间隔，默认为1s(vrrp组播周期秒数)
    #授权访问
    authentication {
        auth_type PASS #设置验证类型和密码，MASTER和BACKUP必须使用相同的密码才能正常通信
        auth_pass 123456
    }
    track_script {
        chk_http_port            #（调用检测脚本）
    }
    virtual_ipaddress {
        172.21.74.20            # 定义虚拟ip(VIP)，可多设，每行一个
    }
}
EOF
3、nginx检测脚本

nginx.sh

#!/bin/bash
#检测nginx是否启动了
A=`ps -C nginx --no-header |wc -l`
if [ $A -eq 0 ];then    #如果nginx没有启动就启动nginx
      docker start nginx                #重启nginx
      if [ `ps -C nginx --no-header |wc -l` -eq 0 ];then    #nginx重启失败，则停掉keepalived服务，进行VIP转移
              killall keepalived
      fi
fi

 
4、启动

systemctl start keepalived.service 
systemctl enable keepalived.service