主机	ip
harbor	172.25.254.11
master	172.25.254.12
node1	172.25.254.13
node2	172.25.254.14

在/etc/hosts解析：

172.25.254.11      harbor  reg.timinglee.org
172.25.254.12     master
172.25.254.13     node1

172.25.254.14      node2

1.部署docker本地仓库

[root@harbor ~]# cat > /etc/yum.repos.d/docker.repo <<EOF
[docker]
name = docker
baseurl = https://mirrors.aliyun.com/docker-ce/linux/rhel/9.6/x86_64/stable/
gpgcheck = 0
EOF

[root@harbor ~]# cat > /etc/yum.repos.d/docker.repo <<EOF
[docker]
name = docker
baseurl = https://mirrors.aliyun.com/docker-ce/linux/rhel/9.6/x86_64/stable/
gpgcheck = 0
EOF

[root@harbor ~]# dnf install httpd createrepo-y
[root@harbor ~]# mkdir  /var/www/html/docker/ -p
[root@harbor ~]# vim /etc/httpd/conf/httpd.conf
Listen 4444
[root@harbor ~]# systemctl enable --now httpd

[root@harbor ~]# dnf  install docker-ce -downloadonly  --destdir /mnt/ -y
[root@harbor ~]# mv /mnt/*.rpm /var/www/html/docker/
[root@harbor ~]# createrepo -v /var/www/html/docker/

[root@harbor ~]# cat > /etc/yum.repos.d/docker.repo <<EOF
[docker]
name = docker
baseurl = http://172.25.254.11:4444/docker
gpgcheck = 0
EOF

[root@harbor ~]# dnf install docker-ce -y

2.安装docker在harbor仓库节点

[root@harbor ~]# dnf install docker-ce -y

[root@harbor ~]# echo br_netfilter > /etc/modules-load.d/docker_mod.conf
[root@harbor ~]# modprobe -a br_netfilter
[root@harbor ~]# vim /etc/sysctl.d/docker.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1

[root@harbor ~]# systemctl restart  systemd-modules-load.service
[root@harbor ~]# sysctl  --system

[root@harbor ~]# vim /lib/systemd/system/docker.service
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --iptables=true
[root@harbor ~]# systemctl daemon-reload
[root@harbor ~]# systemctl enable --now docker

2.生成key

[root@harbor ~]# mkdir /data/certs -p
[root@harbor ~]# mkdir /data/certs -p
[root@harbor ~]# openssl req -newkey  rsa:4096 \
-nodes -sha256 -keyout /data/certs/timinglee.org.key \
-addext "subjectAltName = DNS:reg.timinglee.org" \
-x509 -days 365 -out /data/certs/timinglee.org.crt

You are about to be asked to enter information that will be incorporated
into your certificate request.
What you are about to enter is what is called a Distinguished Name or a DN.
There are quite a few fields but you can leave some blank
For some fields there will be a default value,
If you enter '.', the field will be left blank.
-----
Country Name (2 letter code) [XX]:CN
State or Province Name (full name) []:Shannxi
Locality Name (eg, city) [Default City]:Xi'an
Organization Name (eg, company) [Default Company Ltd]:kubernetes
Organizational Unit Name (eg, section) []:harbor
Common Name (eg, your name or your server's hostname) []:reg.timinglee.org
Email Address []:admin@timinglee.org

3.编辑harbor配置文件

[root@harbor ~]# tar zxf  harbor-offline-installer-v2.5.4.tgz -C /opt/
[root@harbor ~]# cd /opt/harbor/
[root@harbor harbor]# ls
common.sh  harbor.v2.5.4.tar.gz  harbor.yml.tmpl  install.sh  LICENSE  prepare
[root@harbor harbor]# cp harbor.yml.tmpl harbor.yml
[root@harbor harbor]# vim harbor.yml

一.安装ansible

[root@harbor ~]# vim /etc/yum.repos.d/epel.repo
[epel]
name = epel
baseurl = https://mirrors.aliyun.com/epel/9/Everything/x86_64/
gpgcheck = 0

[root@harbor ~]# dnf install ansible-core.x86_64 ansible.noarch -y

二.配置所需用户

三.生成ansible的临时配置文件并建立运行环境

vim inventory

vim ansible.cfg

做免密登陆：

#先切devops
su - devops
#生成devops自己的密钥
ssh-keygen -t ed25519
#全部回车，不要输入密码
#推送公钥到被控机root账号
ssh-copy-id root@172.25.254.12
ssh-copy-id root@172.25.254.13
ssh-copy-id root@172.25.254.14

[devops@harbor ansible]$ ansible all -m ping

在所有被控服务器上：

1. 创建普通用户 devops
2. 设置密码：lee
3. 赋予 devops 免密 sudo 权限，可以 sudo 变成 root，不用输密码

[devops@harbor ansible]$ ansible all -m user -a "name=devops state=present"
[devops@harbor ansible]$ ansible all -m shell -a 'echo "lee" | passwd --stdin devops'
[devops@harbor ansible]$ ansible all -m lineinfile -a 'path=/etc/sudoers line="devops    ALL=(ALL)    NOPASSWD: ALL" insertafter=EOF'

四.生成ansible的主配置文件并测试

五.利用ansible剧本部署docker

vim kubernetes-devops.yml

- name: devops k8s from ansible
  hosts: all
  tasks:
  - name: setup docker repo
    yum_repository:
      name: docker
      description: docker
      baseurl: http://172.25.254.11:4444/docker
      file: docker
      gpgcheck: no

  - name: install docker
    dnf:
      name: docker-ce
      state: present

  - name: setup docker.service
    replace:
      path: /lib/systemd/system/docker.service
      regexp: "ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock"
      replace: "ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock  --iptables=true"

  - name: setup docker registry
    copy:
      content: |
        {
          "registry-mirrors":["https://reg.timinglee.org"]
        }
      dest: /etc/docker/daemon.json

  - name: setup docker certs
    file:
      path: /etc/docker/certs.d/reg.timinglee.org
      state: directory

  - name: cp certs file
    copy:
      src: "{{item.src}}"
      dest: "{{item.dest}}"
    loop:
      - src: /etc/docker/certs.d/reg.timinglee.org/ca.crt
        dest: /etc/docker/certs.d/reg.timinglee.org/ca.crt
      - src: /etc/docker/certs.d/reg.timinglee.org/ca.crt
        dest: /etc/pki/ca-trust/source/anchors/ca.crt

  - name: load module
    copy:
      dest: "{{item.dest}}"
      content: "{{item.value}}"
    loop:
      - dest: /etc/modules-load.d/docker_mod.conf
        value: br_netfilter
      - dest: /etc/sysctl.d/docker.conf
        value: |
          net.bridge.bridge-nf-call-iptables = 1
          net.bridge.bridge-nf-call-ip6tables = 1
          net.ipv4.ip_forward = 1

  - name: update ca cert
    shell: "update-ca-trust extract;sysctl --system"

  - name: start services
    service:
      name: "{{item}}"
      state: restarted
      enabled: yes
    loop:
      - docker
      - systemd-modules-load.service

六.利用ansible安装cri-dockerd

[devops@harbor ansible]$ tar zxf cri-dockerd-0.4.4.amd64.tgz
[devops@harbor ansible]$ ls
ansible.cfg  cri-dockerd  cri-dockerd-0.4.4.amd64.tgz  inventory  kubernetes-devops.yml
[devops@harbor ansible]$ cd cri-dockerd/
[devops@harbor cri-dockerd]$ ls
cri-dockerd  cri-docker.service  cri-docker.socket
[devops@harbor cri-dockerd]$ vim cri-docker.service
[Service]
Type=notify
ExecStart=/usr/bin/cri-dockerd --container-runtime-endpoint fd:// --network-plugin=cni --pod-infra-container-image=reg.timinglee.org/k8s/pause:3.10.1 
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always

[devops@harbor ansible]$ vim kubernetes-devops.yml

- name: devops k8s from ansible
  hosts: all
  tasks:
  - name: setup docker repo
    yum_repository:
      name: docker
      description: docker
      baseurl: http://172.25.254.11:4444/docker
      file: docker
      gpgcheck: no
  - name: install docker
    dnf:
      name: docker-ce
      state: present
  - name: setup docker.service
    replace:
      path: /lib/systemd/system/docker.service
      regexp: "ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock"
      replace: "ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock  --iptables=true"
  - name: setup docker registry
    copy:
      content: |
        {
          "registry-mirrors":["https://172.25.254.11"]
        }
      dest: /etc/docker/daemon.json
  - name: setup docker certs
    file:
      path: /etc/docker/certs.d/172.25.254.11
      state: directory
  - name: cp certs file
    copy:
      src: "{{item.src}}"
      dest: "{{item.dest}}"
    loop:
      - src: ./ca.crt
        dest: /etc/docker/certs.d/172.25.254.11/ca.crt
      - src: ./ca.crt
        dest: /etc/pki/ca-trust/source/anchors/ca.crt
########################################################
  - name: stup cri-dockerd
    copy:
      src: "{{item.src}}"
      dest: "{{item.dest}}"
      mode: '0755'
    loop:
      - src: ./cri-dockerd/cri-dockerd
        dest: /usr/bin/cri-dockerd
      - src: "./cri-dockerd/cri-docker.service"
        dest: "/lib/systemd/system/cri-docker.service"
      - src: "./cri-dockerd/cri-docker.socket"
        dest: "/lib/systemd/system/cri-docker.socket"
  - name: update ca cert
    shell: "update-ca-trust extract;sysctl --system;systemctl daemon-reload"
  - name: start services
    service:
      name: "{{item}}"
      state: restarted
      enabled: yes
    loop:
      - docker
      - systemd-modules-load.service
      - cri-docker

七.利用ansible部署k8s软件

- name: install kubernetes from ansible
  hosts: all
  tasks:
  - name: setup k8s repo
    yum_repository:
      name: kubernetes
      description: kubernetes
      baseurl: https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.35/rpm
      file: kubernetes
      gpgcheck: no

  - name: install software in master
    dnf:
      name: "{{item}}"
      state: present
    loop:
      - kubeadm-1.35.7-150500.1.1
      - kubelet-1.35.7-150500.1.1
      - kubectl-1.35.7-150500.1.1
    when: inventory_hostname == "172.25.254.12"

  - name: install software in work node
    dnf:
      name: "{{item}}"
      state: present
    loop:
      - kubeadm-1.35.7-150500.1.1
      - kubelet-1.35.7-150500.1.1
    when: inventory_hostname == "172.25.254.13" or inventory_hostname == "172.25.254.14"
  
  - name: setup kubectl kubeadm complication
    lineinfile:
      path: /root/.bashrc
      line: |
        source <(kubectl completion bash)
        source <(kubeadm completion bash)
    when: inventory_hostname == "172.25.254.12"

  - name: start kubelet
    service:
      name: kubelet
      state: started
      enabled: yes

#让补齐功能生效