📁 目录结构（prepare/）

prepare/
├── files/               # 静态文件，直接复制到目标主机
│   └── sctp.conf        # 示例：一个配置文件，会被拷贝到远程机器
│
├── tasks/               # 执行任务的核心部分
│   ├── common.yml       # 共同任务（适用于所有系统）
│   ├── debian.yml       # Debian/Ubuntu 系统专用任务
│   ├── main.yml         # 主任务文件（默认执行入口）
│   ├── offline.yml      # 离线部署时使用的任务
│   ├── redhat.yml       # RHEL/CentOS/Fedora 专用任务
│   └── suse.yml         # SUSE Linux 专用任务
│
├── templates/           # 模板文件（Jinja2），根据变量动态生成
│   ├── 10-k8s-modules.conf.j2
│   ├── 30-k8s-ulimits.conf.j2
│   ├── 95-k8s-journald.conf.j2
│   └── 95-k8s-sysctl.conf.j2
│
└── vars/                # 角色变量定义（全局或平台相关）
    └── main.yml

---

📄 1.1 files/sctp.conf

# 将 SCTP 协议加入内核模块黑名单，防止意外加载
# 用于避免 Kubernetes 环境中不必要的内核模块干扰
install sctp /bin/true

---

📄 1.2 tasks/ 任务文件

📄 1.2.1 common.yml

# 公共系统初始化任务（适用于所有 Linux 发行版）

# 禁用 swap（Kubernetes 要求）
- name: 禁用系统 swap
  shell: "source /etc/profile; swapoff -a && sysctl -w vm.swappiness=0"
  ignore_errors: true

- name: 删除 fstab 中 swap 相关配置
  lineinfile:
    path: /etc/fstab
    regexp: 'swap'
    state: absent
    backup: 'yes'

# 加载 Kubernetes 所需的内核模块
- name: 加载内核模块
  modprobe:
    name: "{{ item }}"
    state: present
  loop:
    - br_netfilter
    - dm_snapshot
    - dm_thin_pool
    - ip_vs
    - ip_vs_rr
    - ip_vs_wrr
    - ip_vs_sh
    - nf_conntrack
  ignore_errors: true

# 尝试加载 nf_conntrack_ipv4（旧内核可能需要）
- name: 尝试加载 nf_conntrack_ipv4
  shell: 'modprobe nf_conntrack_ipv4 || echo "NoFound"'
  register: NF_CONNTRACK_IPV4

# 启用 systemd-modules-load 服务，确保模块开机自动加载
- name: 启用 systemd 自动加载模块服务
  service:
    name: systemd-modules-load
    enabled: yes

- name: 增加内核模块开机加载配置
  template:
    src: 10-k8s-modules.conf.j2
    dest: /etc/modules-load.d/10-k8s-modules.conf

# 设置 Kubernetes 所需的 sysctl 参数
- name: 设置系统参数
  template:
    src: 95-k8s-sysctl.conf.j2
    dest: /etc/sysctl.d/95-k8s-sysctl.conf

# 动态判断是否需要设置 fs.may_detach_mounts（某些内核版本需要）
- name: 查看是否需要设置 fs.may_detach_mounts
  stat:
    path: /proc/sys/fs/may_detach_mounts
    get_attributes: no
    get_checksum: no
    get_mime: no
  register: fs_may_detach_mounts
  ignore_errors: true

- name: 当需要时，设置 fs.may_detach_mounts 为 1
  sysctl:
    sysctl_file: "/etc/sysctl.d/95-k8s-sysctl.conf"
    name: fs.may_detach_mounts
    value: 1
    state: present
    reload: yes
  when: fs_may_detach_mounts.stat.exists | bool

# tcp_tw_recycle 在较新内核中已废弃，但旧系统可能仍存在
- name: 查看是否需要设置 net.ipv4.tcp_tw_recycle
  stat:
    path: /proc/sys/net/ipv4/tcp_tw_recycle
  register: tcp_tw_recycle
  ignore_errors: true

- name: 设置 tcp_tw_recycle 为 0（避免 NAT 环境问题）
  sysctl:
    sysctl_file: "/etc/sysctl.d/95-k8s-sysctl.conf"
    name: net.ipv4.tcp_tw_recycle
    value: 0
    state: present
    reload: yes
  when: tcp_tw_recycle.stat.exists | bool

- name: 生效系统参数
  shell: "source /etc/profile; sysctl -p /etc/sysctl.d/95-k8s-sysctl.conf"
  ignore_errors: true

# 设置系统级 ulimits（提升文件描述符等限制）
- name: 创建 systemd 配置目录
  file:
    path: /etc/systemd/system.conf.d
    state: directory

- name: 设置系统 ulimits
  template:
    src: 30-k8s-ulimits.conf.j2
    dest: /etc/systemd/system.conf.d/30-k8s-ulimits.conf

# 黑名单 SCTP 模块（安全加固）
- name: 把 SCTP 列入内核模块黑名单
  copy:
    src: sctp.conf
    dest: /etc/modprobe.d/sctp.conf

---

📄 1.2.2 debian.yml

# Debian/Ubuntu 系统专用初始化任务

# 更新 apt 缓存（仅在线模式）
- name: apt 更新缓存刷新
  apt:
    update_cache: yes
    cache_valid_time: 72000
  ignore_errors: true
  when: INSTALL_SOURCE != "offline"

# 移除 Ubuntu 默认安装的无用组件
- name: 删除 ubuntu 默认安装
  package:
    name:
      - ufw
      - lxd
      - lxcfs
      - lxc-common
    state: absent
  ignore_errors: true

# 安装 Kubernetes 所需的基础软件包（在线模式）
- name: 安装 ubuntu/debian 基础软件
  package:
    name:
      - bash-completion     # bash 补全
      - conntrack           # 连接跟踪清理
      - ipset               # IPVS 模式依赖
      - ipvsadm             # IPVS 管理工具
      - jq                  # JSON 处理
      - libseccomp2         # containerd 依赖
      - lvm2                # LVM 存储插件
      - nfs-common          # NFS 客户端
      - psmisc              # killall 命令
      - rsync               # 配置同步
      - socat               # 端口转发
    state: present
  when: INSTALL_SOURCE != "offline"

# 离线安装逻辑
- import_tasks: offline.yml
  when: INSTALL_SOURCE == "offline"

# 优化 journald 日志配置
- name: 准备 journal 日志相关目录
  file:
    path: "{{ item }}"
    state: directory
  loop:
    - "/etc/systemd/journald.conf.d"
    - "/var/log/journal"

- name: 优化设置 journal 日志
  template:
    src: 95-k8s-journald.conf.j2
    dest: /etc/systemd/journald.conf.d/95-k8s-journald.conf

- name: 重启 journald 服务
  service:
    name: systemd-journald
    state: restarted

---

📄 1.2.3 main.yml

# 主任务入口：根据系统类型调用对应子任务，并执行通用配置

- name: 检查是否已执行过 prepare_tasks（幂等性控制）
  stat:
    path: "/opt/kubeasz_prepare_tasks"
  register: p

- name: 已完成提示
  debug:
    msg: "节点已完成 prepare_tasks，跳过"
  when: p.stat.isreg is defined

- block:
    # 根据操作系统类型导入对应任务
    - import_tasks: debian.yml
      when: ansible_distribution_file_variety in ["Debian"]

    - import_tasks: redhat.yml
      when: ansible_distribution_file_variety in ["RedHat"] or ansible_distribution in ["CentOS"]

    - import_tasks: suse.yml
      when: ansible_distribution_file_variety in ["SUSE"]

    - import_tasks: redhat.yml
      when: ansible_distribution in ["Anolis OS", "openEuler"]

    # 执行通用任务
    - import_tasks: common.yml

    # 创建必要目录
    - name: prepare some dirs
      file:
        path: "{{ item }}"
        state: directory
      loop:
        - "{{ bin_dir }}"
        - "{{ ca_dir }}"
        - /root/.kube

    # 分发 kubeconfig（仅 master 和 node 节点）
    - name: 复制 kubectl kubeconfig
      copy:
        src: "{{ cluster_dir }}/kubectl.kubeconfig"
        dest: /root/.kube/config
      when: inventory_hostname in groups['kube_master'] or inventory_hostname in groups['kube_node']

    # 设置 PATH 环境变量
    - name: 写入环境变量 $PATH
      lineinfile:
        dest: ~/.bashrc
        state: present
        regexp: 'custom PATH'
        line: 'export PATH={{ bin_dir }}:$PATH  # generated by kubeasz:custom PATH'

    # 为 Ansible 控制机设置别名（方便调试）
    - name: ansible 控制端写入命令别名
      lineinfile:
        dest: ~/.bashrc
        state: present
        regexp: 'docker exec'
        line: "alias dk='docker exec -it kubeasz'  # generated by kubeasz"
      when: inventory_hostname == ansible_env.SSH_CLIENT.split(' ')[0]

    # 添加本地 registry 的 hosts 解析
    - name: 添加 local registry hosts 解析
      lineinfile:
        dest: /etc/hosts
        state: present
        regexp: 'easzlab.io.local'
        line: "{{ ansible_env.SSH_CLIENT.split(' ')[0] }}    easzlab.io.local"

    # 设置唯一 hostname（Calico 要求）
    - name: 设置节点 hostname
      hostname:
        name: "{{ K8S_NODENAME }}"
        use: systemd
      when: ENABLE_SETTING_HOSTNAME | bool

    # 在第一个 master 节点记录 nodename 解析
    - name: 设置 k8s_nodename 在 master[0] 节点 /etc/hosts 地址解析
      lineinfile:
        dest: /etc/hosts
        state: present
        regexp: "{{ K8S_NODENAME }} .* generated by kubeasz"
        line: "{{ inventory_hostname }}    {{ K8S_NODENAME }} # generated by kubeasz"
      delegate_to: "{{ groups.kube_master[0] }}"
      when: inventory_hostname != K8S_NODENAME
      tags: set_hosts

    # 同步解析到所有节点
    - block:
        - name: 获取 master[0] 节点由 kubeasz 创建的 /etc/hosts 地址解析
          shell: 'grep "generated by kubeasz" /etc/hosts | sort | uniq'
          register: RESOLVE_K8S_NODENAME
          delegate_to: "{{ groups.kube_master[0] }}"

        - name: 清理 master[0] 的临时记录
          lineinfile:
            dest: /etc/hosts
            state: absent
            regexp: "generated by kubeasz"
          delegate_to: "{{ groups.kube_master[0] }}"

        - name: 设置 k8s_nodename 在所有节点的 /etc/hosts 地址解析
          blockinfile:
            path: /etc/hosts
            block: |
              {{ RESOLVE_K8S_NODENAME.stdout }}
            marker: "### {mark} KUBEASZ MANAGED BLOCK"
          delegate_to: "{{ item }}"
          loop: "{{ groups.kube_master + groups.kube_node }}"
      run_once: true
      tags: set_hosts

    # 标记任务已完成（幂等性）
    - name: 设置 prepare_tasks 文件
      shell: "echo done > /opt/kubeasz_prepare_tasks"
  when: p.stat.isreg is not defined

---

📄 1.2.4 offline.yml

# 离线安装基础系统包（适用于无网络环境）

- name: 检查是否已下载离线软件包 {{ SYS_PKG_NAME }}
  stat:
    path: "{{ base_dir }}/down/packages/{{ SYS_PKG_NAME }}"
  register: pkg
  connection: local
  run_once: true

- name: 准备离线安装包目录
  file:
    path: /opt/kube/packages/basic
    state: directory

- block:
    - name: 分发离线软件包 {{ SYS_PKG_NAME }}
      copy:
        src: "{{ base_dir }}/down/packages/{{ SYS_PKG_NAME }}"
        dest: "/opt/kube/packages/basic/{{ SYS_PKG_NAME }}"

    # Debian 系：使用 dpkg 安装
    - name: 安装离线软件包 {{ SYS_PKG_NAME }}
      shell: |
        cd /opt/kube/packages/basic && tar zxf {{ SYS_PKG_NAME }} && \
        dpkg -i *.deb > /tmp/install_basic.log 2>&1
      register: install_info
      until: not install_info.failed
      retries: 3
      delay: 1
      when: ansible_distribution_file_variety in ["Debian"]

    # RedHat/SUSE 系：使用 rpm 安装
    - name: 安装离线软件包 {{ SYS_PKG_NAME }}
      shell: |
        cd /opt/kube/packages/basic && tar zxf {{ SYS_PKG_NAME }} && \
        rpm -Uvh --force --nodeps *.rpm > /tmp/install_basic.log 2>&1
      when: ansible_distribution_file_variety in ["RedHat", "SUSE"]
  when: pkg.stat.exists | bool
  ignore_errors: true

---

📄 1.2.5 redhat.yml

# RHEL/CentOS/Fedora/openEuler/Anolis OS 专用任务

- name: 删除 centos/redhat 默认安装的防火墙组件
  package:
    name:
      - firewalld
      - python-firewall
      - firewalld-filesystem
    state: absent
  ignore_errors: true

- name: 安装基础软件包
  package:
    name:
      - bash-completion
      - conntrack-tools
      - ipset
      - ipvsadm
      - libseccomp
      - lvm2
      - nfs-utils
      - psmisc
      - rsync
      - socat
    state: present
  when: INSTALL_SOURCE != "offline"

- import_tasks: offline.yml
  when: INSTALL_SOURCE == "offline"

# 关闭 SELinux（Kubernetes 不兼容）
- name: 临时关闭 selinux
  shell: "setenforce 0"
  failed_when: false

- name: 检查 /etc/selinux/config 是否存在
  stat:
    path: "/etc/selinux/config"
  register: se

- name: 永久关闭 selinux（文件存在时）
  lineinfile:
    dest: /etc/selinux/config
    regexp: "^SELINUX="
    line: "SELINUX=disabled"
  ignore_errors: true
  when: se.stat.isreg is defined

- name: 永久关闭 selinux（文件不存在时）
  shell: |
    echo 'SELINUX=disabled' > /etc/selinux/config && \
    echo 'SELINUXTYPE=targeted' >> /etc/selinux/config
  when: se.stat.isreg is not defined

# 优化日志：禁止 rsyslog 重复采集 journald 日志
- name: 禁止 rsyslog 获取 journald 日志1
  lineinfile:
    dest: /etc/rsyslog.conf
    state: present
    regexp: 'ModLoad imjournal'
    line: '#$ModLoad imjournal # provides access to the systemd journal'
  ignore_errors: true

- name: 禁止 rsyslog 获取 journald 日志2
  lineinfile:
    dest: /etc/rsyslog.conf
    state: present
    regexp: 'IMJournalStateFile'
    line: '#$IMJournalStateFile imjournal.state'
  ignore_errors: true

- name: 重启 rsyslog 服务
  service:
    name: rsyslog
    state: restarted
  ignore_errors: true

---

📄 1.2.6 suse.yml

# SUSE Linux 专用任务

- name: 安装基础软件包
  package:
    name:
      - bash-completion
      - conntrack-tools
      - ipset
      - ipvsadm
      - libseccomp2
      - lvm2
      - nfs-client
      - psmisc
      - rsync
      - socat
    state: present
  when: INSTALL_SOURCE != "offline"

- import_tasks: offline.yml
  when: INSTALL_SOURCE == "offline"

---

📄 1.3 templates/ 模板文件

📄 1.3.1 10-k8s-modules.conf.j2

# Kubernetes 所需内核模块列表
br_netfilter
dm_snapshot
dm_thin_pool
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
{% if 'NoFound' not in NF_CONNTRACK_IPV4.stdout %}
nf_conntrack_ipv4
{% endif %}

📄 1.3.2 30-k8s-ulimits.conf.j2

[Manager]
# 提升系统资源限制（文件描述符、进程数等）
DefaultLimitCORE=infinity
DefaultLimitNOFILE=100000
DefaultLimitNPROC=100000

📄 1.3.3 95-k8s-journald.conf.j2

[Journal]
# journald 日志持久化与限制
Storage=persistent
SystemMaxUse=2G
SystemMaxFileSize=200M
MaxRetentionSec=2week
ForwardToSyslog=no
ForwardToWall=no

📄 1.3.4 95-k8s-sysctl.conf.j2

## 网络优化
net.ipv4.ip_forward = 1
net.ipv4.tcp_tw_reuse = 2
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_fin_timeout = 30
net.core.somaxconn = 65535
net.core.netdev_max_backlog = 65536
net.ipv4.tcp_max_syn_backlog = 65536
net.ipv4.tcp_rmem = 4096 12582912 16777216
net.ipv4.tcp_wmem = 4096 12582912 16777216
net.netfilter.nf_conntrack_max = 1048576
{% if PROXY_MODE == "ipvs" %}
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 10
{% endif %}

# 文件系统
fs.file-max = 2097152
fs.inotify.max_user_instances = 8192
fs.inotify.max_user_watches = 524288

# 内存管理
vm.swappiness = 0
vm.max_map_count = 262144
vm.overcommit_memory = 1
vm.panic_on_oom = 0
kernel.panic = 10

# 容器支持
kernel.pid_max = 4194304
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-arptables = 1

# Kubernetes 要求
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0
kernel.softlockup_panic = 1

---

📄 1.4 vars/main.yml

# 离线软件包命名规则：根据系统类型和版本生成
SYS_PKG_NAME: "{{ ansible_distribution | lower | replace(' ', '_') }}_{{ ansible_distribution_major_version }}.tgz"