修改历史

2024.3.27 部署高可用k8s lucas

2024.3.28 部署flannel host-gw lucas

2024.3.29 部署calico bgp-rr,移动参考链接到对应步骤下  lucas

2024.4.8  部署efk集群抓取istio日志  lucas

2024.4.11 reindexes跨集群索引迁移 lucas

2024.4.12 istio部署 lucas

一:服务器环境

本地虚拟机环境 centos7.9 NAT模式

192.168.178.138    master01 (双网卡ens33 VIP192.168.178.141) 操作机
192.168.178.139    master02
192.168.178.140    master03

192.168.178.142    slave01

192.168.178.143    slave02

192.168.178.144    slave03

192.168.178.129    Calico虚拟路由

准备工作

#设置主机名
hostnamectl hostname=XXX
#配置免密(包括操作机)
ssh-keygen 
ssh-copy-id master*/slave*
#传输hosts
cat > /etc/hosts <<EOF
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6

192.168.178.138	master01
192.168.178.139	master02
192.168.178.140	master03
192.168.178.142 slave01
192.168.178.143 slave02
192.168.178.144 slave03
EOF
scp /etc/hosts master/slave:/etc/

---
#如果到这都看不懂就放弃吧

初始化(所有服务器)

# 更新
yum update -y

# 卸载 firewalld
systemctl stop firewalld
yum remove firewalld -y

# 卸载 networkmanager
systemctl stop NetworkManager
yum remove NetworkManager -y

# 同步服务器时间
yum install chrony -y
systemctl enable --now chronyd
chronyc sources

# 关闭 selinux
setenforce 0
sed -i '/^SELINUX=/cSELINUX=disabled' /etc/selinux/config
getenforce

# 关闭swap分区
swapoff -a # 临时
sed -i '/ swap / s/^/# /g' /etc/fstab #永久

# 安装常用工具包
yum install -y net-tools sysstat vim wget lsof unzip zip bind-utils lrzsz telnet bash-completion


# 安装ipvs
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules
bash /etc/sysconfig/modules/ipvs.modules
lsmod | grep -e ip_vs -e nf_conntrack_ipv4
yum install ipset ipvsadm -y

# 允许检查桥接流量
cat <<EOF | tee /etc/modules-load.d/k8s.conf
br_netfilter
EOF
sysctl --system
cat <<EOF | tee /etc/sysctl.d/k8s.conf
vm.swappiness = 0
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF
modprobe br_netfilter
lsmod | grep netfilter
sysctl net.bridge.bridge-nf-call-iptables net.bridge.bridge-nf-call-ip6tables net.ipv4.ip_forward

# 安装containerd
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum list containerd.io --showduplicates
yum install -y containerd.io
mkdir -p /etc/containerd
containerd config default | tee /etc/containerd/config.toml

#crictl命令配置
cat <<EOF | tee /etc/crictl.yaml
runtime-endpoint: "unix:///run/containerd/containerd.sock"
image-endpoint: "unix:///run/containerd/containerd.sock"
timeout: 10
debug: false
pull-image-on-create: false
disable-pull-on-run: false
EOF

#使用阿里云镜像下载
sed -i "s#registry.k8s.io#registry.aliyuncs.com/google_containers#g"  /etc/containerd/config.toml
#使用如下命令修改cgroup驱动
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml

systemctl daemon-reload
systemctl enable containerd
systemctl restart containerd

# 添加kubernetes yum软件源
cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.29/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v1.29/rpm/repodata/repomd.xml.key
exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni
EOF

# 安装kubeadm,kubelet和kubectl
yum list kubeadm --showduplicates
yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes

# 设置开机自启
systemctl daemon-reload
systemctl enable --now kubelet
# kubelet每隔几秒就会重启,陷入等待 kubeadm 指令的死循环


# 命令自动补全
yum install -y bash-completion
source <(crictl completion bash)
crictl completion bash >/etc/bash_completion.d/crictl
source <(kubectl completion bash)
kubectl completion bash >/etc/bash_completion.d/kubectl
source /usr/share/bash-completion/bash_completion

# NFS配置(忽略)
yum install -y nfs-common nfs-utils rpcbind
mkdir /nfs
chown nfsnobody /nfs/
#配置
vim /etc/exports
/nfs  *(rw,no_root_squash,sync)
exportfs -r
systemctl start rpcbind && systemctl enable rpcbind
systemctl start nfs && systemctl enable nfs
showmount -e

API server高可用部署

#安装 haproxy
yum -y install haproxy

#配置 haproxy
tee /etc/haproxy/haproxy.cfg <<EOF
global
log         127.0.0.1 local2
chroot      /var/lib/haproxy
pidfile     /var/run/haproxy.pid
maxconn     4000
user        haproxy
group       haproxy
daemon

stats socket /var/lib/haproxy/stats

defaults
mode                    http
log                     global
option                  httplog
option                  dontlognull
option  http-server-close
option                  redispatch
retries                 3
timeout http-request    10s
timeout queue           1m
timeout connect         10s
timeout client          1m
timeout server          1m
timeout http-keep-alive 10s
timeout check           10s
maxconn                 3000

listen  k8s-apiserver 
  bind *:8443
  mode tcp
  timeout client 1h
  timeout connect 1h

  log global
  option tcplog
  balance     roundrobin
  server  master01 192.168.178.138:6443 check
  server  master02 192.168.178.139:6443 check
  server  master03 192.168.178.140:6443 check
  acl is_websocket hdr(Upgrade) -i WebSocket
  acl is_websocket hdr_beg(Host) -i ws
EOF

#运行 haproxy
systemctl enable --now haproxy

#安装 keepalived
yum -y install keepalived

#配置 keepalived
tee > /etc/keepalived/keepalived.conf <<EOF
global_defs {
    router_id 100
    vrrp_version 2
    vrrp_garp_master_delay 1
    vrrp_mcast_group4 224.0.0.18  #后续podSubnet需要一致
}   

vrrp_script chk_haproxy {
    script       "/usr/bin/nc -nvz -w 2 127.0.0.1 8443"
    timeout 1
    interval 1   # check every 1 second
    fall 2       # require 2 failures for KO
    rise 2       # require 2 successes for OK
}   

vrrp_instance lb-vips {
    state MASTER
    interface ens33 #VIP网卡名字
    virtual_router_id 100
    priority 150
    advert_int 1
    nopreempt
    track_script {
        chk_haproxy
    }
    authentication {
        auth_type PASS
        auth_pass blahblah
    }
    virtual_ipaddress {
        192.168.178.141/24 dev eth0 #VIP的IP
    }
}
EOF

#运行 keepalived
systemctl enable --now keepalived

#检查 vip 的情况
ip a
journalctl -fu keepalived

生成 kubeadm 默认配置文件 kubeadm-config.yaml

照抄:

1. 二、安装 Kubernetes 高可用集群 · Istio实战指南

2.利用 kubeadm 创建高可用集群 | Kubernetes

修改过的已经加#,具体以你自己的版本为主。

#生成 kubeadm 默认配置文件

kubeadm config print init-defaults --component-configs \
KubeProxyConfiguration,KubeletConfiguration > kubeadm-config.yaml

#修改后完整的配置文件
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 192.168.178.138 #操作机IP
  bindPort: 6443
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  name: master01
  taints: null
---
controlPlaneEndpoint: 192.168.178.141:8443 #VIP
apiServer:
  timeoutForControlPlane: 4m0s
  extraArgs:
    authorization-mode: "Node,RBAC"
    enable-admission-plugins: "DefaultIngressClass,DefaultStorageClass,DefaultTolerationSeconds,LimitRanger,MutatingAdmissionWebhook,NamespaceLifecycle,PersistentVolumeClaimResize,PodSecurity,Priority,ResourceQuota,RuntimeClass,ServiceAccount,StorageObjectInUseProtection,TaintNodesByCondition,ValidatingAdmissionPolicy,ValidatingAdmissionWebhook" #准入控制
    etcd-servers: https://master01:2379,https://master02:2379,https://master03:2379 #master节点
  certSANs:
  - 192.168.178.141 # VIP 地址
  - 10.96.0.1  # service cidr的第一个ip
  - 127.0.0.1 # 多个master的时候负载均衡出问题了能够快速使用localhost调试
  - master01
  - master02
  - master03
  - kubernetes
  - kubernetes.default
  - kubernetes.default.svc
  - kubernetes.default.svc.cluster.local
  extraVolumes:
  - hostPath: /etc/localtime
    mountPath: /etc/localtime
    name: timezone
    readOnly: true
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd
    serverCertSANs: #证书分发
    - master01
    - master02
    - master03
    peerCertSANs:
    - master01
    - master02
    - master03
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: 1.29.0
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
  podSubnet: 10.244.0.0/16 #跟keep alive保持一致
scheduler:
  extraVolumes:
  - hostPath: /etc/localtime
    mountPath: /etc/localtime
    name: timezone
    readOnly: true
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
bindAddressHardFail: false
clientConnection:
  acceptContentTypes: ""
  burst: 0
  contentType: ""
  kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
  qps: 0
clusterCIDR: ""
configSyncPeriod: 0s
conntrack:
  maxPerCore: null
  min: null
  tcpBeLiberal: false
  tcpCloseWaitTimeout: null
  tcpEstablishedTimeout: null
  udpStreamTimeout: 0s
  udpTimeout: 0s
detectLocal:
  bridgeInterface: ""
  interfaceNamePrefix: ""
detectLocalMode: ""
enableProfiling: false
healthzBindAddress: ""
hostnameOverride: ""
iptables:
  localhostNodePorts: null
  masqueradeAll: false
  masqueradeBit: null
  minSyncPeriod: 0s
  syncPeriod: 0s
ipvs:
  excludeCIDRs: null
  minSyncPeriod: 0s
  scheduler: ""
  strictARP: false
  syncPeriod: 0s
  tcpFinTimeout: 0s
  tcpTimeout: 0s
  udpTimeout: 0s
kind: KubeProxyConfiguration
logging:
  flushFrequency: 0
  options:
    json:
      infoBufferSize: "0"
  verbosity: 0
metricsBindAddress: ""
mode: "ipvs" #IPVS模式
nftables:
  masqueradeAll: false
  masqueradeBit: null
  minSyncPeriod: 0s
  syncPeriod: 0s
nodePortAddresses: null
oomScoreAdj: null
portRange: ""
showHiddenMetricsForVersion: ""
winkernel:
  enableDSR: false
  forwardHealthCheckVip: false
  networkName: ""
  rootHnsEndpointName: ""
  sourceVip: ""
---
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 0s
    enabled: true
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 0s
    cacheUnauthorizedTTL: 0s
cgroupDriver: systemd #systemd模式
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
containerRuntimeEndpoint: ""
cpuManagerReconcilePeriod: 0s
evictionPressureTransitionPeriod: 0s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageMaximumGCAge: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
logging:
  flushFrequency: 0
  options:
    json:
      infoBufferSize: "0"
  verbosity: 0
memorySwap: {}
nodeStatusReportFrequency: 0s
nodeStatusUpdateFrequency: 0s
rotateCertificates: true
runtimeRequestTimeout: 0s
shutdownGracePeriod: 0s
shutdownGracePeriodCriticalPods: 0s
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
volumeStatsAggPeriod: 0s


#验证语法
kubeadm init --config kubeadm-config.yaml --dry-run

#预先拉取镜像
kubeadm config images pull --config kubeadm-config.yaml
#其它节点
kubeadm config images pull

#初始化集群
kubeadm init --config kubeadm-config.yaml --upload-certs 
--upload-certs 标志用来将在所有控制平面实例之间的共享证书上传到集群。

等待初始化完成信息如下
···
Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of the control-plane node running the following command on each as root:

  kubeadm join 192.168.178.141:8443 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:0bc9dd684a2e3e1417e85765ef826208d2acfdbc530b6d641bb7f09e3a7e069f \
	--control-plane --certificate-key 1c5a48c6d5ea3765c69f42458fda18381752a16618796be6798e117d4cc55ac3

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.178.141:8443 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:0bc9dd684a2e3e1417e85765ef826208d2acfdbc530b6d641bb7f09e3a7e069f 
···

#master节点加入
[root@master02 ~]# kubeadm join 192.168.178.141:8443 --token abcdef.0123456789abcdef --discovery-token-ca-cert-hash sha256:0bc9dd684a2e3e1417e85765ef826208d2acfdbc530b6d641bb7f09e3a7e069f --control-plane --certificate-key 1c5a48c6d5ea3765c69f42458fda18381752a16618796be6798e117d4cc55ac3

#slave节点加入
kubeadm join 192.168.178.141:8443 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:0bc9dd684a2e3e1417e85765ef826208d2acfdbc530b6d641bb7f09e3a7e069f 

#控制节点检查
kubectl get nodes

二:网络部署

flannel host-gw模式

#下载 For Kubernetes v1.17+ 对于 Kubernetes v1.17+
wget https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml

#编辑
vim kube-flannel.yml 
 89   net-conf.json: |
 90     {
 91       "Network": "10.244.0.0/16",
 92       "Backend": {
 93         "Type": "host-gw"
 94       }
 95     }

#安装前,运行下面命令
ip addr show
route -n

#安装
kubectl create  -f kube-flannel.yml

#查看
kubectl get all -n kube-flannel
···
NAME                        READY   STATUS    RESTARTS      AGE
pod/kube-flannel-ds-4wl74   1/1     Running   0             17m
pod/kube-flannel-ds-r7j2n   1/1     Running   0             17m
pod/kube-flannel-ds-w79wg   1/1     Running   0             17m
···

#检验
ip addr show
route -n

用安装前后的命令行输出来看结果
route -n:
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.178.2   0.0.0.0         UG    0      0        0 ens33
10.244.0.0      0.0.0.0         255.255.255.0   U     0      0        0 cni0
10.244.1.0      192.168.178.139 255.255.255.0   UG    0      0        0 ens33
10.244.2.0      192.168.178.140 255.255.255.0   UG    0      0        0 ens33
169.254.0.0     0.0.0.0         255.255.0.0     U     1002   0        0 ens32
169.254.0.0     0.0.0.0         255.255.0.0     U     1003   0        0 ens33
192.168.178.0   0.0.0.0         255.255.255.0   U     0      0        0 ens32
192.168.178.0   0.0.0.0         255.255.255.0   U     0      0        0 ens33

和ip addr show:
cni0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 7e:95:b7:11:3e:f6 brd ff:ff:ff:ff:ff:ff
    inet 10.244.0.1/24 brd 10.244.0.255 scope global cni0
       valid_lft forever preferred_lft forever
    inet6 fe80::7c95:b7ff:fe11:3ef6/64 scope link 
       valid_lft forever preferred_lft forever

对应的10.244.0.0/23 10.244.1.0/24 10.244.2.0/24 归属于10.244.0.0/12 CIDR地址块

可知部署正常,也可以安装pod互相调用containerIP即可知道了。

calico BGP模式

支持的版本
我们针对以下 Kubernetes 版本测试 Calico v3.27。 其他版本可能也有效,但我们没有积极测试它们。
v1.27
v1.28
v1.29

#安装calicoctl https://docs.tigera.io/calico/latest/operations/calicoctl/install
curl -L https://github.com/projectcalico/calico/releases/download/v3.27.2/calicoctl-linux-amd64 -o calicoctl
chmod +x calicoctl
cp calicoctl /usr/bin/calicoctl
calicoctl version

#下载calico https://docs.tigera.io/calico/latest/getting-started/kubernetes/quickstart
wget https://raw.githubusercontent.com/projectcalico/calico/v3.27.2/manifests/tigera-operator.yaml
wget https://raw.githubusercontent.com/projectcalico/calico/v3.27.2/manifests/custom-resources.yaml

#修改custom-resources.yaml 部分内容
```
apiVersion: operator.tigera.io/v1
kind: Installation
metadata:
  name: default
spec:
  # 选择网卡-多网卡环境
  calicoNetwork:
    nodeAddressAutodetectionV4:
      interface: ens32
    # 设置cidr为k8s创建时的podSubnet
    ipPools:
    - blockSize: 26
      cidr: 10.244.0.0/16
      encapsulation: VXLANCrossSubnet
      natOutgoing: Enabled
      nodeSelector: all()
```

#安装calico
calicoctl create -f tigera-operator.yaml
calicoctl create -f custom-resources.yaml

#等待pod启动完成才行
kubectl get pod -n calico-apiserver -o wide
kubectl get pod -n calico-system -o wide

#此时的状态
[root@master01 ~]# calicoctl node status
Calico process is running.

IPv4 BGP status
+-----------------+-------------------+-------+----------+-------------+
|  PEER ADDRESS   |     PEER TYPE     | STATE |  SINCE   |    INFO     |
+-----------------+-------------------+-------+----------+-------------+
| 192.168.178.139 | node-to-node mesh | up    | 13:11:48 | Established |
| 192.168.178.140 | node-to-node mesh | up    | 13:11:48 | Established |
+-----------------+-------------------+-------+----------+-------------+

#修改为RR路由模式
#新建一个路由虚拟机192.168.178.129,因为端口会跟现有机器冲突。
#路由模式选择quagga,照抄https://blog.csdn.net/cloud_engineer/article/details/127706088
#也可以安装路由vyos-ISO:https://docs.vyos.io/en/latest/installation/install.html

#关闭防火墙
systemctl stop firewalld
yum remove firewalld -y

#安装quagga
yum install quagga telnet -y
cp /usr/share/doc/quagga-0.99.22.4/zebra.conf.sample /etc/quagga/zebra.conf
cp /usr/share/doc/quagga-0.99.22.4/bgpd.conf.sample /etc/quagga/bgpd.conf
cd /etc/quagga/
vim bgpd.conf
```
router bgp 63500
 bgp router-id 192.168.178.129 #路由节点IP
 network 192.168.178.0 #k8s集群路由
 neighbor 192.168.178.138 remote-as 63500
 neighbor 192.168.178.138 route-reflector-client
 neighbor 192.168.178.139 remote-as 63500
 neighbor 192.168.178.139 route-reflector-client
 neighbor 192.168.178.140 remote-as 63500
 neighbor 192.168.178.140 route-reflector-client
```
systemctl enable zebra --now
systemctl enable bgpd --now
vtysh
```
[root@localhost quagga]# vtysh 

Hello, this is Quagga (version 0.99.22.4).
Copyright 1996-2005 Kunihiro Ishiguro, et al.

localhost.localdomain# configure  t
localhost.localdomain(config)# hostname  bgp-rr
bgp-rr(config)# int ens32
bgp-rr(config-if)# ip add 192.168.178.129/24
bgp-rr(config-if)# no shutdown
bgp-rr(config-if)# end
bgp-rr# wr
Building Configuration...
Can't open configuration file /etc/quagga/zebra.conf.EHOX90.
Can't open configuration file /etc/quagga/bgpd.conf.9AEAb1.
[OK]
```

#calico-rr模式部署 
#配置文件configuration详解https://docs.tigera.io/calico/latest/reference/resources/bgpconfig
cat calico-bgp-configuration.yaml 
```
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
  name: default
spec:
#https://docs.tigera.io/calico/latest/reference/resources/bgpconfig 具体解释
  logSeverityScreen: Info
  nodeToNodeMeshEnabled: false
#  nodeMeshMaxRestartTime: 120s
  asNumber: 63500
  serviceClusterIPs:
    - cidr: 10.244.128.0/24
  serviceExternalIPs:
    - cidr: 10.244.129.0/24
  listenPort: 178
  bindMode: NodeIP
  communities:
    - name: bgp-large-community
      value: 63400:300:100
  prefixAdvertisements:
    - cidr: 10.244.0.0/16
      communities:
        - bgp-large-community
        - 63400:120

```
#配置文件peer详解https://docs.tigera.io/calico/latest/reference/resources/bgppeer
cat calico-bgp-peer.yaml 
```
apiVersion: projectcalico.org/v3
kind: BGPPeer
metadata:
  name: vmware-peer
spec:
  peerIP: 192.168.178.129
  keepOriginalNextHop: true
  asNumber: 63500
  nodeSelector: rack == 'vmwarepeer'
```
#node添加标签
kubectl  label  node -all rack="vmwarepeer"
#部署
calicoctl create -f calico-bgp-configuration.yaml
calicoctl create -f calico-bgp-peer.yaml
#等待结果正常
```
[root@master01 ~]#  calicoctl node status
Calico process is running.

IPv4 BGP status
+-----------------+---------------+-------+----------+-------------+
|  PEER ADDRESS   |   PEER TYPE   | STATE |  SINCE   |    INFO     |
+-----------------+---------------+-------+----------+-------------+
| 192.168.178.129 | node specific | up    | 03:03:39 | Established |
+-----------------+---------------+-------+----------+-------------+

IPv6 BGP status
No IPv6 peers found.


```
#同时路由端
```
[root@localhost quagga]# vtysh
localhost.localdomain# show ip bgp  summary  
BGP router identifier 192.168.178.129, local AS number 63500
RIB entries 11, using 1232 bytes of memory
Peers 3, using 13 KiB of memory

Neighbor        V    AS MsgRcvd MsgSent   TblVer  InQ OutQ Up/Down  State/PfxRcd
192.168.178.138 4 63500      59      55        0    0    0 00:48:47        5
192.168.178.139 4 63500     177     165        0    0    0 00:48:15        3
192.168.178.140 4 63500     176     173        0    0    0 00:48:48        3

Total number of neighbors 3

```

#这样的话我们的Calico-BGP-RR模式就部署好了
#测试的话可以tcpdump -i ens32 host container-ip
#也可以traceroute container-ip
#验证路由的响应速度和路线。

三:部署istio

版本支持Istio / 版本支持

#下载istio1.21.1版本
wget https://github.com/istio/istio/releases/download/1.21.1/istio-1.21.1-linux-amd64.tar.gz

#解压安装
tar -xzf istio-1.21.1-linux-amd64.tar.gz 
cp istio-1.21.1/bin/istioctl /usr/bin/
istioctl manifest apply --set profile=demo

#安装kiali
cd istio/istio-1.21.1/samples/addons/
kubectl create -f jaeger.yaml -f kiali.yaml -f loki.yaml -f prometheus.yaml -f grafana.yaml

#等待启动完成即可

#访问kiali nodeport方式
kubectl edit svc -n istio-system kiali
```
42   type: NodePort
34     nodePort: 30021
```

#访问kiali,prometheus类似
http://192.168.178.138:30021/

#创建一个vs,gw,验收istio是否正常
vim nginx.yaml
```
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: nginx-deployment
spec:
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx:latest
        ports:
        - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: nginx-service
spec:
  selector:
    app: nginx
  ports:
    - protocol: TCP
      port: 80
      targetPort: 80
      nodePort: 30080
  type: NodePort
```

vim /root/nginx-vs.yaml
```
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: nginx-vs
  namespace: default
spec:
  gateways:
    - nginx-vs-gateway
  hosts:
    - nginx.test.com
  http:
    - match:
        - uri:
            prefix: /api/
      rewrite:
        uri: /
      route:
        - destination:
            host: java.java.svc.cluster.local
            port:
              number: 9999
    - match:
        - uri:
            prefix: /
      route:
        - destination:
            host: nginx-service.default.svc.cluster.local
            port:
              number: 80
```

vim nginx-gw.yaml
```
apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
  name: nginx-vs-gateway
  namespace: default
spec:
  selector:
    istio: ingressgateway
  servers:
    - hosts:
        - nginx.test.com
      port:
        name: http
        number: 80
        protocol: HTTP
#      tls:
#        httpsRedirect: true
#    - hosts:
#        - nginx.test.com
#      port:
#        name: https
#        number: 443
#        protocol: HTTPS
#      tls:
#        credentialName: nginx-test-com-crt
#        mode: SIMPLE
```

#安装
kubectl create -f nginx.yaml -f nginx-vs.yaml -f nginx-gw.yaml

#查看vs,gw
[root@master01 conf.d]# kubectl get virtualservices,gateways -n default
NAME                                          GATEWAYS               HOSTS                AGE
virtualservice.networking.istio.io/nginx-vs   ["nginx-vs-gateway"]   ["nginx.test.com"]   21m

NAME                                           AGE
gateway.networking.istio.io/nginx-vs-gateway   23m

#查看istio的svc
[root@master01 conf.d]# kubectl get svc -n istio-system
NAME                   TYPE           CLUSTER-IP       EXTERNAL-IP   PORT(S)                                                                      AGE
istio-egressgateway    ClusterIP      10.103.138.124   <none>        80/TCP,443/TCP                                                               51m
istio-ingressgateway   LoadBalancer   10.105.22.202    <pending>     15021:30854/TCP,80:30306/TCP,443:31237/TCP,31400:30331/TCP,15443:30332/TCP   51m
istiod                 ClusterIP      10.107.108.221   <none>        15010/TCP,15012/TCP,443/TCP,15014/TCP                                        52m
jaeger-collector       ClusterIP      10.104.53.49     <none>        14268/TCP,14250/TCP,9411/TCP,4317/TCP,4318/TCP                               42m
kiali                  NodePort       10.111.51.85     <none>        20001:30021/TCP                                                              42m
loki-headless          ClusterIP      None             <none>        3100/TCP                                                                     42m
tracing                ClusterIP      10.106.95.240    <none>        80/TCP,16685/TCP                                                             42m
zipkin                 ClusterIP      10.102.223.213   <none>        9411/TCP                                                                     42m


#访问配置,因为没有LB,所以使用nginx代理istio-ingress80端口,才能正常访问,否则就使用metallb
#k8s外部安装nginx
yum install epel-release
yum install nginx -y
vim /etc/nginx/conf.d/nginx.conf
```
upstream nginx_http {
  server 192.168.178.138:30306;
  server 192.168.178.139:30306;
  server 192.168.178.140:30306;
}

server {
        listen 80;
        server_name nginx.test.com;

        location / {
            index index.jsp  index.html index.htm;
            proxy_pass  http://nginx_http;
            proxy_redirect off;
            proxy_set_header X-Real_IP $remote_addr;
            proxy_set_header Host $host;
            proxy_set_header X_Forward_For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header X-Request-ID $request_id;
            proxy_next_upstream error timeout;
            client_max_body_size 20m;
            proxy_read_timeout 1200s;
        }
}

```

#自己电脑
#C:\Windows\System32\drivers\etc
#编辑hosts文件
```
192.168.178.138 nginx.test.com
```
#然后本机访问即可
http://nginx.test.com/
http://nginx.test.com/api


#虽然现在装了istio-kiali,但是没有pod注册进来,无法查看,所以需要注入配置
#注入
kubectl label namespace <namespace> istio-injection=enabled
#取消注入
kubectl label namespace <namespace> istio-injection=disabled
#查看注入
kubectl get pod <pod-name> -n <namespace> -o jsonpath='{.spec.containers[*].name}'

#总结:
#Istio还有很多功能包括流量控制链路追踪等高阶操作,可以查看组件官网如何操作。

四:部署EFK集群

helm部署elasticsearch8.13.1+logstash8.13.1+kibana8.13.1+filebeat8.13.1+kafka-cluster3.7.0+elastalert2

部署NFS存储efk应用数据

#nfs安装
#server
yum install -y nfs-common nfs-utils rpcbind
mkdir /nfs
chown nfsnobody /nfs/
#配置
vim /etc/exports
```
/nfs  192.168.0.0/24(rw,no_root_squash,sync)
```
exportfs -r
systemctl start rpcbind && systemctl enable rpcbind
systemctl start nfs && systemctl enable nfs
showmount -e


#client
yum install -y nfs-utils rpcbind
systemctl start nfs-utils.service rpcbind.service
systemctl enable nfs-utils.service rpcbind.service
showmount -e master01

创建SC存储

#创建nfs的es文件夹
mkdir /nfs/elastic/
#rbac
cat rbac.yaml 
```
apiVersion: v1
kind: ServiceAccount
metadata:
  name: elastic-nfs-client-provisioner
  namespace: elastic
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: elastic-nfs-client-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-elastic-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: elastic-nfs-client-provisioner
    namespace: elastic
roleRef:
  kind: ClusterRole
  name: elastic-nfs-client-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-elastic-nfs-client-provisioner
  namespace: elastic
rules:
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-elastic-nfs-client-provisioner
  namespace: elastic
subjects:
  - kind: ServiceAccount
    name: elastic-nfs-client-provisioner
    namespace: elastic
roleRef:
  kind: Role
  name: leader-locking-elastic-nfs-client-provisioner
  apiGroup: rbac.authorization.k8s.io
```

#provisioner
cat provisioner.yaml
```
apiVersion: apps/v1
kind: Deployment # provisioner的类型是一个deployment
metadata:
  name: elastic-nfs-client-provisioner
  labels:
    app: elastic-nfs-client-provisioner
  namespace: elastic # 指定provisioner所属的namespace,改成你自己的namespace
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: elastic-nfs-client-provisioner
  template:
    metadata:
      labels:
        app: elastic-nfs-client-provisioner
    spec:
      serviceAccountName: elastic-nfs-client-provisioner # 指定provisioner使用的sa
      containers:
        - name: elastic-nfs-client-provisioner
          image: vbouchaud/nfs-client-provisioner:latest # 指定provisioner的镜像
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes # 固定写法
          env:
            - name: PROVISIONER_NAME
              value: elastic-storage-class # 指定分配器的名称,创建storageclass会用到
            - name: NFS_SERVER
              value: 192.168.178.138  # 指定使用哪一块存储,这里用的是nfs,此处填写nfs的地址
            - name: NFS_PATH
              value: /nfs/elastic # 使用nfs哪一块盘符
      volumes:
        - name: nfs-client-root
          nfs:
            server: 192.168.178.138 # 和上面指定的nfs地址保持一致
            path: /nfs/elastic # 和上面指定的盘符保持一致
```

#sc
cat sc.yaml 
```
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: elasticsearch-sc
provisioner: elastic-storage-class
parameters:
  archiveOnDelete: "true"  ## 删除pv的时候,pv的内容是否要备份
allowVolumeExpansion: true
```

#创建
kubectl create -f rbac.yaml -f provisioner.yaml -f sc.yaml

#查看
kubectl get sc | grep elasticsearch-sc
elasticsearch-sc                 elastic-storage-class             Delete          Immediate              true                   2h


安装es+kibana

1.安装helm https://helm.sh/docs/intro/install/
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
chmod 700 get_helm.sh
./get_helm.sh
helm version
 
2.安装es+kibana 证书模式 https://artifacthub.io/packages/helm/bitnami/elasticsearch
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update
helm pull bitnami/elasticsearch --version 20.0.3
tar -xzf elasticsearch-20.0.3.tgz
#根据你的自定义修改 --我的示例
 23   storageClass: "elasticsearch-sc"
 29   kibanaEnabled: true
 218 security:
 221   enabled: true
 225   elasticPassword: "123456"
 233   fipsMode: false
 236   tls:
 239     restEncryption: true
 244     autoGenerated: true
 302 service:
 305   type: ClusterIP
 360 ingress:
 363   enabled: false
 462 master:
 465   masterOnly: true
 468   replicaCount: 1
 512      requests:
 513        cpu: 2
 514        memory: 512Mi
 515      limits:
 516        cpu: 3
 517        memory: 1024Mi
 570   networkPolicy:
 573     enabled: false
 862 data:
 865   replicaCount: 1
 963   networkPolicy:
 966     enabled: false
1255 coordinating:
1258   replicaCount: 1
1357   networkPolicy:
1360     enabled: false
1610 ingest:
1613   enabled: true
1616   replicaCount: 1
1720   networkPolicy:
1723     enabled: false
2143 metrics:
2146   enabled: true ##给prometheus的
2682 kibana:
2683   elasticsearch:
2684     hosts:
2685       - '{{ include "elasticsearch.service.name" . }}'
2686     port: '{{ include "elasticsearch.service.ports.restAPI" . }}'
2687     security:
2688       auth:
2689         enabled: true
2691         kibanaUsername: "kibana_system"
2692         kibanaPassword: "123456"
2693       tls:
2695         enabled: true
2697         existingSecret: elastic-elasticsearch-coordinating-crt
2699         usePemCerts: true

#创建ns
kubectl create ns elastic 
#安装应用
helm install elastic -n elastic elasticsearch/
#修改kibana_system密码 --待优化的bug
#进入
kubectl exec -it -n elastic elastic-elasticsearch-coordinating-0 -- bash
#修改
elasticsearch-reset-password -u kibana_system --url "http://elastic-elasticsearch:9200"
#会生成一个随机密码类似=C=c1Kd1asLfMoF0fg4e,将密码修改
#拿到加密密码
echo =C=c1Kd1asLfMoF0fg4e | base64
#输出PUM9YzFLZDFhc0xmTW9GMGZnNGUK
#修改密码,把密码换上去
kubectl edit secrets -n elastic elastic-kibana
```
  kibana-password: PUM9YzFLZDFhc0xmTW9GMGZnNGUK
```
#等待容器重启后修改kibana的svc
kubectl edit svc -n elastic elastic-kibana
```
  ports:
  - name: http
    nodePort: 30056
    port: 5601
    protocol: TCP
    targetPort: 5601

  type: NodePort

```
#这时候就可以正常访问地址了
#访问http://192.168.178.138:30056

安装kafka-cluster

#helm安装https://artifacthub.io/packages/helm/bitnami/kafka
helm repo add bitnami https://charts.bitnami.com/bitnami
helm pull bitnami/kafka --version 28.0.3
tar -xzf kafka-28.0.3.tgz
#根据你的自定义修改 --我的示例
vim kafka/values.yaml 
```
 21   storageClass: "elasticsearch-sc"
 156     protocol: PLAINTEXT ##不使用sasl模式
 167     protocol: PLAINTEXT
 175     protocol: PLAINTEXT
 184     protocol: PLAINTEXT
 470 controller:
 474   replicaCount: 3
 884 broker:
 887   replicaCount: 3
 1634 networkPolicy:
 1637   enabled: false

```
#安装
helm  install -n elastic kafka kafka/

#查看topic和group命令和查看偏移量
kafka-topics.sh --list --bootstrap-server kafka-controller-0.kafka-controller-headless.elastic.svc.cluster.local:9092,kafka-broker-0.kafka-broker-headless.elastic.svc.cluster.local:9092,kafka-broker-1.kafka-broker-headless.elastic.svc.cluster.local:9092,kafka-broker-2.kafka-broker-headless.elastic.svc.cluster.local:9092

kafka-consumer-groups.sh --list --bootstrap-server kafka-controller-0.kafka-controller-headless.elastic.svc.cluster.local:9092,kafka-broker-0.kafka-broker-headless.elastic.svc.cluster.local:9092,kafka-broker-1.kafka-broker-headless.elastic.svc.cluster.local:9092,kafka-broker-2.kafka-broker-headless.elastic.svc.cluster.local:9092

kafka-consumer-groups.sh --describe --group  logstash --bootstrap-server kafka-controller-0.kafka-controller-headless.elastic.svc.cluster.local:9092,kafka-broker-0.kafka-broker-headless.elastic.svc.cluster.local:9092,kafka-broker-1.kafka-broker-headless.elastic.svc.cluster.local:9092,kafka-broker-2.kafka-broker-headless.elastic.svc.cluster.local:9092

安装logstash

#helm安装https://artifacthub.io/packages/helm/bitnami/logstash
helm repo add bitnami https://charts.bitnami.com/bitnami
helm pull bitnami/logstash --version 6.0.2
tar -xzf  logstash-6.0.2.tgz
#根据你的自定义修改 --我的示例
vim logstash/values.yaml
```
20   storageClass: "elasticsearch-sc"
518 networkPolicy:
521   enabled: false
```

#安装
helm install logstash -n elastic logstash/

安装elastalert2

#helm安装https://artifacthub.io/packages/helm/elastalert2/elastalert2
helm repo add elastalert2 https://jertel.github.io/elastalert2
helm pull elastalert2/elastalert2 --version 2.17.0
tar -xzf elastalert2-2.17.0.tgz

#修改
vim elastalert2/values.yaml
```
51行
elasticsearch:
  host: elastic-elasticsearch.elastic
  port: 9200
  useSsl: "true"
  username: "elastic"
  password: "123456"
  credentialsSecret: "elastic-elasticsearch-coordinating-crt"
  credentialsSecretUsernameKey: ""
  credentialsSecretPasswordKey: ""
  verifyCerts: "True"
  clientCert: "/certs/tls.crt"
  clientKey: "/certs/tls.key"
  caCerts: "/certs/ca.crt"
  certsVolumes:
    - name: es-certs
      secret:
        defaultMode: 420
        secretName: elastic-elasticsearch-coordinating-crt
  certsVolumeMounts:
    - name: es-certs
      mountPath: /certs
      readOnly: true
```

#安装
helm install -n elastic elastalert2 elastalert2/

安装飞书告警通知和IP归属地展示

提前下载文件elastalert.tar.gz 和GeoLite2-City.tar.gz

--私信我拿

#保存到文件
mkdir /nfs/elastic-system
chmod 777 elastalert2  GeoLite2-City -R

创建pv,pvc

#创建elastalert2的pv,pvc
cat elastalert2-pv.yaml 
```
apiVersion: v1
kind: PersistentVolume
metadata:
  name: elastic-pvc-elastalert2-pvc
  namespace: elastic
spec:
  capacity:
    storage: 2Gi
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  nfs:
    server: 192.168.178.138
    path: /nfs/elastic/elastalert2
---

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: elastalert2-pvc
  namespace: elastic
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 2Gi
```

#创建geolite2city的pv,pvc
cat geolite2city-pv.yaml 
```
apiVersion: v1
kind: PersistentVolume
metadata:
  name: elastic-pvc-geolite2city-pvc
  namespace: elastic
spec:
  capacity:
    storage: 2Gi
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  nfs:
    server: 192.168.178.138
    path: /nfs/elastic/GeoLite2-City
---

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: geolite2city-pvc
  namespace: elastic
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 2Gi
```

#安装
kubectl create -f elastalert2-pv.yaml -f geolite2city-pv.yaml 

挂载到POD

#elastalert2挂载飞书告警
kubectl edit pod -n elastic elastalert2-xxx-xxx
```
    - mountPath: /usr/local/lib/python3.11/site-packages/elastalert/
      name: volume
  - name: volume
    persistentVolumeClaim:
      claimName: elastalert2-pvc
```

#GeoLite2-City挂载到logstash,顺便挂载es证书
kubectl edit pod -n elastic logstash-0
```
    - mountPath: /mnt
      name: volume
    - mountPath: /etc/certificate/ca.crt
      name: cert
      subPath: ca.crt

  - name: volume
    persistentVolumeClaim:
      claimName: geolite2city-pvc
  - name: cert
    secret:
      defaultMode: 420
      secretName: elastic-elasticsearch-coordinating-crt
```

修改logstash和elastalert2的CM

#修改logstash的cm --grok日志切割
kubectl exec -it -n elastic logstash-0 -- bash
vim /bitnami/logstash/config/logstash.conf
```
input {
  kafka {
    bootstrap_servers => "kafka.elastic:9092"
    topics_pattern => "[a-z].*"
    codec => "json"
  }
}
filter {
if [fields][log_topic] == "istio-ingress" {
  grok {
    match => {
          "message" => "\[%{TIMESTAMP_ISO8601:timestamp}\] \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}\" %{INT:response} %{NOTSPACE:upstream_header} %{GREEDYDATA:upstream_error} %{NOTSPACE:upstream_error1} \"%{NOTSPACE:referrer}\" %{INT:bytes_in} %{INT:bytes_out} %{INT:duration} %{NOTSPACE:duration2} \"%{NOTSPACE:client_ip}\" \"%{GREEDYDATA:user_agent}\" \"%{UUID:request_id}\" \"%{HOSTNAME:hostname}\" \"%{GREEDYDATA:x_forwarded_for}\" %{GREEDYDATA:outbound_route} %{IPORHOST:destination_ip}:%{INT:destination_port0} %{IPORHOST:destination_ip}:%{INT:destination_port} %{IPORHOST:source_ip}:%{INT:source_port} %{HOSTNAME:source_hostname} %{NOTSPACE:after}"
    }
  }
  geoip {
          source => "source_ip"
          target => "geoip"
          database  =>  "/mnt/GeoLite2-City_20230707/GeoLite2-City.mmdb"
          add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
          add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
  }
  mutate {
    convert => [ "[geoip][coordinates]", "float"]
  }
 }
}
output {
  elasticsearch {
      hosts => ["https://elastic-elasticsearch.elastic.svc.cluster.local:9200"]
      user => "elastic"
      password => "123456"
      index => "logstash-%{[fields][log_topic]}-%{+YYYY.MM.dd}"
      ssl => "true"
      cacert => "/etc/certificate/ca.crt"
}
}
```


#添加elastalert2的规则 --实例404告警
kubectl exec -it -n elastic elastalert2-xxx-xxx -- bash
vim rules/nginx_4xx.yaml
```
name: nginx_access_4xx
type: frequency
index: logstash-istio-ingress-* 
is_enabled: true
num_events: 1
terms_size: 50
timeframe:
  minutes: 3
timestamp_type: "iso"
use_strftime_index: false
filter: 
     - query: 
        query_string: 
          query: "response:4*" 

include: ["@timestamp","source_hostname","response","verb","request","message"]
alert:
- "elastalert.elastalert_modules.feishu_alert.FeishuAlert"
# 飞书机器人接口地址
feishualert_url: "https://open.feishu.cn/open-apis/bot/v2/hook/"
# 飞书机器人id
feishualert_botid:
  "填入你的机器人ID"

# 告警标题
feishualert_title:
  "前端程序4XX告警"

# 告警内容
feishualert_body:
  "
  【告警主题】:  {feishualert_title}\n
  【告警时间】:  {@timestamp}\n
  【告警域名】:  {source_hostname}\n
  【状态码】:  {response}\n
  【请求URL】:  {request}\n
  【请求协议】:  {verb}\n
  【全部信息】:  {message}\n
  【告警条件】:  3min 内 {response} 状态码 超过 {num_hits} 次
  "
```

配置istio-ingressgateway抓取日志

#添加filebeat的cm
kubectl create -f filebeat-istio.yml
```
filebeat.inputs:
- type: log
  paths:
   - /tmp/log/*
  fields:
    log_topic: istio-ingress
setup.template.enabled: false
setup.template.name: "istio-ingress"
setup.template.pattern: "istio-ingress-*"
setup.ilm.enabled: false

output.kafka:
  hosts: ["kafka.elastic:9092"]
  topic: '%{[fields][log_topic]}'
  max_message_bytes: 5242880
  partition.round_robin:
    reachable_only: false
  keep-alive: 120
  compression: gzip
  required_acks: 1
```

#修改istio的日志输出
kubectl edit cm -n istio-system istio
```
      8     accessLogFile: /tmp/log/istio.log
```

#修改POD添加filebeat的sidecar

```
#istio-ingressgateway
          image: 'docker.io/istio/proxyv2:1.15.2'
          volumeMounts:
            - mountPath: /tmp/log
              name: datalog
#filebeat
        - args:
            - '-c'
            - /data/filebeat-istio.yml
            - '-e'
          image: 'docker.elastic.co/beats/filebeat:8.13.1'
          imagePullPolicy: IfNotPresent
          name: filebeat-istio
          resources: {}
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          tty: true
          volumeMounts:
            - mountPath: /tmp/log/
              name: datalog
            - mountPath: /etc/localtime
              name: timezone
            - mountPath: /data/filebeat-istio.yml
              name: config
              readOnly: true
              subPath: filebeat-istio.yml
            - mountPath: /etc/certificate/ca.crt
              name: cert
              subPath: ca.crt
#volumes
      volumes:
        - emptyDir: {}
          name: datalog
        - name: cert
          secret:
            defaultMode: 420
            secretName: elastic-elasticsearch-master-crt
```

#重启等待完成,修改istio的日志回去
kubectl edit cm -n istio-system istio
```
      8     accessLogFile: /dev/stdout
```

排错思路

根据filebeat日志 ->  kafka检查topic -> logstash日志 -> es-codding日志 路径排错。

#pod
[root@master01 elastic]# kubectl get pod -n elastic
NAME                                              READY   STATUS    RESTARTS   AGE
elastalert2-7d4b7c6dd5-7648t                      1/1     Running   0          1h
elastic-elasticsearch-coordinating-0              1/1     Running   0          1h
elastic-elasticsearch-data-0                      1/1     Running   0          1h
elastic-elasticsearch-ingest-0                    1/1     Running   0          1h
elastic-elasticsearch-master-0                    1/1     Running   0          1h
elastic-elasticsearch-metrics-556b489666-kwl88    1/1     Running   0          1h
elastic-kibana-7864b77684-cpzfm                   1/1     Running   0          1h
elastic-nfs-client-provisioner-78f49b6798-cz4jj   1/1     Running   0          1h
kafka-broker-0                                    1/1     Running   0          1h
kafka-broker-1                                    1/1     Running   0          1h
kafka-broker-2                                    1/1     Running   0          1h
kafka-controller-0                                1/1     Running   0          1h
logstash-0                                        1/1     Running   0          1h

#svc
[root@master01 elastic]# kubectl get svc -n elastic
NAME                                    TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                      AGE
elastic-elasticsearch                   ClusterIP   10.200.28.154   <none>        9200/TCP,9300/TCP            1h
elastic-elasticsearch-coordinating-hl   ClusterIP   None            <none>        9200/TCP,9300/TCP            1h
elastic-elasticsearch-data-hl           ClusterIP   None            <none>        9200/TCP,9300/TCP            1h
elastic-elasticsearch-ingest-hl         ClusterIP   None            <none>        9200/TCP,9300/TCP            1h
elastic-elasticsearch-master-hl         ClusterIP   None            <none>        9200/TCP,9300/TCP            1h
elastic-elasticsearch-metrics           ClusterIP   10.200.240.82   <none>        9114/TCP                     1h
elastic-kibana                          NodePort    10.200.60.175   <none>        5601:30056/TCP               1h
kafka                                   ClusterIP   10.200.132.43   <none>        9092/TCP                     1h
kafka-broker-headless                   ClusterIP   None            <none>        9094/TCP,9092/TCP            1h
kafka-controller-headless               ClusterIP   None            <none>        9094/TCP,9092/TCP,9093/TCP   1h
logstash                                ClusterIP   10.200.67.109   <none>        8080/TCP                     1h
logstash-headless                       ClusterIP   None            <none>        8080/TCP                     1h

#cm
[root@master01 elastic]# kubectl get cm -n elastic
NAME                             DATA   AGE
elastalert2-config               1      1h
elastalert2-rules                1      1h
elastic-kibana-conf              1      1h
istio-ca-root-cert               1      1h
kafka-broker-configuration       1      1h
kafka-controller-configuration   1      1h
kafka-scripts                    1      1h
kube-root-ca.crt                 1      1h
logstash                         1      1h

#pv,pvc,sc
[root@master01 elastic]# kubectl get pv,pvc,sc -n elastic
NAME                                                        CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS     CLAIM                                                       STORAGECLASS            REASON   AGE
persistentvolume/elastic-pvc-elastalert2-pvc                2Gi        RWX            Retain           Bound      elastic/elastalert2-pvc                                     nfs                              1h
persistentvolume/elastic-pvc-geolite2city-pvc               2Gi        RWX            Retain           Bound      elastic/geolite2city-pvc                                    nfs                              1h
persistentvolume/pvc-09ec83e6-b014-4fd3-bff2-0e68033de6be   8Gi        RWO            Delete           Bound      elastic/data-kafka-broker-0                                 elasticsearch-sc                 1h
persistentvolume/pvc-396a3857-b8d0-43fc-b0a5-9014035e4f32   10Gi       RWO            Delete           Bound      elastic/elastic-kibana                                      elasticsearch-sc                 1h
persistentvolume/pvc-69fa8060-9583-407a-9649-f2415d443cf3   8Gi        RWO            Delete           Bound      elastic/data-kafka-broker-1                                 elasticsearch-sc                 1h
persistentvolume/pvc-b3f3d5d4-388b-47ae-9023-56411cfc75e0   8Gi        RWO            Delete           Bound      elastic/data-kafka-broker-2                                 elasticsearch-sc                 1h
persistentvolume/pvc-d99fce15-cc30-41ba-be34-d3488cf17f58   100Gi      RWO            Delete           Bound      elastic/data-elastic-elasticsearch-data-0                   elasticsearch-sc                 1h
persistentvolume/pvc-f3989dfa-c4f9-4a8f-981c-0f1cf17cbd74   8Gi        RWO            Delete           Bound      elastic/data-elastic-elasticsearch-master-0                 elasticsearch-sc                 1h
persistentvolume/pvc-fd1fd640-94ab-4404-a179-26267fb467be   8Gi        RWO            Delete           Bound      elastic/data-kafka-controller-0                             elasticsearch-sc                 1h

NAME                                                        STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS       AGE
persistentvolumeclaim/data-elastic-elasticsearch-data-0     Bound    pvc-d99fce15-cc30-41ba-be34-d3488cf17f58   100Gi      RWO            elasticsearch-sc   1h
persistentvolumeclaim/data-elastic-elasticsearch-master-0   Bound    pvc-f3989dfa-c4f9-4a8f-981c-0f1cf17cbd74   8Gi        RWO            elasticsearch-sc   1h
persistentvolumeclaim/data-kafka-broker-0                   Bound    pvc-09ec83e6-b014-4fd3-bff2-0e68033de6be   8Gi        RWO            elasticsearch-sc   1h
persistentvolumeclaim/data-kafka-broker-1                   Bound    pvc-69fa8060-9583-407a-9649-f2415d443cf3   8Gi        RWO            elasticsearch-sc   1h
persistentvolumeclaim/data-kafka-broker-2                   Bound    pvc-b3f3d5d4-388b-47ae-9023-56411cfc75e0   8Gi        RWO            elasticsearch-sc   1h
persistentvolumeclaim/data-kafka-controller-0               Bound    pvc-fd1fd640-94ab-4404-a179-26267fb467be   8Gi        RWO            elasticsearch-sc   1h
persistentvolumeclaim/elastalert2-pvc                       Bound    elastic-pvc-elastalert2-pvc                2Gi        RWX            nas                1h
persistentvolumeclaim/elastic-kibana                        Bound    pvc-396a3857-b8d0-43fc-b0a5-9014035e4f32   10Gi       RWO            elasticsearch-sc   1h
persistentvolumeclaim/geolite2city-pvc                      Bound    elastic-pvc-geolite2city-pvc               2Gi        RWX            nas                1h

NAME                                                         PROVISIONER                       RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION   AGE
storageclass.storage.k8s.io/elasticsearch-sc                 elastic-storage-class             Delete          Immediate              true                   1h

#secrets
[root@master01 elastic]# kubectl get secrets -n elastic
NAME                                     TYPE                 DATA   AGE
elastic-elasticsearch                    Opaque               1      1h
elastic-elasticsearch-coordinating-crt   kubernetes.io/tls    3      1h
elastic-elasticsearch-data-crt           kubernetes.io/tls    3      1h
elastic-elasticsearch-ingest-crt         kubernetes.io/tls    3      1h
elastic-elasticsearch-master-crt         kubernetes.io/tls    3      1h
elastic-kibana                           Opaque               1      1h
kafka-kraft-cluster-id                   Opaque               1      1h
sh.helm.release.v1.elastalert2.v1        helm.sh/release.v1   1      1h
sh.helm.release.v1.elastic.v1            helm.sh/release.v1   1      1h
sh.helm.release.v1.elastic.v2            helm.sh/release.v1   1      1h
sh.helm.release.v1.kafka.v1              helm.sh/release.v1   1      1h
sh.helm.release.v1.kafka.v2              helm.sh/release.v1   1      1h
sh.helm.release.v1.kafka.v3              helm.sh/release.v1   1      1h
sh.helm.release.v1.kafka.v4              helm.sh/release.v1   1      1h
sh.helm.release.v1.kafka.v5              helm.sh/release.v1   1      1h
sh.helm.release.v1.kafka.v6              helm.sh/release.v1   1      1h
sh.helm.release.v1.logstash.v1           helm.sh/release.v1   1      1h

#nfs
[root@master01 elastic]# ls /nfs/elastic/
elastic-data-elastic-elasticsearch-data-0-pvc-d99fce15-cc30-41ba-be34-d3488cf17f58    elastic-data-kafka-broker-2-pvc-b3f3d5d4-388b-47ae-9023-56411cfc75e0
elastic-data-elastic-elasticsearch-master-0-pvc-f3989dfa-c4f9-4a8f-981c-0f1cf17cbd74  elastic-data-kafka-controller-0-pvc-fd1fd640-94ab-4404-a179-26267fb467be
elastic-data-kafka-broker-0-pvc-09ec83e6-b014-4fd3-bff2-0e68033de6be                  elastic-elastic-kibana-pvc-396a3857-b8d0-43fc-b0a5-9014035e4f32
elastic-data-kafka-broker-1-pvc-69fa8060-9583-407a-9649-f2415d443cf3                  
elastalert2  GeoLite2-City

kibana创建Discover

登录kibana,使用elastic用户登录

左侧导航栏打开最后一个Stack Management,点击Index Management查看是否存在index,如果不存在,参考上面的排错思路一个个检查处理。

点击左侧 Data Views

创建一个Data Views,关联logstash-istio-ingress即可。

点击导航栏,打开Discover,查看你的日志。

1.打开index 2.筛选mesaage或者其它字段展示,会很精准。

一个简单的EFK环境就部署好了。

明天把istio加上去。

五:Elasticsearch跨集群index索引数据迁移

1.reindex迁移方式

注:我的两个es是同一个k8s,所以ip地址是内网的。

es原集群es目标集群
http://10.200.111.171:9200https://10.200.28.154:9200
es7.10.0es8.13.3
ns :elastic-clusterns: elastic

 1.目标集群配置白名单

#目标集群操作
#我的es是bitnami helm安装的,elasticsearch.yaml文件未挂载,所以需要修改value.yaml
#如果你的其它方式购买的,就修改elasticsearch.yaml就行了,哪种都需要重启

#修改文件
vim elasticsearch/values.yaml
```
 110 extraConfig:
 111   reindex.remote.whitelist: ["10.200.111.171:9200"]
```

#更新
helm upgrade -n elastic elastic elasticsearch/

2. tls证书挂载 (原TLS模式需要)

如果嫌麻烦直接目标es集群内一次性的vim ca.crt也是可以的,不用那么死板

#原es拿到ca的值
kubectl get secrets -n elastic-cluster "es证书tls密钥" -o yaml >> elastic-cluster-ca.yaml

#删除掉其它的,只保留ca.crt
vim elastic-cluster-ca.yaml

#目标es创建cm 
kubectl create -f elastic-cluster-ca.yaml
```
apiVersion: v1
kind: Secret
metadata:
  name: elastic-cluster-ca
  namespace: elastic
type: kubernetes.io/tls
data:
  ca.crt: XXXXXX==

```

#挂载到目标集群coordinating-0,同时也在这里操作
kubectl edit pod -n elastic elastic-elasticsearch-coordinating-0
#添加 格式按照yaml格式调下
```
volumeMounts:
- mountPath: /etc/certs
      name: elastic-system-ca
      readOnly: true
volumes:
  - name: elastic-system-ca
    secret:
      defaultMode: 256
      secretName: elastic-system-ca

```

 3.kibana dev tools可视化操作

#具体代码
POST _reindex?wait_for_completion=false
{
  "source": {
    "remote": {
      "host": "http://10.200.111.171:9200",
      "username": "elastic",
      "password": "123123",
      "ssl": {  
        "certificate_authorities": "none",  # 使用 "certificate" 来验证证书  
        "ca_file": "/etc/cert/ca.crt"  # CA证书附上,我演示的是没有tls的,删掉ssl{}就行。
      }
    },
    "index": "INNDEX_NAME",
    "size": 10000
  },
  "dest": {
    "index": "INNDEX_NAME_NEW" #目标为空可以和原index一样
  },
  "script": {
    "source": "ctx._source.remove('@timestamp')" #移除字段
  }
}


#查看任务
GET _tasks/-F7dEMs-TReMFqqy81x_GQ:32154

#查看索引和大小
GET _cat/indices

4.shell批量导入

调试不易,点赞再拿!!!

#脚本文件包括抓取需要迁移的index并通过for循环持续迁移到目标es集群。
cat reindex.sh
```
#!/bin/bash 

#迁移es脚本 2024-4-11 lucas

set -x

#需要迁移的索引名
INDEX_NAME='istio-2023'

#筛选出具体索引名
OLD_INDEX=$(curl -k -u elastic:123123 -X GET  "http://10.200.111.171:9200/_cat/indices" | grep ${INDEX_NAME} | awk '{print $3}')

#筛选出具体索引名 -ssl方式
#OLD_INDEX=$(curl -u username:password --cacert /path/to/old_ca.crt  'https://10.200.111.171:9200/_cat/indices' | grep ${INDEX_NAME} | awk '{print $3}')

for INDEX_NAME in ${OLD_INDEX}
do
  curl -X POST "https://10.200.28.154:9200/_reindex" \
    -H 'Content-Type: application/json' \
    -u "elastic:123123" \
    --cacert /path/to/new_ca.crt \
    --insecure \
    -d "
    {
      \"source\": {
        \"remote\": {
          \"host\": \"http://10.200.111.171:9200\",
          \"username\": \"elastic\",
          \"password\": \"123123\"
        },
        \"index\": \"${INDEX_NAME}\",
        \"size\": 10000
      },
      \"dest\": {
        \"index\": \"${INDEX_NAME}\"
      }
    }"
done
```

#如果原es集群是tls,则根据上面给的devtools给的照葫芦画瓢改就行了

脚本亲测同名的索引会自动覆盖不会报错,这点要小心的。

执行完后查看索引条目对比下

版本不一样,索引内容不一样,大小会有出入,但是我只需要条目对就行了,因为我没有日志切割。

Logo

K8S/Kubernetes社区为您提供最前沿的新闻资讯和知识内容

更多推荐