k8s1.29.0 集群部署
k8s1.29.0 集群部署，kubeadm 3master+3slave keepalived+haproxy flannel0.24-host-gw calico3.27-bgp-rr istio1.21.1 efk系统 elasticsearch8.13.1+logstash8.13.1+kibana8.13.1+filebeat8.13.1+kafka-cluster3.7.0+elast
·
修改历史
2024.3.27 部署高可用k8s lucas
2024.3.28 部署flannel host-gw lucas
2024.3.29 部署calico bgp-rr，移动参考链接到对应步骤下 lucas
2024.4.8 部署efk集群抓取istio日志 lucas
2024.4.11 reindexes跨集群索引迁移 lucas
2024.4.12 istio部署 lucas
一：服务器环境
本地虚拟机环境 centos7.9 NAT模式
192.168.178.138 master01 （双网卡ens33 VIP192.168.178.141） 操作机
192.168.178.139 master02
192.168.178.140 master03192.168.178.142 slave01
192.168.178.143 slave02
192.168.178.144 slave03
192.168.178.129 Calico虚拟路由
准备工作
#设置主机名
hostnamectl hostname=XXX
#配置免密(包括操作机)
ssh-keygen
ssh-copy-id master*/slave*
#传输hosts
cat > /etc/hosts <<EOF
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.178.138 master01
192.168.178.139 master02
192.168.178.140 master03
192.168.178.142 slave01
192.168.178.143 slave02
192.168.178.144 slave03
EOF
scp /etc/hosts master/slave:/etc/
---
#如果到这都看不懂就放弃吧初始化（所有服务器）
# 更新
yum update -y
# 卸载 firewalld
systemctl stop firewalld
yum remove firewalld -y
# 卸载 networkmanager
systemctl stop NetworkManager
yum remove NetworkManager -y
# 同步服务器时间
yum install chrony -y
systemctl enable --now chronyd
chronyc sources
# 关闭 selinux
setenforce 0
sed -i '/^SELINUX=/cSELINUX=disabled' /etc/selinux/config
getenforce
# 关闭swap分区
swapoff -a # 临时
sed -i '/ swap / s/^/# /g' /etc/fstab #永久
# 安装常用工具包
yum install -y net-tools sysstat vim wget lsof unzip zip bind-utils lrzsz telnet bash-completion
# 安装ipvs
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules
bash /etc/sysconfig/modules/ipvs.modules
lsmod | grep -e ip_vs -e nf_conntrack_ipv4
yum install ipset ipvsadm -y
# 允许检查桥接流量
cat <<EOF | tee /etc/modules-load.d/k8s.conf
br_netfilter
EOF
sysctl --system
cat <<EOF | tee /etc/sysctl.d/k8s.conf
vm.swappiness = 0
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF
modprobe br_netfilter
lsmod | grep netfilter
sysctl net.bridge.bridge-nf-call-iptables net.bridge.bridge-nf-call-ip6tables net.ipv4.ip_forward
# 安装containerd
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum list containerd.io --showduplicates
yum install -y containerd.io
mkdir -p /etc/containerd
containerd config default | tee /etc/containerd/config.toml
#crictl命令配置
cat <<EOF | tee /etc/crictl.yaml
runtime-endpoint: "unix:///run/containerd/containerd.sock"
image-endpoint: "unix:///run/containerd/containerd.sock"
timeout: 10
debug: false
pull-image-on-create: false
disable-pull-on-run: false
EOF
#使用阿里云镜像下载
sed -i "s#registry.k8s.io#registry.aliyuncs.com/google_containers#g" /etc/containerd/config.toml
#使用如下命令修改cgroup驱动
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
systemctl daemon-reload
systemctl enable containerd
systemctl restart containerd
# 添加kubernetes yum软件源
cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.29/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v1.29/rpm/repodata/repomd.xml.key
exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni
EOF
# 安装kubeadm,kubelet和kubectl
yum list kubeadm --showduplicates
yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
# 设置开机自启
systemctl daemon-reload
systemctl enable --now kubelet
# kubelet每隔几秒就会重启，陷入等待 kubeadm 指令的死循环
# 命令自动补全
yum install -y bash-completion
source <(crictl completion bash)
crictl completion bash >/etc/bash_completion.d/crictl
source <(kubectl completion bash)
kubectl completion bash >/etc/bash_completion.d/kubectl
source /usr/share/bash-completion/bash_completion
# NFS配置（忽略）
yum install -y nfs-common nfs-utils rpcbind
mkdir /nfs
chown nfsnobody /nfs/
#配置
vim /etc/exports
/nfs *(rw,no_root_squash,sync)
exportfs -r
systemctl start rpcbind && systemctl enable rpcbind
systemctl start nfs && systemctl enable nfs
showmount -eAPI server高可用部署
#安装 haproxy
yum -y install haproxy
#配置 haproxy
tee /etc/haproxy/haproxy.cfg <<EOF
global
log 127.0.0.1 local2
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 4000
user haproxy
group haproxy
daemon
stats socket /var/lib/haproxy/stats
defaults
mode http
log global
option httplog
option dontlognull
option http-server-close
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 1m
timeout http-keep-alive 10s
timeout check 10s
maxconn 3000
listen k8s-apiserver
bind *:8443
mode tcp
timeout client 1h
timeout connect 1h
log global
option tcplog
balance roundrobin
server master01 192.168.178.138:6443 check
server master02 192.168.178.139:6443 check
server master03 192.168.178.140:6443 check
acl is_websocket hdr(Upgrade) -i WebSocket
acl is_websocket hdr_beg(Host) -i ws
EOF
#运行 haproxy
systemctl enable --now haproxy
#安装 keepalived
yum -y install keepalived
#配置 keepalived
tee > /etc/keepalived/keepalived.conf <<EOF
global_defs {
router_id 100
vrrp_version 2
vrrp_garp_master_delay 1
vrrp_mcast_group4 224.0.0.18 #后续podSubnet需要一致
}
vrrp_script chk_haproxy {
script "/usr/bin/nc -nvz -w 2 127.0.0.1 8443"
timeout 1
interval 1 # check every 1 second
fall 2 # require 2 failures for KO
rise 2 # require 2 successes for OK
}
vrrp_instance lb-vips {
state MASTER
interface ens33 #VIP网卡名字
virtual_router_id 100
priority 150
advert_int 1
nopreempt
track_script {
chk_haproxy
}
authentication {
auth_type PASS
auth_pass blahblah
}
virtual_ipaddress {
192.168.178.141/24 dev eth0 #VIP的IP
}
}
EOF
#运行 keepalived
systemctl enable --now keepalived
#检查 vip 的情况
ip a
journalctl -fu keepalived生成 kubeadm 默认配置文件 kubeadm-config.yaml
照抄：
1. 二、安装 Kubernetes 高可用集群 · Istio实战指南
2.利用 kubeadm 创建高可用集群 | Kubernetes
修改过的已经加#，具体以你自己的版本为主。
#生成 kubeadm 默认配置文件
kubeadm config print init-defaults --component-configs \
KubeProxyConfiguration,KubeletConfiguration > kubeadm-config.yaml
#修改后完整的配置文件
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.168.178.138 #操作机IP
bindPort: 6443
nodeRegistration:
criSocket: unix:///var/run/containerd/containerd.sock
imagePullPolicy: IfNotPresent
name: master01
taints: null
---
controlPlaneEndpoint: 192.168.178.141:8443 #VIP
apiServer:
timeoutForControlPlane: 4m0s
extraArgs:
authorization-mode: "Node,RBAC"
enable-admission-plugins: "DefaultIngressClass,DefaultStorageClass,DefaultTolerationSeconds,LimitRanger,MutatingAdmissionWebhook,NamespaceLifecycle,PersistentVolumeClaimResize,PodSecurity,Priority,ResourceQuota,RuntimeClass,ServiceAccount,StorageObjectInUseProtection,TaintNodesByCondition,ValidatingAdmissionPolicy,ValidatingAdmissionWebhook" #准入控制
etcd-servers: https://master01:2379,https://master02:2379,https://master03:2379 #master节点
certSANs:
- 192.168.178.141 # VIP 地址
- 10.96.0.1 # service cidr的第一个ip
- 127.0.0.1 # 多个master的时候负载均衡出问题了能够快速使用localhost调试
- master01
- master02
- master03
- kubernetes
- kubernetes.default
- kubernetes.default.svc
- kubernetes.default.svc.cluster.local
extraVolumes:
- hostPath: /etc/localtime
mountPath: /etc/localtime
name: timezone
readOnly: true
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
local:
dataDir: /var/lib/etcd
serverCertSANs: #证书分发
- master01
- master02
- master03
peerCertSANs:
- master01
- master02
- master03
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: 1.29.0
networking:
dnsDomain: cluster.local
serviceSubnet: 10.96.0.0/12
podSubnet: 10.244.0.0/16 #跟keep alive保持一致
scheduler:
extraVolumes:
- hostPath: /etc/localtime
mountPath: /etc/localtime
name: timezone
readOnly: true
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
bindAddressHardFail: false
clientConnection:
acceptContentTypes: ""
burst: 0
contentType: ""
kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
qps: 0
clusterCIDR: ""
configSyncPeriod: 0s
conntrack:
maxPerCore: null
min: null
tcpBeLiberal: false
tcpCloseWaitTimeout: null
tcpEstablishedTimeout: null
udpStreamTimeout: 0s
udpTimeout: 0s
detectLocal:
bridgeInterface: ""
interfaceNamePrefix: ""
detectLocalMode: ""
enableProfiling: false
healthzBindAddress: ""
hostnameOverride: ""
iptables:
localhostNodePorts: null
masqueradeAll: false
masqueradeBit: null
minSyncPeriod: 0s
syncPeriod: 0s
ipvs:
excludeCIDRs: null
minSyncPeriod: 0s
scheduler: ""
strictARP: false
syncPeriod: 0s
tcpFinTimeout: 0s
tcpTimeout: 0s
udpTimeout: 0s
kind: KubeProxyConfiguration
logging:
flushFrequency: 0
options:
json:
infoBufferSize: "0"
verbosity: 0
metricsBindAddress: ""
mode: "ipvs" #IPVS模式
nftables:
masqueradeAll: false
masqueradeBit: null
minSyncPeriod: 0s
syncPeriod: 0s
nodePortAddresses: null
oomScoreAdj: null
portRange: ""
showHiddenMetricsForVersion: ""
winkernel:
enableDSR: false
forwardHealthCheckVip: false
networkName: ""
rootHnsEndpointName: ""
sourceVip: ""
---
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 0s
enabled: true
x509:
clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
mode: Webhook
webhook:
cacheAuthorizedTTL: 0s
cacheUnauthorizedTTL: 0s
cgroupDriver: systemd #systemd模式
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
containerRuntimeEndpoint: ""
cpuManagerReconcilePeriod: 0s
evictionPressureTransitionPeriod: 0s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageMaximumGCAge: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
logging:
flushFrequency: 0
options:
json:
infoBufferSize: "0"
verbosity: 0
memorySwap: {}
nodeStatusReportFrequency: 0s
nodeStatusUpdateFrequency: 0s
rotateCertificates: true
runtimeRequestTimeout: 0s
shutdownGracePeriod: 0s
shutdownGracePeriodCriticalPods: 0s
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
volumeStatsAggPeriod: 0s
#验证语法
kubeadm init --config kubeadm-config.yaml --dry-run
#预先拉取镜像
kubeadm config images pull --config kubeadm-config.yaml
#其它节点
kubeadm config images pull
#初始化集群
kubeadm init --config kubeadm-config.yaml --upload-certs
--upload-certs 标志用来将在所有控制平面实例之间的共享证书上传到集群。
等待初始化完成信息如下
···
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
Alternatively, if you are the root user, you can run:
export KUBECONFIG=/etc/kubernetes/admin.conf
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
You can now join any number of the control-plane node running the following command on each as root:
kubeadm join 192.168.178.141:8443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:0bc9dd684a2e3e1417e85765ef826208d2acfdbc530b6d641bb7f09e3a7e069f \
--control-plane --certificate-key 1c5a48c6d5ea3765c69f42458fda18381752a16618796be6798e117d4cc55ac3
Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 192.168.178.141:8443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:0bc9dd684a2e3e1417e85765ef826208d2acfdbc530b6d641bb7f09e3a7e069f
···
#master节点加入
[root@master02 ~]# kubeadm join 192.168.178.141:8443 --token abcdef.0123456789abcdef --discovery-token-ca-cert-hash sha256:0bc9dd684a2e3e1417e85765ef826208d2acfdbc530b6d641bb7f09e3a7e069f --control-plane --certificate-key 1c5a48c6d5ea3765c69f42458fda18381752a16618796be6798e117d4cc55ac3
#slave节点加入
kubeadm join 192.168.178.141:8443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:0bc9dd684a2e3e1417e85765ef826208d2acfdbc530b6d641bb7f09e3a7e069f
#控制节点检查
kubectl get nodes二：网络部署
flannel host-gw模式
#下载 For Kubernetes v1.17+ 对于 Kubernetes v1.17+
wget https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml
#编辑
vim kube-flannel.yml
89 net-conf.json: |
90 {
91 "Network": "10.244.0.0/16",
92 "Backend": {
93 "Type": "host-gw"
94 }
95 }
#安装前，运行下面命令
ip addr show
route -n
#安装
kubectl create -f kube-flannel.yml
#查看
kubectl get all -n kube-flannel
···
NAME READY STATUS RESTARTS AGE
pod/kube-flannel-ds-4wl74 1/1 Running 0 17m
pod/kube-flannel-ds-r7j2n 1/1 Running 0 17m
pod/kube-flannel-ds-w79wg 1/1 Running 0 17m
···
#检验
ip addr show
route -n
用安装前后的命令行输出来看结果
route -n：
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.178.2 0.0.0.0 UG 0 0 0 ens33
10.244.0.0 0.0.0.0 255.255.255.0 U 0 0 0 cni0
10.244.1.0 192.168.178.139 255.255.255.0 UG 0 0 0 ens33
10.244.2.0 192.168.178.140 255.255.255.0 UG 0 0 0 ens33
169.254.0.0 0.0.0.0 255.255.0.0 U 1002 0 0 ens32
169.254.0.0 0.0.0.0 255.255.0.0 U 1003 0 0 ens33
192.168.178.0 0.0.0.0 255.255.255.0 U 0 0 0 ens32
192.168.178.0 0.0.0.0 255.255.255.0 U 0 0 0 ens33
和ip addr show：
cni0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
link/ether 7e:95:b7:11:3e:f6 brd ff:ff:ff:ff:ff:ff
inet 10.244.0.1/24 brd 10.244.0.255 scope global cni0
valid_lft forever preferred_lft forever
inet6 fe80::7c95:b7ff:fe11:3ef6/64 scope link
valid_lft forever preferred_lft forever
对应的10.244.0.0/23 10.244.1.0/24 10.244.2.0/24 归属于10.244.0.0/12 CIDR地址块
可知部署正常，也可以安装pod互相调用containerIP即可知道了。
calico BGP模式
支持的版本
我们针对以下 Kubernetes 版本测试 Calico v3.27。 其他版本可能也有效，但我们没有积极测试它们。
v1.27
v1.28
v1.29
#安装calicoctl https://docs.tigera.io/calico/latest/operations/calicoctl/install
curl -L https://github.com/projectcalico/calico/releases/download/v3.27.2/calicoctl-linux-amd64 -o calicoctl
chmod +x calicoctl
cp calicoctl /usr/bin/calicoctl
calicoctl version
#下载calico https://docs.tigera.io/calico/latest/getting-started/kubernetes/quickstart
wget https://raw.githubusercontent.com/projectcalico/calico/v3.27.2/manifests/tigera-operator.yaml
wget https://raw.githubusercontent.com/projectcalico/calico/v3.27.2/manifests/custom-resources.yaml
#修改custom-resources.yaml 部分内容
```
apiVersion: operator.tigera.io/v1
kind: Installation
metadata:
name: default
spec:
# 选择网卡-多网卡环境
calicoNetwork:
nodeAddressAutodetectionV4:
interface: ens32
# 设置cidr为k8s创建时的podSubnet
ipPools:
- blockSize: 26
cidr: 10.244.0.0/16
encapsulation: VXLANCrossSubnet
natOutgoing: Enabled
nodeSelector: all()
```
#安装calico
calicoctl create -f tigera-operator.yaml
calicoctl create -f custom-resources.yaml
#等待pod启动完成才行
kubectl get pod -n calico-apiserver -o wide
kubectl get pod -n calico-system -o wide
#此时的状态
[root@master01 ~]# calicoctl node status
Calico process is running.
IPv4 BGP status
+-----------------+-------------------+-------+----------+-------------+
| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO |
+-----------------+-------------------+-------+----------+-------------+
| 192.168.178.139 | node-to-node mesh | up | 13:11:48 | Established |
| 192.168.178.140 | node-to-node mesh | up | 13:11:48 | Established |
+-----------------+-------------------+-------+----------+-------------+
#修改为RR路由模式
#新建一个路由虚拟机192.168.178.129，因为端口会跟现有机器冲突。
#路由模式选择quagga,照抄https://blog.csdn.net/cloud_engineer/article/details/127706088
#也可以安装路由vyos-ISO：https://docs.vyos.io/en/latest/installation/install.html
#关闭防火墙
systemctl stop firewalld
yum remove firewalld -y
#安装quagga
yum install quagga telnet -y
cp /usr/share/doc/quagga-0.99.22.4/zebra.conf.sample /etc/quagga/zebra.conf
cp /usr/share/doc/quagga-0.99.22.4/bgpd.conf.sample /etc/quagga/bgpd.conf
cd /etc/quagga/
vim bgpd.conf
```
router bgp 63500
bgp router-id 192.168.178.129 #路由节点IP
network 192.168.178.0 #k8s集群路由
neighbor 192.168.178.138 remote-as 63500
neighbor 192.168.178.138 route-reflector-client
neighbor 192.168.178.139 remote-as 63500
neighbor 192.168.178.139 route-reflector-client
neighbor 192.168.178.140 remote-as 63500
neighbor 192.168.178.140 route-reflector-client
```
systemctl enable zebra --now
systemctl enable bgpd --now
vtysh
```
[root@localhost quagga]# vtysh
Hello, this is Quagga (version 0.99.22.4).
Copyright 1996-2005 Kunihiro Ishiguro, et al.
localhost.localdomain# configure t
localhost.localdomain(config)# hostname bgp-rr
bgp-rr(config)# int ens32
bgp-rr(config-if)# ip add 192.168.178.129/24
bgp-rr(config-if)# no shutdown
bgp-rr(config-if)# end
bgp-rr# wr
Building Configuration...
Can't open configuration file /etc/quagga/zebra.conf.EHOX90.
Can't open configuration file /etc/quagga/bgpd.conf.9AEAb1.
[OK]
```
#calico-rr模式部署
#配置文件configuration详解https://docs.tigera.io/calico/latest/reference/resources/bgpconfig
cat calico-bgp-configuration.yaml
```
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
#https://docs.tigera.io/calico/latest/reference/resources/bgpconfig 具体解释
logSeverityScreen: Info
nodeToNodeMeshEnabled: false
# nodeMeshMaxRestartTime: 120s
asNumber: 63500
serviceClusterIPs:
- cidr: 10.244.128.0/24
serviceExternalIPs:
- cidr: 10.244.129.0/24
listenPort: 178
bindMode: NodeIP
communities:
- name: bgp-large-community
value: 63400:300:100
prefixAdvertisements:
- cidr: 10.244.0.0/16
communities:
- bgp-large-community
- 63400:120
```
#配置文件peer详解https://docs.tigera.io/calico/latest/reference/resources/bgppeer
cat calico-bgp-peer.yaml
```
apiVersion: projectcalico.org/v3
kind: BGPPeer
metadata:
name: vmware-peer
spec:
peerIP: 192.168.178.129
keepOriginalNextHop: true
asNumber: 63500
nodeSelector: rack == 'vmwarepeer'
```
#node添加标签
kubectl label node -all rack="vmwarepeer"
#部署
calicoctl create -f calico-bgp-configuration.yaml
calicoctl create -f calico-bgp-peer.yaml
#等待结果正常
```
[root@master01 ~]# calicoctl node status
Calico process is running.
IPv4 BGP status
+-----------------+---------------+-------+----------+-------------+
| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO |
+-----------------+---------------+-------+----------+-------------+
| 192.168.178.129 | node specific | up | 03:03:39 | Established |
+-----------------+---------------+-------+----------+-------------+
IPv6 BGP status
No IPv6 peers found.
```
#同时路由端
```
[root@localhost quagga]# vtysh
localhost.localdomain# show ip bgp summary
BGP router identifier 192.168.178.129, local AS number 63500
RIB entries 11, using 1232 bytes of memory
Peers 3, using 13 KiB of memory
Neighbor V AS MsgRcvd MsgSent TblVer InQ OutQ Up/Down State/PfxRcd
192.168.178.138 4 63500 59 55 0 0 0 00:48:47 5
192.168.178.139 4 63500 177 165 0 0 0 00:48:15 3
192.168.178.140 4 63500 176 173 0 0 0 00:48:48 3
Total number of neighbors 3
```
#这样的话我们的Calico-BGP-RR模式就部署好了
#测试的话可以tcpdump -i ens32 host container-ip
#也可以traceroute container-ip
#验证路由的响应速度和路线。
三：部署istio
版本支持Istio / 版本支持
#下载istio1.21.1版本
wget https://github.com/istio/istio/releases/download/1.21.1/istio-1.21.1-linux-amd64.tar.gz
#解压安装
tar -xzf istio-1.21.1-linux-amd64.tar.gz
cp istio-1.21.1/bin/istioctl /usr/bin/
istioctl manifest apply --set profile=demo
#安装kiali
cd istio/istio-1.21.1/samples/addons/
kubectl create -f jaeger.yaml -f kiali.yaml -f loki.yaml -f prometheus.yaml -f grafana.yaml
#等待启动完成即可
#访问kiali nodeport方式
kubectl edit svc -n istio-system kiali
```
42 type: NodePort
34 nodePort: 30021
```
#访问kiali，prometheus类似
http://192.168.178.138:30021/
#创建一个vs，gw，验收istio是否正常
vim nginx.yaml
```
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: nginx-deployment
spec:
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:latest
ports:
- containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
name: nginx-service
spec:
selector:
app: nginx
ports:
- protocol: TCP
port: 80
targetPort: 80
nodePort: 30080
type: NodePort
```
vim /root/nginx-vs.yaml
```
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
name: nginx-vs
namespace: default
spec:
gateways:
- nginx-vs-gateway
hosts:
- nginx.test.com
http:
- match:
- uri:
prefix: /api/
rewrite:
uri: /
route:
- destination:
host: java.java.svc.cluster.local
port:
number: 9999
- match:
- uri:
prefix: /
route:
- destination:
host: nginx-service.default.svc.cluster.local
port:
number: 80
```
vim nginx-gw.yaml
```
apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
name: nginx-vs-gateway
namespace: default
spec:
selector:
istio: ingressgateway
servers:
- hosts:
- nginx.test.com
port:
name: http
number: 80
protocol: HTTP
# tls:
# httpsRedirect: true
# - hosts:
# - nginx.test.com
# port:
# name: https
# number: 443
# protocol: HTTPS
# tls:
# credentialName: nginx-test-com-crt
# mode: SIMPLE
```
#安装
kubectl create -f nginx.yaml -f nginx-vs.yaml -f nginx-gw.yaml
#查看vs,gw
[root@master01 conf.d]# kubectl get virtualservices,gateways -n default
NAME GATEWAYS HOSTS AGE
virtualservice.networking.istio.io/nginx-vs ["nginx-vs-gateway"] ["nginx.test.com"] 21m
NAME AGE
gateway.networking.istio.io/nginx-vs-gateway 23m
#查看istio的svc
[root@master01 conf.d]# kubectl get svc -n istio-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
istio-egressgateway ClusterIP 10.103.138.124 <none> 80/TCP,443/TCP 51m
istio-ingressgateway LoadBalancer 10.105.22.202 <pending> 15021:30854/TCP,80:30306/TCP,443:31237/TCP,31400:30331/TCP,15443:30332/TCP 51m
istiod ClusterIP 10.107.108.221 <none> 15010/TCP,15012/TCP,443/TCP,15014/TCP 52m
jaeger-collector ClusterIP 10.104.53.49 <none> 14268/TCP,14250/TCP,9411/TCP,4317/TCP,4318/TCP 42m
kiali NodePort 10.111.51.85 <none> 20001:30021/TCP 42m
loki-headless ClusterIP None <none> 3100/TCP 42m
tracing ClusterIP 10.106.95.240 <none> 80/TCP,16685/TCP 42m
zipkin ClusterIP 10.102.223.213 <none> 9411/TCP 42m
#访问配置，因为没有LB，所以使用nginx代理istio-ingress80端口，才能正常访问，否则就使用metallb
#k8s外部安装nginx
yum install epel-release
yum install nginx -y
vim /etc/nginx/conf.d/nginx.conf
```
upstream nginx_http {
server 192.168.178.138:30306;
server 192.168.178.139:30306;
server 192.168.178.140:30306;
}
server {
listen 80;
server_name nginx.test.com;
location / {
index index.jsp index.html index.htm;
proxy_pass http://nginx_http;
proxy_redirect off;
proxy_set_header X-Real_IP $remote_addr;
proxy_set_header Host $host;
proxy_set_header X_Forward_For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header X-Request-ID $request_id;
proxy_next_upstream error timeout;
client_max_body_size 20m;
proxy_read_timeout 1200s;
}
}
```
#自己电脑
#C:\Windows\System32\drivers\etc
#编辑hosts文件
```
192.168.178.138 nginx.test.com
```
#然后本机访问即可
http://nginx.test.com/
http://nginx.test.com/api
#虽然现在装了istio-kiali，但是没有pod注册进来，无法查看，所以需要注入配置
#注入
kubectl label namespace <namespace> istio-injection=enabled
#取消注入
kubectl label namespace <namespace> istio-injection=disabled
#查看注入
kubectl get pod <pod-name> -n <namespace> -o jsonpath='{.spec.containers[*].name}'
#总结：
#Istio还有很多功能包括流量控制链路追踪等高阶操作，可以查看组件官网如何操作。四：部署EFK集群
helm部署elasticsearch8.13.1+logstash8.13.1+kibana8.13.1+filebeat8.13.1+kafka-cluster3.7.0+elastalert2
部署NFS存储efk应用数据
#nfs安装
#server
yum install -y nfs-common nfs-utils rpcbind
mkdir /nfs
chown nfsnobody /nfs/
#配置
vim /etc/exports
```
/nfs 192.168.0.0/24(rw,no_root_squash,sync)
```
exportfs -r
systemctl start rpcbind && systemctl enable rpcbind
systemctl start nfs && systemctl enable nfs
showmount -e
#client
yum install -y nfs-utils rpcbind
systemctl start nfs-utils.service rpcbind.service
systemctl enable nfs-utils.service rpcbind.service
showmount -e master01创建SC存储
#创建nfs的es文件夹
mkdir /nfs/elastic/
#rbac
cat rbac.yaml
```
apiVersion: v1
kind: ServiceAccount
metadata:
name: elastic-nfs-client-provisioner
namespace: elastic
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: elastic-nfs-client-provisioner-runner
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-elastic-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: elastic-nfs-client-provisioner
namespace: elastic
roleRef:
kind: ClusterRole
name: elastic-nfs-client-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-elastic-nfs-client-provisioner
namespace: elastic
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-elastic-nfs-client-provisioner
namespace: elastic
subjects:
- kind: ServiceAccount
name: elastic-nfs-client-provisioner
namespace: elastic
roleRef:
kind: Role
name: leader-locking-elastic-nfs-client-provisioner
apiGroup: rbac.authorization.k8s.io
```
#provisioner
cat provisioner.yaml
```
apiVersion: apps/v1
kind: Deployment # provisioner的类型是一个deployment
metadata:
name: elastic-nfs-client-provisioner
labels:
app: elastic-nfs-client-provisioner
namespace: elastic # 指定provisioner所属的namespace，改成你自己的namespace
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app: elastic-nfs-client-provisioner
template:
metadata:
labels:
app: elastic-nfs-client-provisioner
spec:
serviceAccountName: elastic-nfs-client-provisioner # 指定provisioner使用的sa
containers:
- name: elastic-nfs-client-provisioner
image: vbouchaud/nfs-client-provisioner:latest # 指定provisioner的镜像
volumeMounts:
- name: nfs-client-root
mountPath: /persistentvolumes # 固定写法
env:
- name: PROVISIONER_NAME
value: elastic-storage-class # 指定分配器的名称，创建storageclass会用到
- name: NFS_SERVER
value: 192.168.178.138 # 指定使用哪一块存储，这里用的是nfs，此处填写nfs的地址
- name: NFS_PATH
value: /nfs/elastic # 使用nfs哪一块盘符
volumes:
- name: nfs-client-root
nfs:
server: 192.168.178.138 # 和上面指定的nfs地址保持一致
path: /nfs/elastic # 和上面指定的盘符保持一致
```
#sc
cat sc.yaml
```
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: elasticsearch-sc
provisioner: elastic-storage-class
parameters:
archiveOnDelete: "true" ## 删除pv的时候，pv的内容是否要备份
allowVolumeExpansion: true
```
#创建
kubectl create -f rbac.yaml -f provisioner.yaml -f sc.yaml
#查看
kubectl get sc | grep elasticsearch-sc
elasticsearch-sc elastic-storage-class Delete Immediate true 2h
安装es+kibana
1.安装helm https://helm.sh/docs/intro/install/
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
chmod 700 get_helm.sh
./get_helm.sh
helm version
2.安装es+kibana 证书模式 https://artifacthub.io/packages/helm/bitnami/elasticsearch
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update
helm pull bitnami/elasticsearch --version 20.0.3
tar -xzf elasticsearch-20.0.3.tgz
#根据你的自定义修改 --我的示例
23 storageClass: "elasticsearch-sc"
29 kibanaEnabled: true
218 security:
221 enabled: true
225 elasticPassword: "123456"
233 fipsMode: false
236 tls:
239 restEncryption: true
244 autoGenerated: true
302 service:
305 type: ClusterIP
360 ingress:
363 enabled: false
462 master:
465 masterOnly: true
468 replicaCount: 1
512 requests:
513 cpu: 2
514 memory: 512Mi
515 limits:
516 cpu: 3
517 memory: 1024Mi
570 networkPolicy:
573 enabled: false
862 data:
865 replicaCount: 1
963 networkPolicy:
966 enabled: false
1255 coordinating:
1258 replicaCount: 1
1357 networkPolicy:
1360 enabled: false
1610 ingest:
1613 enabled: true
1616 replicaCount: 1
1720 networkPolicy:
1723 enabled: false
2143 metrics:
2146 enabled: true ##给prometheus的
2682 kibana:
2683 elasticsearch:
2684 hosts:
2685 - '{{ include "elasticsearch.service.name" . }}'
2686 port: '{{ include "elasticsearch.service.ports.restAPI" . }}'
2687 security:
2688 auth:
2689 enabled: true
2691 kibanaUsername: "kibana_system"
2692 kibanaPassword: "123456"
2693 tls:
2695 enabled: true
2697 existingSecret: elastic-elasticsearch-coordinating-crt
2699 usePemCerts: true
#创建ns
kubectl create ns elastic
#安装应用
helm install elastic -n elastic elasticsearch/
#修改kibana_system密码 --待优化的bug
#进入
kubectl exec -it -n elastic elastic-elasticsearch-coordinating-0 -- bash
#修改
elasticsearch-reset-password -u kibana_system --url "http://elastic-elasticsearch:9200"
#会生成一个随机密码类似=C=c1Kd1asLfMoF0fg4e，将密码修改
#拿到加密密码
echo =C=c1Kd1asLfMoF0fg4e | base64
#输出PUM9YzFLZDFhc0xmTW9GMGZnNGUK
#修改密码，把密码换上去
kubectl edit secrets -n elastic elastic-kibana
```
kibana-password: PUM9YzFLZDFhc0xmTW9GMGZnNGUK
```
#等待容器重启后修改kibana的svc
kubectl edit svc -n elastic elastic-kibana
```
ports:
- name: http
nodePort: 30056
port: 5601
protocol: TCP
targetPort: 5601
type: NodePort
```
#这时候就可以正常访问地址了
#访问http://192.168.178.138:30056安装kafka-cluster
#helm安装https://artifacthub.io/packages/helm/bitnami/kafka
helm repo add bitnami https://charts.bitnami.com/bitnami
helm pull bitnami/kafka --version 28.0.3
tar -xzf kafka-28.0.3.tgz
#根据你的自定义修改 --我的示例
vim kafka/values.yaml
```
21 storageClass: "elasticsearch-sc"
156 protocol: PLAINTEXT ##不使用sasl模式
167 protocol: PLAINTEXT
175 protocol: PLAINTEXT
184 protocol: PLAINTEXT
470 controller:
474 replicaCount: 3
884 broker:
887 replicaCount: 3
1634 networkPolicy:
1637 enabled: false
```
#安装
helm install -n elastic kafka kafka/
#查看topic和group命令和查看偏移量
kafka-topics.sh --list --bootstrap-server kafka-controller-0.kafka-controller-headless.elastic.svc.cluster.local:9092,kafka-broker-0.kafka-broker-headless.elastic.svc.cluster.local:9092,kafka-broker-1.kafka-broker-headless.elastic.svc.cluster.local:9092,kafka-broker-2.kafka-broker-headless.elastic.svc.cluster.local:9092
kafka-consumer-groups.sh --list --bootstrap-server kafka-controller-0.kafka-controller-headless.elastic.svc.cluster.local:9092,kafka-broker-0.kafka-broker-headless.elastic.svc.cluster.local:9092,kafka-broker-1.kafka-broker-headless.elastic.svc.cluster.local:9092,kafka-broker-2.kafka-broker-headless.elastic.svc.cluster.local:9092
kafka-consumer-groups.sh --describe --group logstash --bootstrap-server kafka-controller-0.kafka-controller-headless.elastic.svc.cluster.local:9092,kafka-broker-0.kafka-broker-headless.elastic.svc.cluster.local:9092,kafka-broker-1.kafka-broker-headless.elastic.svc.cluster.local:9092,kafka-broker-2.kafka-broker-headless.elastic.svc.cluster.local:9092
安装logstash
#helm安装https://artifacthub.io/packages/helm/bitnami/logstash
helm repo add bitnami https://charts.bitnami.com/bitnami
helm pull bitnami/logstash --version 6.0.2
tar -xzf logstash-6.0.2.tgz
#根据你的自定义修改 --我的示例
vim logstash/values.yaml
```
20 storageClass: "elasticsearch-sc"
518 networkPolicy:
521 enabled: false
```
#安装
helm install logstash -n elastic logstash/
安装elastalert2
#helm安装https://artifacthub.io/packages/helm/elastalert2/elastalert2
helm repo add elastalert2 https://jertel.github.io/elastalert2
helm pull elastalert2/elastalert2 --version 2.17.0
tar -xzf elastalert2-2.17.0.tgz
#修改
vim elastalert2/values.yaml
```
51行
elasticsearch:
host: elastic-elasticsearch.elastic
port: 9200
useSsl: "true"
username: "elastic"
password: "123456"
credentialsSecret: "elastic-elasticsearch-coordinating-crt"
credentialsSecretUsernameKey: ""
credentialsSecretPasswordKey: ""
verifyCerts: "True"
clientCert: "/certs/tls.crt"
clientKey: "/certs/tls.key"
caCerts: "/certs/ca.crt"
certsVolumes:
- name: es-certs
secret:
defaultMode: 420
secretName: elastic-elasticsearch-coordinating-crt
certsVolumeMounts:
- name: es-certs
mountPath: /certs
readOnly: true
```
#安装
helm install -n elastic elastalert2 elastalert2/安装飞书告警通知和IP归属地展示
提前下载文件elastalert.tar.gz 和GeoLite2-City.tar.gz
--私信我拿
#保存到文件
mkdir /nfs/elastic-system
chmod 777 elastalert2 GeoLite2-City -R创建pv,pvc
#创建elastalert2的pv,pvc
cat elastalert2-pv.yaml
```
apiVersion: v1
kind: PersistentVolume
metadata:
name: elastic-pvc-elastalert2-pvc
namespace: elastic
spec:
capacity:
storage: 2Gi
accessModes:
- ReadWriteMany
persistentVolumeReclaimPolicy: Retain
nfs:
server: 192.168.178.138
path: /nfs/elastic/elastalert2
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: elastalert2-pvc
namespace: elastic
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 2Gi
```
#创建geolite2city的pv,pvc
cat geolite2city-pv.yaml
```
apiVersion: v1
kind: PersistentVolume
metadata:
name: elastic-pvc-geolite2city-pvc
namespace: elastic
spec:
capacity:
storage: 2Gi
accessModes:
- ReadWriteMany
persistentVolumeReclaimPolicy: Retain
nfs:
server: 192.168.178.138
path: /nfs/elastic/GeoLite2-City
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: geolite2city-pvc
namespace: elastic
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 2Gi
```
#安装
kubectl create -f elastalert2-pv.yaml -f geolite2city-pv.yaml 挂载到POD
#elastalert2挂载飞书告警
kubectl edit pod -n elastic elastalert2-xxx-xxx
```
- mountPath: /usr/local/lib/python3.11/site-packages/elastalert/
name: volume
- name: volume
persistentVolumeClaim:
claimName: elastalert2-pvc
```
#GeoLite2-City挂载到logstash，顺便挂载es证书
kubectl edit pod -n elastic logstash-0
```
- mountPath: /mnt
name: volume
- mountPath: /etc/certificate/ca.crt
name: cert
subPath: ca.crt
- name: volume
persistentVolumeClaim:
claimName: geolite2city-pvc
- name: cert
secret:
defaultMode: 420
secretName: elastic-elasticsearch-coordinating-crt
```
修改logstash和elastalert2的CM
#修改logstash的cm --grok日志切割
kubectl exec -it -n elastic logstash-0 -- bash
vim /bitnami/logstash/config/logstash.conf
```
input {
kafka {
bootstrap_servers => "kafka.elastic:9092"
topics_pattern => "[a-z].*"
codec => "json"
}
}
filter {
if [fields][log_topic] == "istio-ingress" {
grok {
match => {
"message" => "\[%{TIMESTAMP_ISO8601:timestamp}\] \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}\" %{INT:response} %{NOTSPACE:upstream_header} %{GREEDYDATA:upstream_error} %{NOTSPACE:upstream_error1} \"%{NOTSPACE:referrer}\" %{INT:bytes_in} %{INT:bytes_out} %{INT:duration} %{NOTSPACE:duration2} \"%{NOTSPACE:client_ip}\" \"%{GREEDYDATA:user_agent}\" \"%{UUID:request_id}\" \"%{HOSTNAME:hostname}\" \"%{GREEDYDATA:x_forwarded_for}\" %{GREEDYDATA:outbound_route} %{IPORHOST:destination_ip}:%{INT:destination_port0} %{IPORHOST:destination_ip}:%{INT:destination_port} %{IPORHOST:source_ip}:%{INT:source_port} %{HOSTNAME:source_hostname} %{NOTSPACE:after}"
}
}
geoip {
source => "source_ip"
target => "geoip"
database => "/mnt/GeoLite2-City_20230707/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float"]
}
}
}
output {
elasticsearch {
hosts => ["https://elastic-elasticsearch.elastic.svc.cluster.local:9200"]
user => "elastic"
password => "123456"
index => "logstash-%{[fields][log_topic]}-%{+YYYY.MM.dd}"
ssl => "true"
cacert => "/etc/certificate/ca.crt"
}
}
```
#添加elastalert2的规则 --实例404告警
kubectl exec -it -n elastic elastalert2-xxx-xxx -- bash
vim rules/nginx_4xx.yaml
```
name: nginx_access_4xx
type: frequency
index: logstash-istio-ingress-*
is_enabled: true
num_events: 1
terms_size: 50
timeframe:
minutes: 3
timestamp_type: "iso"
use_strftime_index: false
filter:
- query:
query_string:
query: "response:4*"
include: ["@timestamp","source_hostname","response","verb","request","message"]
alert:
- "elastalert.elastalert_modules.feishu_alert.FeishuAlert"
# 飞书机器人接口地址
feishualert_url: "https://open.feishu.cn/open-apis/bot/v2/hook/"
# 飞书机器人id
feishualert_botid:
"填入你的机器人ID"
# 告警标题
feishualert_title:
"前端程序4XX告警"
# 告警内容
feishualert_body:
"
【告警主题】: {feishualert_title}\n
【告警时间】: {@timestamp}\n
【告警域名】: {source_hostname}\n
【状态码】: {response}\n
【请求URL】: {request}\n
【请求协议】: {verb}\n
【全部信息】: {message}\n
【告警条件】: 3min 内 {response} 状态码 超过 {num_hits} 次
"
```配置istio-ingressgateway抓取日志
#添加filebeat的cm
kubectl create -f filebeat-istio.yml
```
filebeat.inputs:
- type: log
paths:
- /tmp/log/*
fields:
log_topic: istio-ingress
setup.template.enabled: false
setup.template.name: "istio-ingress"
setup.template.pattern: "istio-ingress-*"
setup.ilm.enabled: false
output.kafka:
hosts: ["kafka.elastic:9092"]
topic: '%{[fields][log_topic]}'
max_message_bytes: 5242880
partition.round_robin:
reachable_only: false
keep-alive: 120
compression: gzip
required_acks: 1
```
#修改istio的日志输出
kubectl edit cm -n istio-system istio
```
8 accessLogFile: /tmp/log/istio.log
```
#修改POD添加filebeat的sidecar
```
#istio-ingressgateway
image: 'docker.io/istio/proxyv2:1.15.2'
volumeMounts:
- mountPath: /tmp/log
name: datalog
#filebeat
- args:
- '-c'
- /data/filebeat-istio.yml
- '-e'
image: 'docker.elastic.co/beats/filebeat:8.13.1'
imagePullPolicy: IfNotPresent
name: filebeat-istio
resources: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
tty: true
volumeMounts:
- mountPath: /tmp/log/
name: datalog
- mountPath: /etc/localtime
name: timezone
- mountPath: /data/filebeat-istio.yml
name: config
readOnly: true
subPath: filebeat-istio.yml
- mountPath: /etc/certificate/ca.crt
name: cert
subPath: ca.crt
#volumes
volumes:
- emptyDir: {}
name: datalog
- name: cert
secret:
defaultMode: 420
secretName: elastic-elasticsearch-master-crt
```
#重启等待完成，修改istio的日志回去
kubectl edit cm -n istio-system istio
```
8 accessLogFile: /dev/stdout
```
排错思路
根据filebeat日志 -> kafka检查topic -> logstash日志 -> es-codding日志 路径排错。
#pod
[root@master01 elastic]# kubectl get pod -n elastic
NAME READY STATUS RESTARTS AGE
elastalert2-7d4b7c6dd5-7648t 1/1 Running 0 1h
elastic-elasticsearch-coordinating-0 1/1 Running 0 1h
elastic-elasticsearch-data-0 1/1 Running 0 1h
elastic-elasticsearch-ingest-0 1/1 Running 0 1h
elastic-elasticsearch-master-0 1/1 Running 0 1h
elastic-elasticsearch-metrics-556b489666-kwl88 1/1 Running 0 1h
elastic-kibana-7864b77684-cpzfm 1/1 Running 0 1h
elastic-nfs-client-provisioner-78f49b6798-cz4jj 1/1 Running 0 1h
kafka-broker-0 1/1 Running 0 1h
kafka-broker-1 1/1 Running 0 1h
kafka-broker-2 1/1 Running 0 1h
kafka-controller-0 1/1 Running 0 1h
logstash-0 1/1 Running 0 1h
#svc
[root@master01 elastic]# kubectl get svc -n elastic
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
elastic-elasticsearch ClusterIP 10.200.28.154 <none> 9200/TCP,9300/TCP 1h
elastic-elasticsearch-coordinating-hl ClusterIP None <none> 9200/TCP,9300/TCP 1h
elastic-elasticsearch-data-hl ClusterIP None <none> 9200/TCP,9300/TCP 1h
elastic-elasticsearch-ingest-hl ClusterIP None <none> 9200/TCP,9300/TCP 1h
elastic-elasticsearch-master-hl ClusterIP None <none> 9200/TCP,9300/TCP 1h
elastic-elasticsearch-metrics ClusterIP 10.200.240.82 <none> 9114/TCP 1h
elastic-kibana NodePort 10.200.60.175 <none> 5601:30056/TCP 1h
kafka ClusterIP 10.200.132.43 <none> 9092/TCP 1h
kafka-broker-headless ClusterIP None <none> 9094/TCP,9092/TCP 1h
kafka-controller-headless ClusterIP None <none> 9094/TCP,9092/TCP,9093/TCP 1h
logstash ClusterIP 10.200.67.109 <none> 8080/TCP 1h
logstash-headless ClusterIP None <none> 8080/TCP 1h
#cm
[root@master01 elastic]# kubectl get cm -n elastic
NAME DATA AGE
elastalert2-config 1 1h
elastalert2-rules 1 1h
elastic-kibana-conf 1 1h
istio-ca-root-cert 1 1h
kafka-broker-configuration 1 1h
kafka-controller-configuration 1 1h
kafka-scripts 1 1h
kube-root-ca.crt 1 1h
logstash 1 1h
#pv,pvc,sc
[root@master01 elastic]# kubectl get pv,pvc,sc -n elastic
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
persistentvolume/elastic-pvc-elastalert2-pvc 2Gi RWX Retain Bound elastic/elastalert2-pvc nfs 1h
persistentvolume/elastic-pvc-geolite2city-pvc 2Gi RWX Retain Bound elastic/geolite2city-pvc nfs 1h
persistentvolume/pvc-09ec83e6-b014-4fd3-bff2-0e68033de6be 8Gi RWO Delete Bound elastic/data-kafka-broker-0 elasticsearch-sc 1h
persistentvolume/pvc-396a3857-b8d0-43fc-b0a5-9014035e4f32 10Gi RWO Delete Bound elastic/elastic-kibana elasticsearch-sc 1h
persistentvolume/pvc-69fa8060-9583-407a-9649-f2415d443cf3 8Gi RWO Delete Bound elastic/data-kafka-broker-1 elasticsearch-sc 1h
persistentvolume/pvc-b3f3d5d4-388b-47ae-9023-56411cfc75e0 8Gi RWO Delete Bound elastic/data-kafka-broker-2 elasticsearch-sc 1h
persistentvolume/pvc-d99fce15-cc30-41ba-be34-d3488cf17f58 100Gi RWO Delete Bound elastic/data-elastic-elasticsearch-data-0 elasticsearch-sc 1h
persistentvolume/pvc-f3989dfa-c4f9-4a8f-981c-0f1cf17cbd74 8Gi RWO Delete Bound elastic/data-elastic-elasticsearch-master-0 elasticsearch-sc 1h
persistentvolume/pvc-fd1fd640-94ab-4404-a179-26267fb467be 8Gi RWO Delete Bound elastic/data-kafka-controller-0 elasticsearch-sc 1h
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
persistentvolumeclaim/data-elastic-elasticsearch-data-0 Bound pvc-d99fce15-cc30-41ba-be34-d3488cf17f58 100Gi RWO elasticsearch-sc 1h
persistentvolumeclaim/data-elastic-elasticsearch-master-0 Bound pvc-f3989dfa-c4f9-4a8f-981c-0f1cf17cbd74 8Gi RWO elasticsearch-sc 1h
persistentvolumeclaim/data-kafka-broker-0 Bound pvc-09ec83e6-b014-4fd3-bff2-0e68033de6be 8Gi RWO elasticsearch-sc 1h
persistentvolumeclaim/data-kafka-broker-1 Bound pvc-69fa8060-9583-407a-9649-f2415d443cf3 8Gi RWO elasticsearch-sc 1h
persistentvolumeclaim/data-kafka-broker-2 Bound pvc-b3f3d5d4-388b-47ae-9023-56411cfc75e0 8Gi RWO elasticsearch-sc 1h
persistentvolumeclaim/data-kafka-controller-0 Bound pvc-fd1fd640-94ab-4404-a179-26267fb467be 8Gi RWO elasticsearch-sc 1h
persistentvolumeclaim/elastalert2-pvc Bound elastic-pvc-elastalert2-pvc 2Gi RWX nas 1h
persistentvolumeclaim/elastic-kibana Bound pvc-396a3857-b8d0-43fc-b0a5-9014035e4f32 10Gi RWO elasticsearch-sc 1h
persistentvolumeclaim/geolite2city-pvc Bound elastic-pvc-geolite2city-pvc 2Gi RWX nas 1h
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
storageclass.storage.k8s.io/elasticsearch-sc elastic-storage-class Delete Immediate true 1h
#secrets
[root@master01 elastic]# kubectl get secrets -n elastic
NAME TYPE DATA AGE
elastic-elasticsearch Opaque 1 1h
elastic-elasticsearch-coordinating-crt kubernetes.io/tls 3 1h
elastic-elasticsearch-data-crt kubernetes.io/tls 3 1h
elastic-elasticsearch-ingest-crt kubernetes.io/tls 3 1h
elastic-elasticsearch-master-crt kubernetes.io/tls 3 1h
elastic-kibana Opaque 1 1h
kafka-kraft-cluster-id Opaque 1 1h
sh.helm.release.v1.elastalert2.v1 helm.sh/release.v1 1 1h
sh.helm.release.v1.elastic.v1 helm.sh/release.v1 1 1h
sh.helm.release.v1.elastic.v2 helm.sh/release.v1 1 1h
sh.helm.release.v1.kafka.v1 helm.sh/release.v1 1 1h
sh.helm.release.v1.kafka.v2 helm.sh/release.v1 1 1h
sh.helm.release.v1.kafka.v3 helm.sh/release.v1 1 1h
sh.helm.release.v1.kafka.v4 helm.sh/release.v1 1 1h
sh.helm.release.v1.kafka.v5 helm.sh/release.v1 1 1h
sh.helm.release.v1.kafka.v6 helm.sh/release.v1 1 1h
sh.helm.release.v1.logstash.v1 helm.sh/release.v1 1 1h
#nfs
[root@master01 elastic]# ls /nfs/elastic/
elastic-data-elastic-elasticsearch-data-0-pvc-d99fce15-cc30-41ba-be34-d3488cf17f58 elastic-data-kafka-broker-2-pvc-b3f3d5d4-388b-47ae-9023-56411cfc75e0
elastic-data-elastic-elasticsearch-master-0-pvc-f3989dfa-c4f9-4a8f-981c-0f1cf17cbd74 elastic-data-kafka-controller-0-pvc-fd1fd640-94ab-4404-a179-26267fb467be
elastic-data-kafka-broker-0-pvc-09ec83e6-b014-4fd3-bff2-0e68033de6be elastic-elastic-kibana-pvc-396a3857-b8d0-43fc-b0a5-9014035e4f32
elastic-data-kafka-broker-1-pvc-69fa8060-9583-407a-9649-f2415d443cf3
elastalert2 GeoLite2-City
kibana创建Discover
登录kibana，使用elastic用户登录
左侧导航栏打开最后一个Stack Management，点击Index Management查看是否存在index，如果不存在，参考上面的排错思路一个个检查处理。
点击左侧 Data Views
创建一个Data Views，关联logstash-istio-ingress即可。
点击导航栏，打开Discover，查看你的日志。
1.打开index 2.筛选mesaage或者其它字段展示，会很精准。
一个简单的EFK环境就部署好了。
明天把istio加上去。
五：Elasticsearch跨集群index索引数据迁移
1.reindex迁移方式
注：我的两个es是同一个k8s，所以ip地址是内网的。
| es原集群 | es目标集群 |
| http://10.200.111.171:9200 | https://10.200.28.154:9200 |
| es7.10.0 | es8.13.3 |
| ns :elastic-cluster | ns: elastic |
1.目标集群配置白名单
#目标集群操作
#我的es是bitnami helm安装的，elasticsearch.yaml文件未挂载，所以需要修改value.yaml
#如果你的其它方式购买的，就修改elasticsearch.yaml就行了，哪种都需要重启
#修改文件
vim elasticsearch/values.yaml
```
110 extraConfig:
111 reindex.remote.whitelist: ["10.200.111.171:9200"]
```
#更新
helm upgrade -n elastic elastic elasticsearch/
2. tls证书挂载 （原TLS模式需要）
如果嫌麻烦直接目标es集群内一次性的vim ca.crt也是可以的，不用那么死板
#原es拿到ca的值
kubectl get secrets -n elastic-cluster "es证书tls密钥" -o yaml >> elastic-cluster-ca.yaml
#删除掉其它的，只保留ca.crt
vim elastic-cluster-ca.yaml
#目标es创建cm
kubectl create -f elastic-cluster-ca.yaml
```
apiVersion: v1
kind: Secret
metadata:
name: elastic-cluster-ca
namespace: elastic
type: kubernetes.io/tls
data:
ca.crt: XXXXXX==
```
#挂载到目标集群coordinating-0,同时也在这里操作
kubectl edit pod -n elastic elastic-elasticsearch-coordinating-0
#添加 格式按照yaml格式调下
```
volumeMounts:
- mountPath: /etc/certs
name: elastic-system-ca
readOnly: true
volumes:
- name: elastic-system-ca
secret:
defaultMode: 256
secretName: elastic-system-ca
```
3.kibana dev tools可视化操作
#具体代码
POST _reindex?wait_for_completion=false
{
"source": {
"remote": {
"host": "http://10.200.111.171:9200",
"username": "elastic",
"password": "123123",
"ssl": {
"certificate_authorities": "none", # 使用 "certificate" 来验证证书
"ca_file": "/etc/cert/ca.crt" # CA证书附上，我演示的是没有tls的，删掉ssl{}就行。
}
},
"index": "INNDEX_NAME",
"size": 10000
},
"dest": {
"index": "INNDEX_NAME_NEW" #目标为空可以和原index一样
},
"script": {
"source": "ctx._source.remove('@timestamp')" #移除字段
}
}
#查看任务
GET _tasks/-F7dEMs-TReMFqqy81x_GQ:32154
#查看索引和大小
GET _cat/indices4.shell批量导入
调试不易，点赞再拿！！！
#脚本文件包括抓取需要迁移的index并通过for循环持续迁移到目标es集群。
cat reindex.sh
```
#!/bin/bash
#迁移es脚本 2024-4-11 lucas
set -x
#需要迁移的索引名
INDEX_NAME='istio-2023'
#筛选出具体索引名
OLD_INDEX=$(curl -k -u elastic:123123 -X GET "http://10.200.111.171:9200/_cat/indices" | grep ${INDEX_NAME} | awk '{print $3}')
#筛选出具体索引名 -ssl方式
#OLD_INDEX=$(curl -u username:password --cacert /path/to/old_ca.crt 'https://10.200.111.171:9200/_cat/indices' | grep ${INDEX_NAME} | awk '{print $3}')
for INDEX_NAME in ${OLD_INDEX}
do
curl -X POST "https://10.200.28.154:9200/_reindex" \
-H 'Content-Type: application/json' \
-u "elastic:123123" \
--cacert /path/to/new_ca.crt \
--insecure \
-d "
{
\"source\": {
\"remote\": {
\"host\": \"http://10.200.111.171:9200\",
\"username\": \"elastic\",
\"password\": \"123123\"
},
\"index\": \"${INDEX_NAME}\",
\"size\": 10000
},
\"dest\": {
\"index\": \"${INDEX_NAME}\"
}
}"
done
```
#如果原es集群是tls，则根据上面给的devtools给的照葫芦画瓢改就行了脚本亲测同名的索引会自动覆盖不会报错，这点要小心的。
执行完后查看索引条目对比下
版本不一样，索引内容不一样，大小会有出入，但是我只需要条目对就行了，因为我没有日志切割。
K8S/Kubernetes社区为您提供最前沿的新闻资讯和知识内容
更多推荐
2
0- 0
已为社区贡献4条内容
所有评论(0)