使用Kube-prometheus部署Prometheus (K8S)
使用Kube-prometheus部署Prometheus (K8S)
·
文章目录
Kube-prometheus部署Prometheus
下载 kube-prometheus
# k8s 1.22 只能使用0.10和0.9
wget https://github.com/prometheus-operator/kube-prometheus/archive/refs/tags/v0.10.0.tar.gz
# mkdir -p service-monitor prometheus adapter node-exporter kube-state-metrics grafana alertmanager operator other blackbox-exporter
cd /data/bigdata/kube-prometheus-0.10.0/manifests
# 查看镜像
grep -rn 'image: '
# 镜像tag重置及推送镜像到harbor
docker tag quay.io/prometheus/alertmanager:v0.23.0 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/alertmanager:v0.23.0
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/alertmanager:v0.23.0
docker tag quay.io/prometheus/blackbox-exporter:v0.19.0 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/blackbox-exporter:v0.19.0
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/blackbox-exporter:v0.19.0
docker tag quay.io/brancz/kube-rbac-proxy:v0.11.0 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/kube-rbac-proxy:v0.11.0
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/kube-rbac-proxy:v0.11.0
docker tag quay.io/prometheus/node-exporter:v1.3.1 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/node-exporter:v1.3.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/node-exporter:v1.3.1
docker tag quay.io/prometheus/prometheus:v2.32.1 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus:v2.32.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus:v2.32.1
docker tag quay.io/prometheus-operator/prometheus-config-reloader:v0.53.1 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-config-reloader:v0.53.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-config-reloader:v0.53.1
docker tag quay.io/prometheus-operator/prometheus-operator:v0.53.1 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-operator:v0.53.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-operator:v0.53.1
docker tag bitnami/kube-state-metrics:2.3.0 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/kube-state-metrics:2.3.0
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/kube-state-metrics:2.3.0
docker tag willdockerhub/prometheus-adapter:v0.9.1 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-adapter:v0.9.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-adapter:v0.9.1
docker tag grafana/grafana:8.3.3 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/grafana:8.3.3
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/grafana:8.3.3
docker tag prom/pushgateway:v1.4.3 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/pushgateway:v1.4.3
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/pushgateway:v1.4.3
# 替换为自建镜像仓库
sed -i 's/quay.io/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata/g' alertmanager-alertmanager.yaml
sed -i 's/quay.io/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata/g' blackboxExporter-deployment.yaml
sed -i 's/quay.io\/brancz/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata\/prometheus/g' kubeStateMetrics-deployment.yaml
sed -i 's/quay.io/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata/g' nodeExporter-daemonset.yaml
sed -i 's/quay.io/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata/g' prometheus-prometheus.yaml
sed -i 's/quay.io\/prometheus-operator/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata\/prometheus/g' prometheusOperator-deployment.yaml
# 还需要修改一下地方
# blackboxExporter-deployment.yaml:77: image: bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/brancz/kube-rbac-proxy:v0.11.0
# kubeStateMetrics-deployment.yaml:34: image: k8s.gcr.io/kube-state-metrics/kube-state-metrics:v2.3.0
# prometheusAdapter-deployment.yaml:39: image: k8s.gcr.io/prometheus-adapter/prometheus-adapter:v0.9.1
# grafana-deployment.yaml:32: image: grafana/grafana:8.3.3
# blackboxExporter-deployment.yaml:54: image: jimmidyson/configmap-reload:v0.5.0
修改类型为 NodePort
cd /data/bigdata/kube-prometheus-0.10.0/manifests
vim prometheus-service.yaml
spec:
type: NodePort # 新增
ports:
- name: web
port: 9090
targetPort: web
nodePort: 30090 # 新增
- name: reloader-web
port: 8080
targetPort: reloader-web
vim alertmanager-service.yaml
spec:
type: NodePort # 新增
ports:
- name: web
port: 9093
targetPort: web
nodePort: 30093 # 新增
- name: reloader-web
port: 8080
targetPort: reloader-web
vim grafana-service.yaml
spec:
type: NodePort # 新增
ports:
- name: http
port: 3000
targetPort: http
nodePort: 30300 # 新增 30000-32767
kube-prometheus 集成 pushgateway
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app.kubernetes.io/name: pushgateway
name: pushgateway
namespace: monitoring
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: pushgateway
template:
metadata:
labels:
app.kubernetes.io/name: pushgateway
spec:
nodeSelector:
kubernetes.io/os: linux
containers:
- image: bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/pushgateway:v1.4.3
name: pushgateway
ports:
- containerPort: 9091
name: http
resources:
limits:
cpu: 200m
memory: 512Mi
requests:
cpu: 50m
memory: 128Mi
restartPolicy: Always
---
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/name: pushgateway
name: pushgateway
namespace: monitoring
spec:
type: NodePort
ports:
- name: http
port: 9091
targetPort: http
nodePort: 30391 # 新增
selector:
app.kubernetes.io/name: pushgateway
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
labels:
app.kubernetes.io/name: pushgateway
name: pushgateway
namespace: monitoring
spec:
endpoints:
- interval: 30s
path: /metrics
port: http
scheme: http
selector:
matchLabels:
app.kubernetes.io/name: pushgateway
修改grafana配置
vim grafana-config.yaml
# 增加匿名访问,grafana数据存储到mysql
# 密码不需要base64编码
grafana.ini: |
[security]
allow_embedding = true
[date_formats]
default_timezone = UTC
[auth.anonymous]
enabled = true
[database]
type = mysql
host = mysql-master.mysql:3306
name = grafana
user = grafana
password = YQZi5UUw825h
type: Opaque
Prometheus 持久化存储
vim sc.yaml
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
name: prometheus-local-storage
provisioner: kubernetes.io/no-provisioner # 不自动创建PV
volumeBindingMode: WaitForFirstConsumer # pod引用pvc后,pvc才进入绑定状态
reclaimPolicy: Retain # pvc删除后数据不做清除
vim pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: prometheus-pv-1
spec:
storageClassName: prometheus-local-storage
capacity:
storage: 10Gi
accessModes:
- ReadWriteOnce
local:
path: /data/zxl/prometheus1/ # 创建目录
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- szzb-bg-dev-etl-4 # 指定节点
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: prometheus-pv-2
spec:
storageClassName: prometheus-local-storage
capacity:
storage: 10Gi
accessModes:
- ReadWriteOnce
local:
path: /data/zxl/prometheus2/ # 创建目录
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- szzb-bg-dev-etl-5 # 指定节点
# 末尾增加
vim prometheus-prometheus.yaml
retention: 15d
storage:
volumeClaimTemplate:
spec:
storageClassName: prometheus-local-storage
resources:
requests:
storage: 10Gi
配置Ingress
vim ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: prometheus-ingress
namespace: monitoring
annotations:
kubernetes.io/ingress.class: "nginx"
prometheus.io/http_probe: "true"
spec:
rules:
- host: alert.k8s.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: alertmanager-main
port:
number: 9093
- host: bigdata-uat-k8s-grafana.ky-tech.com.cn
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: grafana
port:
number: 3000
tls:
- hosts:
- bigdata-uat-k8s-grafana.ky-tech.com.cn # https域名
secretName: grafana-secret
- host: bigdata-uat-k8s-prometheus.ky-tech.com.cn
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: prometheus-k8s
port:
number: 9090
tls:
- hosts:
- bigdata-uat-k8s-prometheus.ky-tech.com.cn # https域名
secretName: prometheus-secret
kubectl create secret tls grafana-secret --key /data/k8s-install/ssl/ky-tech.com.cn_nginx/ky-tech.com.cn.key --cert /data/k8s-install/ssl/ky-tech.com.cn_nginx/ky-tech.com.cn_bundle.crt -n monitoring
kubectl create secret tls prometheus-secret --key /data/k8s-install/ssl/ky-tech.com.cn_nginx/ky-tech.com.cn.key --cert /data/k8s-install/ssl/ky-tech.com.cn_nginx/ky-tech.com.cn_bundle.crt -n monitoring
# 使修改生效
kubectl apply -rf .
# delete prometheus-k8s-0 prometheus-k8s-1 pod
# kubectl get ing -n monitoring 查看ingress
部署
# 部署
kubectl apply --server-side -f manifests/setup
kubectl wait \
--for condition=Established \
--all CustomResourceDefinition \
--namespace=monitoring
kubectl apply -f manifests/
# 卸载
kubectl delete --ignore-not-found=true -f manifests/ -f manifests/setup
监控k8s集群外节点
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
labels:
app.kubernetes.io/component: exporter
app.kubernetes.io/name: nginx-exporter
app.kubernetes.io/app: nginx
name: nginx
namespace: monitoring
spec:
endpoints:
- interval: 30s
port: metrcis
selector:
matchLabels:
app.kubernetes.io/component: exporter
app.kubernetes.io/name: nginx-exporter
app.kubernetes.io/app: nginx
namespaceSelector:
matchNames:
- monitoring
---
apiVersion: v1
kind: Service
metadata:
labels:
app: nginx-exporter
app.kubernetes.io/component: exporter
app.kubernetes.io/name: nginx-exporter
app.kubernetes.io/app: nginx
name: nginx
namespace: monitoring
spec:
type: ClusterIP
clusterIP: None
ports:
- name: metrcis
port: 9113
targetPort: 9113
protocol: TCP
---
apiVersion: v1
kind: Endpoints
metadata:
labels:
app.kubernetes.io/component: exporter
app.kubernetes.io/name: nginx-exporter
app.kubernetes.io/app: nginx
name: nginx
namespace: monitoring
subsets:
- addresses:
- ip: 192.168.26.11
- ip: 192.168.26.13
ports:
- name: metrcis
port: 9113
protocol: TCP
更多推荐
已为社区贡献4条内容
所有评论(0)