Kube-prometheus部署Prometheus

下载 kube-prometheus

# k8s 1.22 只能使用0.10和0.9
wget https://github.com/prometheus-operator/kube-prometheus/archive/refs/tags/v0.10.0.tar.gz

# mkdir -p service-monitor prometheus adapter node-exporter kube-state-metrics grafana alertmanager operator other blackbox-exporter

cd /data/bigdata/kube-prometheus-0.10.0/manifests

# 查看镜像
grep -rn 'image: '

# 镜像tag重置及推送镜像到harbor
docker tag quay.io/prometheus/alertmanager:v0.23.0 	 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/alertmanager:v0.23.0
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/alertmanager:v0.23.0

docker tag quay.io/prometheus/blackbox-exporter:v0.19.0 	 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/blackbox-exporter:v0.19.0
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/blackbox-exporter:v0.19.0

docker tag quay.io/brancz/kube-rbac-proxy:v0.11.0 	 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/kube-rbac-proxy:v0.11.0
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/kube-rbac-proxy:v0.11.0

docker tag quay.io/prometheus/node-exporter:v1.3.1 	 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/node-exporter:v1.3.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/node-exporter:v1.3.1

docker tag quay.io/prometheus/prometheus:v2.32.1 	 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus:v2.32.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus:v2.32.1

docker tag quay.io/prometheus-operator/prometheus-config-reloader:v0.53.1 	 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-config-reloader:v0.53.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-config-reloader:v0.53.1

docker tag quay.io/prometheus-operator/prometheus-operator:v0.53.1 	 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-operator:v0.53.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-operator:v0.53.1

docker tag bitnami/kube-state-metrics:2.3.0 	 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/kube-state-metrics:2.3.0
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/kube-state-metrics:2.3.0

docker tag willdockerhub/prometheus-adapter:v0.9.1 	 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-adapter:v0.9.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-adapter:v0.9.1

docker tag grafana/grafana:8.3.3 	 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/grafana:8.3.3
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/grafana:8.3.3

docker tag prom/pushgateway:v1.4.3	 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/pushgateway:v1.4.3
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/pushgateway:v1.4.3


# 替换为自建镜像仓库
sed -i 's/quay.io/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata/g' alertmanager-alertmanager.yaml
sed -i 's/quay.io/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata/g' blackboxExporter-deployment.yaml
sed -i 's/quay.io\/brancz/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata\/prometheus/g' kubeStateMetrics-deployment.yaml
sed -i 's/quay.io/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata/g' nodeExporter-daemonset.yaml
sed -i 's/quay.io/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata/g' prometheus-prometheus.yaml
sed -i 's/quay.io\/prometheus-operator/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata\/prometheus/g' prometheusOperator-deployment.yaml

# 还需要修改一下地方
# blackboxExporter-deployment.yaml:77:        image: bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/brancz/kube-rbac-proxy:v0.11.0

# kubeStateMetrics-deployment.yaml:34:        image: k8s.gcr.io/kube-state-metrics/kube-state-metrics:v2.3.0
# prometheusAdapter-deployment.yaml:39:        image: k8s.gcr.io/prometheus-adapter/prometheus-adapter:v0.9.1
# grafana-deployment.yaml:32:        image: grafana/grafana:8.3.3
# blackboxExporter-deployment.yaml:54:        image: jimmidyson/configmap-reload:v0.5.0

修改类型为 NodePort

cd /data/bigdata/kube-prometheus-0.10.0/manifests
vim prometheus-service.yaml

spec:
  type: NodePort # 新增
  ports:
  - name: web
    port: 9090
    targetPort: web
    nodePort: 30090 # 新增
  - name: reloader-web
    port: 8080
    targetPort: reloader-web

vim alertmanager-service.yaml

spec:
  type: NodePort # 新增
  ports:
  - name: web
    port: 9093
    targetPort: web
    nodePort: 30093 # 新增
  - name: reloader-web
    port: 8080
    targetPort: reloader-web

vim grafana-service.yaml

spec:
  type: NodePort # 新增
  ports:
  - name: http
    port: 3000
    targetPort: http
    nodePort: 30300 # 新增   30000-32767

kube-prometheus 集成 pushgateway

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app.kubernetes.io/name: pushgateway
  name: pushgateway
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: pushgateway
  template:
    metadata:
      labels:
        app.kubernetes.io/name: pushgateway
    spec:
      nodeSelector:
        kubernetes.io/os: linux
      containers:
      - image: bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/pushgateway:v1.4.3
        name: pushgateway
        ports:
        - containerPort: 9091
          name: http
        resources:
          limits:
            cpu: 200m
            memory: 512Mi
          requests:
            cpu: 50m
            memory: 128Mi
      restartPolicy: Always
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app.kubernetes.io/name: pushgateway
  name: pushgateway
  namespace: monitoring
spec:
  type: NodePort
  ports:
  - name: http
    port: 9091
    targetPort: http
    nodePort: 30391 # 新增
  selector:
    app.kubernetes.io/name: pushgateway
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  labels:
    app.kubernetes.io/name: pushgateway
  name: pushgateway
  namespace: monitoring
spec:
  endpoints:
  - interval: 30s
    path: /metrics
    port: http
    scheme: http
  selector:
    matchLabels:
      app.kubernetes.io/name: pushgateway

修改grafana配置

vim grafana-config.yaml
# 增加匿名访问，grafana数据存储到mysql
# 密码不需要base64编码

  grafana.ini: |
    [security]
    allow_embedding = true
    [date_formats]
    default_timezone = UTC
    [auth.anonymous]
    enabled = true
    [database]
    type = mysql
    host = mysql-master.mysql:3306
    name = grafana
    user = grafana
    password = YQZi5UUw825h
type: Opaque

Prometheus 持久化存储

vim sc.yaml

kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: prometheus-local-storage
provisioner: kubernetes.io/no-provisioner # 不自动创建PV
volumeBindingMode: WaitForFirstConsumer  # pod引用pvc后，pvc才进入绑定状态
reclaimPolicy: Retain   # pvc删除后数据不做清除

vim pv.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: prometheus-pv-1
spec:
  storageClassName: prometheus-local-storage
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  local:
    path: /data/zxl/prometheus1/  # 创建目录
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - szzb-bg-dev-etl-4  # 指定节点
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: prometheus-pv-2
spec:
  storageClassName: prometheus-local-storage
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  local:
    path: /data/zxl/prometheus2/  # 创建目录
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - szzb-bg-dev-etl-5  # 指定节点

# 末尾增加
vim prometheus-prometheus.yaml

  retention: 15d
  storage:
    volumeClaimTemplate:
      spec:
        storageClassName: prometheus-local-storage
        resources:
          requests:
            storage: 10Gi

配置Ingress

vim ingress.yaml

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: prometheus-ingress
  namespace: monitoring
  annotations:
    kubernetes.io/ingress.class: "nginx"
    prometheus.io/http_probe: "true"
spec:
  rules:
  - host: alert.k8s.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: alertmanager-main
            port:
              number: 9093
  - host: bigdata-uat-k8s-grafana.ky-tech.com.cn 
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: grafana
            port:
              number: 3000
  tls:
  - hosts:
    - bigdata-uat-k8s-grafana.ky-tech.com.cn  # https域名
    secretName: grafana-secret

  - host: bigdata-uat-k8s-prometheus.ky-tech.com.cn
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: prometheus-k8s
            port:
              number: 9090
  tls:
  - hosts:
    - bigdata-uat-k8s-prometheus.ky-tech.com.cn  # https域名
    secretName: prometheus-secret

kubectl create secret tls grafana-secret --key /data/k8s-install/ssl/ky-tech.com.cn_nginx/ky-tech.com.cn.key --cert /data/k8s-install/ssl/ky-tech.com.cn_nginx/ky-tech.com.cn_bundle.crt -n monitoring

kubectl create secret tls prometheus-secret --key /data/k8s-install/ssl/ky-tech.com.cn_nginx/ky-tech.com.cn.key --cert /data/k8s-install/ssl/ky-tech.com.cn_nginx/ky-tech.com.cn_bundle.crt -n monitoring

# 使修改生效 
kubectl apply -rf .

# delete prometheus-k8s-0  prometheus-k8s-1 pod

# kubectl get ing -n monitoring 查看ingress

部署

# 部署
kubectl apply --server-side -f manifests/setup
kubectl wait \
	--for condition=Established \
	--all CustomResourceDefinition \
	--namespace=monitoring
kubectl apply -f manifests/


# 卸载
kubectl delete --ignore-not-found=true -f manifests/ -f manifests/setup

监控k8s集群外节点

---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  labels:
    app.kubernetes.io/component: exporter
    app.kubernetes.io/name: nginx-exporter
    app.kubernetes.io/app: nginx
  name: nginx
  namespace: monitoring
spec:
  endpoints:
  - interval: 30s
    port: metrcis
  selector:
    matchLabels:
      app.kubernetes.io/component: exporter
      app.kubernetes.io/name: nginx-exporter
      app.kubernetes.io/app: nginx
  namespaceSelector:
      matchNames:
      - monitoring
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: nginx-exporter
    app.kubernetes.io/component: exporter
    app.kubernetes.io/name: nginx-exporter
    app.kubernetes.io/app: nginx
  name: nginx
  namespace: monitoring
spec:
  type: ClusterIP
  clusterIP: None
  ports:
  - name: metrcis
    port: 9113
    targetPort: 9113
    protocol: TCP
---
apiVersion: v1
kind: Endpoints
metadata:
  labels:
    app.kubernetes.io/component: exporter
    app.kubernetes.io/name: nginx-exporter
    app.kubernetes.io/app: nginx
  name: nginx
  namespace: monitoring
subsets:
  - addresses:
    - ip: 192.168.26.11
    - ip: 192.168.26.13
    ports:
    - name: metrcis
      port: 9113
      protocol: TCP

Kube-prometheus 监控 K8s 集群外服务的两种方式