采集思路：

通过组件watch k8s集群所有的事件信息，将信息保存到容器对应的目录，通过sidecar形式在事件监听组件上部署filebeat，配置采集目录下的事件文件，推送到kafka或者es等，持久化保存。

事件监听组件：

类似的很多，比如eventrouter，tke-persistent-event等，以下用eventrouter作为示例：

AtomGit | GitCode - 全球开发者的开源社区,开源代码托管平台

上述链接下载相应源码包

1、安装go环境

wget https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz

tar -C /usr/local -xzf go1.14.2.linux-amd64.tar.gz

echo 'export PATH=$PATH:/usr/local/go/bin' >> /etc/profile

echo 'export GOROOT=/usr/local/go' >> /etc/profile

source /etc/profile

go version

2、编译eventrouter

将文件上传至/gwmfc/app目录下，并解压缩  unzip eventrouter-master.zip，进入主目录

• 修改Makefile中相关参数，以适应中国网络

• • 修改REGISTRY变量为自己私服的地址 dockerdev.gwmfc.com/k8s
  • 修改BUILD_IMAGE为golang:1.14.2以支持通过env设置go-proxy
  • 修改编译步骤中$(DOCKER_BUILD)后边的参数为 "go env -w GO111MODULE=on && go env -w GOPROXY=https://goproxy.cn,direct "

• 修改Dockerfile

• • 修改from配置：FROM ****.com/alpine/alpine:3.9

执行命令make all，如无编译eventrouter文件，则手动编译：CGO_ENABLED=0 go build

3、将镜像上传至私服

docker push ****.com/alpine/alpine:3.9

4、部署配置eventrouter接收k8s事件并对接到elk（注意对应的sa权限）

apiVersion: v1
kind: ServiceAccount
metadata:
  name: eventrouter
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: eventrouter
rules:
- apiGroups: [""]
  resources: ["events"]
  verbs: ["get", "watch", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: eventrouter
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: eventrouter
subjects:
- kind: ServiceAccount
  name: eventrouter
  namespace: kube-system

---
apiVersion: apps/v1
kind: Deployment
metadata:
  annotations: {}
  labels:
    app: eventrouter
    k8s-app: eventrouter
    k8s.kuboard.cn/name: eventrouter
    ns: kube-system
    version: prod
  name: eventrouter
  namespace: kube-system
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: eventrouter
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: eventrouter
        k8s-app: eventrouter
        ns: kube-system
        tier: control-plane-addons
        version: prod
    spec:
      containers:
        - args:
            - '-c'
            - /app/eventrouter -v 3 -log_dir /data/log/eventrouter
          command:
            - /bin/sh
          image: '****.com/k8s/eventrouter:v0.3'
          imagePullPolicy: IfNotPresent
          name: kube-eventrouter
          resources:
            limits:
              cpu: 100m
              memory: 300Mi
            requests:
              cpu: 10m
              memory: 100Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /etc/eventrouter
              name: eventrouter-cm
            - mountPath: /data/log/eventrouter
              name: log-path
        - env:
            - name: k8s_pod_ip
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: status.podIP
            - name: k8s_node_ip
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: status.hostIP
          image: '****.com/elk/filebeat:7.16.3'
          imagePullPolicy: IfNotPresent
          name: filebeat
          resources:
            limits:
              cpu: 100m
              memory: 300Mi
            requests:
              cpu: 10m
              memory: 100Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /usr/share/filebeat/filebeat.yml
              name: eventroute-filebeat-config
              subPath: filebeat.yml
            - mountPath: /data/log/eventrouter
              name: log-path
      dnsPolicy: ClusterFirst
      imagePullSecrets:
        - name: dockerzl1
      initContainers:
        - args:
            - '-c'
            - chmod 777 -R /data/log/eventrouter
          command:
            - /bin/sh
          image: '****.com/baseimage/busybox:1.36'
          imagePullPolicy: IfNotPresent
          name: init-log-dir
          resources: {}
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /data/log/eventrouter
              name: log-path
      nodeSelector:
        type: basic
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      serviceAccount: eventrouter
      serviceAccountName: eventrouter
      terminationGracePeriodSeconds: 30
      volumes:
        - configMap:
            defaultMode: 420
            name: eventrouter-cm
          name: eventrouter-cm
        - configMap:
            defaultMode: 420
            items:
              - key: filebeat.yml
                path: filebeat.yml
            name: eventroute-filebeat-config
          name: eventroute-filebeat-config
        - emptyDir: {}
          name: log-path

---
apiVersion: v1
data:
  filebeat.yml: |-
    filebeat.inputs:
    - type: log
      enabled: true
      paths:
        - "/data/log/eventrouter/*"
      processors:
        - dissect:
            tokenizer: '%{event} %{name}       %{status} %{line} %{msg}'
            field: "message"
            target_prefix: ""
      encoding: utf-8
      #json.key_under_root: true
      #json.overwrite_keys: true
      #json.message_key: log
      fields_under_root: true
      fields:
        source: k8sevent-day
        app: elk
        eventType: trigger
        metric: ELK日志监控告警
        hostname: ${HOSTNAME}
        k8s_node_ip: '${k8s_node_ip}'
        k8s_pod_ip: '${k8s_pod_ip}'
        resourceIp: 127.0.0.1
        resourceName: k8sevent-day
    #  multiline.pattern: '^\{'
    #  multiline.negate: true
    #  multiline.match: after
      tail_files: false
      ignore_older: 24h
      clean_*: 48h
      close_*: 5m
      idle_timeout: 30s
      scan_frequency: 8s
    setup.template.settings:
      index.number_of_shards: 1
      index.number_of_replicas: 1
    logging.level: info
    logging.to_files: false
    setup.kibana:
      host: "http://***.***.***.***:5601"
    output.kafka:
      hosts: ["***.***.***.***:9092"]
      topic: "k8sevent-eventrouter"
      partition.hash:
        reachable_only: true
      required_acks: 1
      compression: gzip
      max_message_bytes: 1000000
    processors:
      - drop_fields:
          fields: ["message","line","status","name","event","host", "tags", "ecs", "log", "prospector", "agent", "input", "beat", "offset"]
          ignore_missing: true
      - rename:
          fields:
            - from: "msg"
              to: "message"
      - decode_json_fields:
          fields: ["message"]  # 指定要解析的字段
          process_array: false # 如果message内容是JSON数组，设置为true
          max_depth: 1         # JSON嵌套的最大深度
          target: "json"           # 解析后字段放置的目标位置，默认为空表示替换原字段
          overwrite_keys: true # 是否允许新解析出来的字段覆盖已存在的同名字段
      - drop_fields:
          fields: ["message","line","status","name","event"]
kind: ConfigMap
metadata:
  annotations: {}
  labels: {}
  name: eventroute-filebeat-config
  namespace: kube-system


---
kind: ConfigMap
apiVersion: v1
metadata:
  name: eventrouter-cm
  namespace: kube-system
data:
  config.json: |-
    {
      "sink": "glog",
      "s3SinkDiscardMessages": true,
      "s3SinkOutputFormat": "flatjson"
    }