k8s部署loki-distributed

loki-distributed

段帅星

558人浏览 · 2024-03-30 08:34:38

段帅星 · 2024-03-30 08:34:38 发布

简介：

微服务部署模式将 Loki 的组件实例化为不同的进程，每个进程都被调用并指定其目标，每个组件都会产生一个用于内部请求的 gRPC 服务器和一个用于外部 API 请求的 HTTP 服务。

将组件作为单独的微服务运行允许通过增加微服务的数量来进行扩展，定制的集群对各个组件具有更好的可观察性。微服务模式部署是最高效的 Loki 安装，但是，它们的设置和维护也是最复杂的

ingester
distributor
query-frontend
query-scheduler
querier
index-gateway
ruler
compactor
-

一、安装

1、获取loki-distributed的安装包

helm repo add grafana https://grafana.github.io/helm-charts
helm repo update
helm pull grafana/loki-distributed --untar --version 0.78.4

2、部署minio，并创建bucket

3、修改loki-distributed 部署文件

helm --create-namespace  loki -n loki  install ./loki-distributed/ -f ./minio-values.yaml

loki:
  structuredConfig:
    ingester:
      max_transfer_retries: 0
      chunk_idle_period: 1h
      chunk_target_size: 1536000
      max_chunk_age: 1h
    storage_config:
      aws:
        s3: http://QeytJ4j0u6Kodh4ki6Ra:ctrCLm0XRWmhZuv2flt5Wgf9QaGpNGFOXoMBCFxT@192.168.86.21:31388
        bucketnames: loki-data
        s3forcepathstyle: true
      boltdb_shipper:
        shared_store: s3
    schema_config:
      configs:
        - from: 2024-03-30
          store: boltdb-shipper
          object_store: aws
          schema: v12
          index:
            prefix: loki_index_
            period: 24h

distributor:
  replicas: 2
  maxUnavailable: 1

ingester:
  replicas: 2
  maxUnavailable: 1
  persistence:
    enabled: true
    size: 1Gi
    storageClass: local-path

querier:
  replicas: 2
  maxUnavailable: 1
  persistence:
    enabled: true
    size: 1Gi
    storageClass: local-path

queryFrontend:
  replicas: 2
  maxUnavailable: 1

gateway:
  nginxConfig:
    httpSnippet: |-
      client_max_body_size 100M;
    serverSnippet: |-
      client_max_body_size 100M;

[root@common01 ~]# kubectl  -n loki get pod,svc
NAME                                                        READY   STATUS    RESTARTS   AGE
pod/loki-loki-distributed-distributor-6546d4844b-fxwdv      1/1     Running   0          100m
pod/loki-loki-distributed-distributor-6546d4844b-pst4t      1/1     Running   0          100m
pod/loki-loki-distributed-gateway-57c9d56bc7-kbbrg          1/1     Running   0          100m
pod/loki-loki-distributed-ingester-0                        1/1     Running   0          100m
pod/loki-loki-distributed-ingester-1                        1/1     Running   0          100m
pod/loki-loki-distributed-querier-0                         1/1     Running   0          100m
pod/loki-loki-distributed-querier-1                         1/1     Running   0          100m
pod/loki-loki-distributed-query-frontend-7ccb5f798b-5z7qr   1/1     Running   0          100m
pod/loki-loki-distributed-query-frontend-7ccb5f798b-tvhxg   1/1     Running   0          100m

NAME                                                    TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)                      AGE
service/loki-loki-distributed-distributor               ClusterIP   10.96.2.143   <none>        3100/TCP,9095/TCP            100m
service/loki-loki-distributed-gateway                   ClusterIP   10.96.0.251   <none>        80/TCP                       100m
service/loki-loki-distributed-ingester                  ClusterIP   10.96.0.175   <none>        3100/TCP,9095/TCP            100m
service/loki-loki-distributed-ingester-headless         ClusterIP   None          <none>        3100/TCP,9095/TCP            100m
service/loki-loki-distributed-memberlist                ClusterIP   None          <none>        7946/TCP                     100m
service/loki-loki-distributed-querier                   ClusterIP   10.96.2.80    <none>        3100/TCP,9095/TCP            100m
service/loki-loki-distributed-querier-headless          ClusterIP   None          <none>        3100/TCP,9095/TCP            100m
service/loki-loki-distributed-query-frontend            ClusterIP   10.96.3.61    <none>        3100/TCP,9095/TCP,9096/TCP   100m
service/loki-loki-distributed-query-frontend-headless   ClusterIP   None          <none>        3100/TCP,9095/TCP,9096/TCP   100m

loki的配置文件

kubectl get cm -n loki loki-loki-distributed -o yaml
apiVersion: v1
data:
  config.yaml: |
    auth_enabled: false
    chunk_store_config:
      max_look_back_period: 0s
    common:
      compactor_address: http://loki-loki-distributed-compactor:3100
    compactor:
      shared_store: filesystem
    distributor:
      ring:
        kvstore:
          store: memberlist
    frontend:
      compress_responses: true
      log_queries_longer_than: 5s
      tail_proxy_url: http://loki-loki-distributed-querier:3100
    frontend_worker:
      frontend_address: loki-loki-distributed-query-frontend-headless:9095
    ingester:
      chunk_block_size: 262144
      chunk_encoding: snappy
      chunk_idle_period: 1h
      chunk_retain_period: 1m
      chunk_target_size: 1536000
      lifecycler:
        ring:
          kvstore:
            store: memberlist
          replication_factor: 1
      max_chunk_age: 1h
      max_transfer_retries: 0
      wal:
        dir: /var/loki/wal
    ingester_client:
      grpc_client_config:
        grpc_compression: gzip
    limits_config:
      enforce_metric_name: false
      max_cache_freshness_per_query: 10m
      reject_old_samples: true
      reject_old_samples_max_age: 168h
      split_queries_by_interval: 15m
    memberlist:
      join_members:
      - loki-loki-distributed-memberlist
    query_range:
      align_queries_with_step: true
      cache_results: true
      max_retries: 5
      results_cache:
        cache:
          embedded_cache:
            enabled: true
            ttl: 24h
    ruler:
      alertmanager_url: https://alertmanager.xx
      external_url: https://alertmanager.xx
      ring:
        kvstore:
          store: memberlist
      rule_path: /tmp/loki/scratch
      storage:
        local:
          directory: /etc/loki/rules
        type: local
    runtime_config:
      file: /var/loki-distributed-runtime/runtime.yaml
    schema_config:
      configs:
      - from: "2022-06-21"
        index:
          period: 24h
          prefix: loki_index_
        object_store: s3
        schema: v12
        store: boltdb-shipper
    server:
      http_listen_port: 3100
    storage_config:
      aws:
        access_key_id: QeytJ4j0u6Kodh4ki6Ra
        bucketnames: loki-bucket
        endpoint: 10.96.0.98:9000
        insecure: true
        s3forcepathstyle: true
        secret_access_key: ctrCLm0XRWmhZuv2flt5Wgf9QaGpNGFOXoMBCFxT
      boltdb_shipper:
        active_index_directory: /var/loki/index
        cache_location: /var/loki/cache
        cache_ttl: 168h
        shared_store: s3
      filesystem:
        directory: /var/loki/chunks
    table_manager:
      retention_deletes_enabled: false
      retention_period: 0s
kind: ConfigMap
metadata:
  annotations:
    meta.helm.sh/release-name: loki
    meta.helm.sh/release-namespace: loki
  creationTimestamp: "2024-03-29T22:00:53Z"
  labels:
    app.kubernetes.io/instance: loki
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: loki-distributed
    app.kubernetes.io/version: 2.9.5
    helm.sh/chart: loki-distributed-0.78.4
  name: loki-loki-distributed
  namespace: loki
  resourceVersion: "347884"
  uid: 600afb4b-dcce-4884-8b50-33da61023f28

gateway配置文件

gateway 组件会来帮助我们将请求路由到正确的组件中去，该组件同样就是一个 nginx 服务，对应的配置如下所示：

kubectl  -n loki exec -it loki-loki-distributed-gateway-57c9d56bc7-kbbrg -- cat /etc/nginx/nginx.conf
worker_processes  5;  ## Default: 1
error_log  /dev/stderr;
pid        /tmp/nginx.pid;
worker_rlimit_nofile 8192;

events {
  worker_connections  4096;  ## Default: 1024
}

http {
  client_body_temp_path /tmp/client_temp;
  proxy_temp_path       /tmp/proxy_temp_path;
  fastcgi_temp_path     /tmp/fastcgi_temp;
  uwsgi_temp_path       /tmp/uwsgi_temp;
  scgi_temp_path        /tmp/scgi_temp;

  proxy_http_version    1.1;

  default_type application/octet-stream;
  log_format   main '$remote_addr - $remote_user [$time_local]  $status '
        '"$request" $body_bytes_sent "$http_referer" '
        '"$http_user_agent" "$http_x_forwarded_for"';
  access_log   /dev/stderr  main;

  sendfile     on;
  tcp_nopush   on;
  resolver kube-dns.kube-system.svc.cluster.local;
  
  client_max_body_size 100M;

  server {
    listen             8080;

    location = / {
      return 200 'OK';
      auth_basic off;
      access_log off;
    }

    location = /api/prom/push {
      set $api_prom_push_backend http://loki-loki-distributed-distributor.loki.svc.cluster.local;
      proxy_pass       $api_prom_push_backend:3100$request_uri;
      proxy_http_version 1.1;
    }

    location = /api/prom/tail {
      set $api_prom_tail_backend http://loki-loki-distributed-querier.loki.svc.cluster.local;
      proxy_pass       $api_prom_tail_backend:3100$request_uri;
      proxy_set_header Upgrade $http_upgrade;
      proxy_set_header Connection "upgrade";
      proxy_http_version 1.1;
    }

    # Ruler
    location ~ /prometheus/api/v1/alerts.* {
      proxy_pass       http://loki-loki-distributed-ruler.loki.svc.cluster.local:3100$request_uri;
    }
    location ~ /prometheus/api/v1/rules.* {
      proxy_pass       http://loki-loki-distributed-ruler.loki.svc.cluster.local:3100$request_uri;
    }
    location ~ /api/prom/rules.* {
      proxy_pass       http://loki-loki-distributed-ruler.loki.svc.cluster.local:3100$request_uri;
    }
    location ~ /api/prom/alerts.* {
      proxy_pass       http://loki-loki-distributed-ruler.loki.svc.cluster.local:3100$request_uri;
    }

    location ~ /api/prom/.* {
      set $api_prom_backend http://loki-loki-distributed-query-frontend-headless.loki.svc.cluster.local;
      proxy_pass       $api_prom_backend:3100$request_uri;
      proxy_http_version 1.1;
    }

    location = /loki/api/v1/push {
      set $loki_api_v1_push_backend http://loki-loki-distributed-distributor.loki.svc.cluster.local;
      proxy_pass       $loki_api_v1_push_backend:3100$request_uri;
      proxy_http_version 1.1;
    }

    location = /loki/api/v1/tail {
      set $loki_api_v1_tail_backend http://loki-loki-distributed-querier.loki.svc.cluster.local;
      proxy_pass       $loki_api_v1_tail_backend:3100$request_uri;
      proxy_set_header Upgrade $http_upgrade;
      proxy_set_header Connection "upgrade";
      proxy_http_version 1.1;
    }

    location ~ /loki/api/.* {
      set $loki_api_backend http://loki-loki-distributed-query-frontend-headless.loki.svc.cluster.local;
      proxy_pass       $loki_api_backend:3100$request_uri;
      proxy_http_version 1.1;
    }
    
    client_max_body_size 100M;
  }
}

4、我们要写入日志数据，现在是写入到 gateway 的 Push 端点上去。为了验证应用是否正常，接下来我们再安装 Promtail 和 Grafana 来进行数据的读写。

4.1 安装 Promtail

helm pull grafana/promtail --untar

cat promtail-minio-values.yaml 
rbac:
  pspEnabled: false
config:
  clients:
    - url: http://loki-loki-distributed-gateway/loki/api/v1/push

helm --create-namespace promtail -n loki  install ./promtail -f ./promtail-minio-values.yaml

4.2 安装grafana

 helm pull grafana/grafana --untar
cat  grafana-minio-values.yaml
service:
  type: NodePort
  nodePort: 32001
rbac:
  pspEnabled: false
persistence:
  enabled: true
  storageClassName: local-path
  accessModes:
    - ReadWriteOnce
  size: 1Gi

helm --create-namespace grafana -n loki  install ./grafana -f ./grafana-minio-values.yaml

获取密码：
kubectl get secret --namespace loki grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo

登录grafana配置数据源：http://loki-loki-distributed-gateway

在这里插入图片描述

5、缓存：

参考：https://docs.youdianzhishi.com/k8s/logging/loki/distributed/#%E7%BC%93%E5%AD%98

K8S/Kubernetes

K8S/Kubernetes社区为您提供最前沿的新闻资讯和知识内容

更多推荐

【深度】阿里巴巴万级规模 K8s 集群全局高可用体系之美

作者 | 韩堂、柘远、沉醉来源 | 阿里巴巴云原生公众号前言台湾作家林清玄在接受记者采访的时候，如此评价自己 30 多年写作生涯：“第一个十年我才华横溢，‘贼光闪现’，令周边黯然失色；第二个十年，我终于‘宝光现形’，不再去抢风头，反而与身边的美丽相得益彰；进入第三个十年，繁华落尽见真醇，我进入了‘醇光初现’的阶段，真正体味到了境界之美”。长夜有穷，真水无香。领略过了 K8s“身在江

K8S/Kubernetes

如何基于 K8s 构建下一代 DevOps 平台？

作者 | 孙健波（天元）导读：当前云原生 DevOps 体系现状如何？面临哪些挑战？如何通过 OAM 解决云原生 DevOps 场景下的诸多问题？云原生开发应用模型 OAM(Open Application Model) 社区核心成员孙健波将为大家一一解答，并分享如何基于 OAM 和 Kubernetes 打造无限能力的下一代 DevOps 平台。什么是 DevOps？为什么基于 Kub