centos7 源码部署elk7.4.2 + filebeat
filebeat安装# 下载filebeat包wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.4.2-linux-x86_64.tar.gz# 解压tar -xvf filebeat-7.4.2-linux-x86_64.tar.gz -C /datamv filebeat-7.4.2-linux-x86_
·
filebeat安装
# 下载filebeat包
wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.4.2-linux-x86_64.tar.gz
# 解压
tar -xvf filebeat-7.4.2-linux-x86_64.tar.gz -C /usr/local
cd /usr/local
mv filebeat-7.4.2-linux-x86_64 filebeat
cd filebeat
# 采集mysql慢日志及错误日志
vim filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /data/mysql-log/mysqld.log
tags: ["mysql_error"]
tail_files: true
fields:
# 根据不同环境配置自定义名
server: test-mysql_error
- type: log
enabled: true
paths:
- /data/mysql-log/mysql_slow.log
tags: ["mysql_slow"]
tail_files: true
fields:
server: test-mysql_slow
#----------------------------- Logstash output --------------------------------
output.logstash:
hosts: ["192.168.23.96:5044"]
# 采集nginx服务、spring cloud微服务
filebeat.inputs:
- type: log
enabled: true
paths:
- /data/app/nginx/logs/access.log
tags: ["nginx_access"]
tail_files: true
fields:
server: test-nginx_access
- type: log
enabled: true
paths:
- /data/app/nginx/logs/error.log
tags: ["nginx_error"]
tail_files: true
fields:
server: test-nginx_error
- type: log
enabled: true
paths:
- /data/app/logs/*.log
tailf_files: true
fields:
server: app1
# 合并jvm堆栈报错,多行合并一行
multiline:
pattern: '^\s*("{)'
negate: true
match: after
max_lines: 100
#----------------------------- Logstash output --------------------------------
output.logstash:
hosts: ["192.168.23.96:5044"]
安装supervisor并配置监控filebeat
# easy_install的方式
yum install -y python-setuptools
easy_install supervisor
echo_supervisord_conf >/etc/supervisord.conf
# 配置supervisord.conf,修改supervisord.conf行尾
vim /etc/supervisord.conf
[include]
files = /etc/supervisor/*.conf
# web界面显示(可选,如果想通过浏览器管理)
[inet_http_server]
port=IP:9001
# 启动
supervisord -c supervisord.conf
# 创建目录
mkdir -p /etc/supervisor
# 创建job服务
vim /etc/supervisor/filebeat.conf
[program:filebeat]
directory=/data/filebeat
command=/data/filebeat/filebeat -e -c filebeat.yml
stdout_logfile = /data/log/supervisor/filebets_out.log
user=root
autostart=true
autorestart=true
startsecs=10
#启动前需先创建log目录,不然会报错(****)
mkdir /data/log/supervisor
# 通过supervisorctl管理启动服务
# 重新加载
supervisorctl -c supervisord.conf reload
# 使用supervisorctl启动filebeat服务
supervisorctl start filebeat
# 查看状态
supervisorctl status
# 自行验证,kill掉服务filebeat看看服务是否会自动重启
ps -ef | grep filebeat
部署ElaticSearch
# 系统配置
swapoff -a
cat >> /etc/sysctl.conf <<EOF
fs.file-max=655360
vm.max_map_count = 262144
EOF
vim /etc/security/limits.conf
* soft nproc 20480
* hard nproc 20480
* soft nofile 65536
* hard nofile 65536
* soft memlock unlimited
* hard memlock unlimited
vim /etc/sysctl.conf
新增vm.max_map_count = 655360
执行sysctl –p
vim /etc/security/limits.d/20-nproc.conf
* soft nproc 20480
# ELK无法用root用户启动,创建新用户并授权
groupadd elk
useradd -g elk elk
passwd elk # 这里密码: 12345678
# 创建运行目录
mkdir /data/elk
chown -R elk:elk /data/elk
# 下载安装包
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.4.2-linux-x86_64.tar.gz
# 解压
tar -xvf elasticsearch-7.4.2-linux-x86_64.tar.gz -C /data/elk
# 切换用户
su elk
cd /data/elk
# 重命名安装目录
mv elasticsearch-7.4.2-linux-x86_64.tar.gz elasticsearch
# 进入目录
cd elasticsearch
# 修改jvm (配置8g)
vim config/jvm.options
-Xms8g
-Xmx8g
vim config/elasticsearch.yml
node.name: node-1
# ----------------------------------- Paths ------------------------------------
path.data: /data/elk/data
#
path.logs: /data/elk/logs
# ----------------------------------- Memory -----------------------------------
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
# ---------------------------------- Network -----------------------------------
network.host: 0.0.0.0
http.cors.enabled: true
http.cors.allow-origin: "*"
http.port: 9200
cluster.initial_master_nodes: ["node-1"]
# 参数优化
discovery.zen.fd.ping_interval: 120s
indices.requests.cache.size: 2%
# refresh_interval不能在配置文件需在命令行执行
curl -H "Content-Type: application/json" -XPUT 'http://localhost:9200/_all/_settings?preserve_existing=true' -d '{
"index.merge.scheduler.max_thread_count" : "1",
"index.refresh_interval" : "300s",
"index.translog.durability" : "async"
}'
#启动
./bin/elasticsearch -d
# 测试是否启动
浏览器访问192.168.23.96:9200
这里注意一个问题,我这里logstash由于需要安装jdk8,而elasticsearch7以后自带jdk不用额外安装所以启动顺序不要变,elastic -> logstash
logstash的安装
wget https://artifacts.elastic.co/downloads/logstash/logstash-7.4.2.tar.gz
su elk
cd /data/elk/logstash
tar -xvf logstash-7.4.2.tar.gz
mv logstash-7.4.2 logstash
# 进入到配置文件目录
cd ./config
# 修改jvm (配置8g)
vim config/jvm.options
-Xms8g
-Xmx8g
# 修改配置文件logstash.yml
input {
beats {
port => 5044
}
}
output {
elasticsearch {
hosts => ["localhost:9200"]
index => ["%{[fields][server]}-%{+YYYY-MM-dd}"]
manage_template => false
}
}
# 执行启动命令
./bin/logstash -f ./config/logstash.conf
#启动成功后可以看到日志
Successfully started Logstash API endpoint {port=>9600}
kibana安装
# 下载
wget https://artifacts.elastic.co/downloads/kibana/kibana-7.4.2-linux-x86_64.tar.gz
su elk
tar -xvf kibana-7.4.2-linux-x86_64.tar.gz -C /data/elk
mv kibana-7.4.2-linux-x86_64 kibana
cd /data/elk/kibana
# 配置信息
server.host: "0.0.0.0"
server.port: 5601
elasticsearch.hosts: ["http://192.168.23.96:9200"]
# 启动
./bin/kibana
# 访问验证,浏览器访问
http://192.168.23.96:5601
这里注意一个问题,我这里logstash由于以来jdk8,而elasticsearch自带jdk不用额外安装所以启动顺序最好不变,elastic -> logstash -> filebeat -> kibana
权限控制x-pack
#启用elasticsearch xpack安全验证
vim elasticsearch.yml
xpack.security.enabled: true
# 单个节点
discovery.type: single-node
# 重新启动
./bin/elasticsearch -d
#设置密码,运行elasticsearch-setup-passwords设置密码(账号默认为elastic):
./elasticsearch-setup-passwords interactive
# 密码要记住,后面有需要
Changed password for user [apm_system] : apm2020
Changed password for user [kibana]: kibana2020
Changed password for user [logstash_system]: logstash2020
Changed password for user [beats_system]: beats2020
Changed password for user [remote_monitoring_user]: remote2002
Changed password for user [elastic]: elastic2020
# logstash配置x-pack:
vim logstash.conf
input {
beats {
port => 5044
}
}
output {
elasticsearch {
hosts => ["localhost:9200"]
user => "elastic"
password => "elastic2020"
index => ["%{[fields][server]}-%{+YYYY-MM-dd}"]
manage_template => false
}
}
# Kibana的配置x-pack:
vim kibana.yml
elasticsearch.username: "kibana"
elasticsearch.password: "kibana2020"
# 随便输入不少于32位字符
xpack.security.encryptionKey: "rcrafrgraffbdsacdefghigklmnopqvvrsvrsrtfdfavfjkadfakfacjdiaofoidaui3cjda"
xpack.security.sessionTimeout: 600000
创建只读账户
x-pack配置成功后就可以看到security,创建一个应用系统角色,选择对应的索引文件,分配对应的权限read
在Management下面的Kibana有一个Security,有User和Role点击Role
更多推荐
已为社区贡献2条内容
所有评论(0)