ELK日誌系統介紹 開源實時日誌分析ELK平臺能夠完美的解決我們上述的問題，ELK由ElasticSearch、Logstash和Kiabana三個開源工具組成。官方網站：https://www.elastic.co/products
1、Elasticsearch是個開源分散式搜尋引擎，它的特點有：分散式，零配置，自動發現，索引自動分片，索引副本機制，restful風格介面，多資料來源，自動搜尋負載等
2、Logstash是一個完全開源的工具，他可以對你的日誌進行收集、過濾，並將其儲存供以後使用（如，搜尋）
3、Kibana 也是一個開源和免費的工具，它Kibana可以為 Logstash 和 ElasticSearch 提供的日誌分析友好的 Web 介面，可以幫助您彙總、分析和搜尋重要資料日誌
安裝環境準備 首先需要下載好相關的軟體安裝包 官方網站：https://www.elastic.co https://artifacts.elastic.co/downloads/logstash/logstash-5.3.1.tar.gz https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.3.1.tar.gz https://artifacts.elastic.co/downloads/kibana/kibana-5.3.1-linux-x86_64.tar.gz 安裝配置JAVA環境
JDK版本：jdk-8u144-linux-x64.tar.gz
[root@centos7-1 ~]# cat /etc/redhat-release
CentOS Linux release 7.4.1708 (Core)
[root@centos7-1 ~]# uname -r
3.10.0-693.el7.x86_64
[root@centos7-1 ~]# tar zxf jdk-8u144-linux-x64.tar.gz -C /usr/local/
[root@centos7-1 ~]# ln -s /usr/local/jdk1.8.0_144 /usr/local/jdk
[root@centos7-1 ~]# cat >>/etc/profile <<EOF
export JAVA_HOME=/usr/local/jdk
export PATH=$PATH: $JAVA_HOME/bin
export CLASSPATH=.CLASSPATH:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
EOF
[root@centos7-1 ~]# source /etc/profile
[root@centos7-1 ~]# java -version
java version "1.8.0_144"
Java(TM) SE Runtime Environment (build 1.8.0_144-b01)
Java HotSpot(TM) 64-Bit Server VM (build 25.144-b01, mixed mode)
複製程式碼安裝Elasticserach 修改系統引數
[root@centos7-1 config]# vim /etc/sysctl.conf
#增加下面的配置
vm.max_map_count=655360
[root@centos7-1 config]# sysctl -p
vm.max_map_count = 655360
[root@centos7-1 config]# tail -5 /etc/security/limits.conf
* soft nofile 65536
* hard nofile 131072
* soft nproc 65536
* hard nproc 131072
# End of file
複製程式碼[root@centos7-1 config]# vim /etc/security/limits.d/20-nproc.conf
#增加下面的配置
elk soft nproc 65536
複製程式碼[root@centos7-1 config]# useradd elk
[root@centos7-1 config]# mkdir /elk/data /elk/logs -p
[root@centos7-1 config]# chown -R elk.elk /elk/
[root@centos7-1 config]# chown -R elk.elk /usr/local/elasticsearch/
複製程式碼安裝與配置
[root@centos7-1 ~]# tar zxf elasticsearch-5.3.1.tar.gz -C /usr/local/
[root@centos7-1 ~]# ln -s /usr/local/elasticsearch-5.3.1 /usr/local/elasticsearch
[root@centos7-1 ~]# cd /usr/local/elasticsearch/config/
複製程式碼修改配置檔案
[root@centos7-1 config]# egrep -v "^#|^$" elasticsearch.yml
cluster.name: myelk #叢集名
node.name: centos7-1
path.data: /elk/data
path.logs: /elk/logs
network.host: 0.0.0.0
http.port: 9200
discovery.zen.ping.unicast.hosts: ["host1", "host2"]
複製程式碼啟動服務
[root@centos7-1 config]# su - elk
[elk@centos7-1 ~]$ who
root pts/0 2017-10-13 22:22 (10.0.0.1)
[elk@centos7-1 ~]$ cd /usr/local/elasticsearch/bin/
[elk@centos7-1 bin]$ ./elasticsearch&
複製程式碼安裝Logstash
[root@centos7-1 ~]# tar zxf logstash-5.3.1.tar.gz -C /usr/local/
[root@centos7-1 ~]# /usr/local/logstash-5.3.1/bin/logstash -e 'input { stdin { } } output { stdout {} }'
Sending Logstash's logs to /usr/local/logstash-5.3.1/logs which is now configured via log4j2.properties
[2017-10-16T01:39:36,983][INFO ][logstash.setting.writabledirectory] Creating directory {:setting=>"path.queue", :path=>"/usr/local/logstash-5.3.1/data/queue"}
[2017-10-16T01:39:37,181][INFO ][logstash.agent ] No persistent UUID file found. Generating new UUID {:uuid=>"a2e3b22a-4785-42f6-a073-f7fad4d60a44", :path=>"/usr/local/logstash-5.3.1/data/uuid"}
[2017-10-16T01:39:37,623][INFO ][logstash.pipeline] Starting pipeline {"id"=>"main", "pipeline.workers"=>1, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>125}
[2017-10-16T01:39:37,700][INFO ][logstash.pipeline ] Pipeline main started
The stdin plugin is now waiting for input:
[2017-10-16T01:39:38,042][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
hello
2017-10-16T05:39:45.692Z centos7-1 hello
複製程式碼創始配置檔案
[root@centos7-1 logstash-5.3.1]# cd config/
[root@centos7-1 config]# ll
total 20
-rw-rw-r-- 1 root root 1738 Apr 17 12:07 jvm.options
-rw-rw-r-- 1 root root 3958 Apr 17 12:07 log4j2.properties
-rw-rw-r-- 1 root root 4433 Apr 17 12:07 logstash.yml
-rw-rw-r-- 1 root root 1701 Apr 17 12:07 startup.options
[root@centos7-1 config]# vim logstash.conf
input { stdin { } }
output {
stdout { codec=> rubydebug }
}
複製程式碼Logstash 使用 input 和 output 定義收集日誌時的輸入和輸出的相關配置，本例中 input 定義了一個叫 "stdin" 的 input ， output 定義一個叫 "stdout" 的 output 。無論我們輸入什麼字元， Logstash 都會按照某種格式來返回我們輸入的字元，其中 output 被定義為 "stdout" 並使用了 codec 引數來指定 logstash 輸出格式
[root@centos7-1 config]# /usr/local/logstash-5.3.1/bin/logstash -f /usr/local/logstash-5.3.1/config/logstash.conf
安裝Kibana
[root@centos7-1 ~]# tar zxf kibana-5.3.1-linux-x86_64.tar.gz -C /usr/local/
[root@centos7-1 ~]# cd /usr/local/kibana-5.3.1-linux-x86_64/config/
[root@centos7-1 config]# vim kibana.yml
# Kibana is served by a back end server. This setting specifies the port to use.
server.port: 5601
# To allow connections from remote users, set this parameter to a non-loopback address.
server.host: "10.0.0.5"
# The URL of the Elasticsearch instance to use for all your queries.
elasticsearch.url: "http://10.0.0.5:9200"
# Kibana uses an index in Elasticsearch to store saved searches, visualizations and
# dashboards. Kibana creates a new index if the index doesn't already exist.
kibana.index: ".kibana"
複製程式碼啟動服務
[root@centos7-1 config]# /usr/local/kibana-5.3.1-linux-x86_64/bin/kibana &
[root@centos7-1 config]# lsof -i :5601
COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME
node 11535 root 12u IPv4 30703 0t0 TCP centos7-1:esmagent (LISTEN)
複製程式碼登陸WEB介面
測試Kibana與Elasticsearch連線
相關元件安裝完成後，就需要測試下幾個元件之間能否相互協同工作，也就是能正常收集日誌，儲存日誌並展示日誌資訊
[root@centos7-1 config]# cd /usr/local/logstash-5.3.1/config/
[root@centos7-1 config]# vim logstash.conf
input {
stdin { }
}
output {
elasticsearch {
action => "index"
hosts => "10.0.0.5:9200"
index => "logstash-%{+YYYY-MM}"
}
}
[root@centos7-1 config]# /usr/local/logstash-5.3.1/bin/logstash -f /usr/local/logstash-5.3.1/config/logstash.conf
複製程式碼登陸WEB檢視是否有日誌產生