最新Centos7.6 部署ELK日誌分析系統

運維工程師日常發表於2019-07-18

下載elasticsearch

建立elk使用者並授權

useradd elk
chown -R elk:elk /home/elk/elasticsearch
chown -R elk:elk /home/elk/elasticsearch1
chown -R elk:elk /home/elk/elasticsearch2
mkdir -p /home/eladata
mkdir -p /var/log/elk
chown -R elk:elk /home/eladata
chown -R elk:elk /var/log/elk

主節點master

elasticsearch解壓,修改配置檔案

/home/elk/elasticsearch/config
[root@localhost config]# grep -v  "^#" elasticsearch.yml 
cluster.name: my-application
node.name: node0
node.master: true
node.attr.rack: r1
node.max_local_storage_nodes: 3
path.data: /home/eladata
path.logs: /var/log/elk
http.cors.enabled: true
http.cors.allow-origin: "*"
network.host: 192.168.1.70
http.port: 9200
transport.tcp.port: 9301
discovery.zen.minimum_master_nodes: 1
cluster.initial_master_nodes: ["node0"]

手動啟動命令

su elk -l -c '/home/elk/elasticsearch/bin/elasticsearch -d'

啟動檔案 elasticsearch.service

[root@localhost system]# pwd
/lib/systemd/system
[root@localhost system]# cat elasticsearch.service 
[Unit]
Description=Elasticsearch
Documentation=http://www.elastic.co
Wants=network-online.target
After=network-online.target
[Service]
RuntimeDirectory=elasticsearch
PrivateTmp=true
Environment=ES_HOME=/home/elk/elasticsearch
Environment=ES_PATH_CONF=/home/elk/elasticsearch/config
Environment=PID_DIR=/var/run/elasticsearch
EnvironmentFile=-/etc/sysconfig/elasticsearch
WorkingDirectory=/home/elk/elasticsearch
User=elk
Group=elk
ExecStart=/home/elk/elasticsearch/bin/elasticsearch -p ${PID_DIR}/elasticsearch.pid --quiet
StandardOutput=journal
StandardError=inherit
LimitNOFILE=65536
LimitNPROC=4096
LimitAS=infinity
LimitFSIZE=infinity
TimeoutStopSec=0
KillSignal=SIGTERM
KillMode=process
SendSIGKILL=no
SuccessExitStatus=143
[Install]
WantedBy=multi-user.target

[root@localhost system]#

Node1節點

/home/elk/elasticsearch1/config
[root@localhost config]# grep -v  "^#" elasticsearch.yml 
cluster.name: my-application
node.name: node1
node.master: false
node.attr.rack: r1
node.max_local_storage_nodes: 3
path.data: /home/eladata
path.logs: /var/log/elk
http.cors.enabled: true
http.cors.allow-origin: "*"
network.host: 192.168.1.70
transport.tcp.port: 9303
http.port: 9302
discovery.zen.ping.unicast.hosts: ["192.168.1.70:9301"]
[root@localhost config]#

手動啟動命令

su elk -l -c '/home/elk/elasticsearch1/bin/elasticsearch1 -d'

啟動檔案 elasticsearch1.service

[root@localhost system]# pwd
/lib/systemd/system
[root@localhost system]# cat elasticsearch1.service 
[Unit]
Description=Elasticsearch
Documentation=http://www.elastic.co
Wants=network-online.target
After=network-online.target
[Service]
RuntimeDirectory=elasticsearch1
PrivateTmp=true
Environment=ES_HOME=/home/elk/elasticsearch1
Environment=ES_PATH_CONF=/home/elk/elasticsearch1/config
Environment=PID_DIR=/var/run/elasticsearch
EnvironmentFile=-/etc/sysconfig/elasticsearch
WorkingDirectory=/home/elk/elasticsearch
User=elk
Group=elk
ExecStart=/home/elk/elasticsearch1/bin/elasticsearch -p ${PID_DIR}/elasticsearch.pid --quiet
StandardOutput=journal
StandardError=inherit
LimitNOFILE=65536
LimitNPROC=4096
LimitAS=infinity
LimitFSIZE=infinity
TimeoutStopSec=0
KillSignal=SIGTERM
KillMode=process
SendSIGKILL=no
SuccessExitStatus=143
[Install]
WantedBy=multi-user.target

[root@localhost system]#

Node2節點

/home/elk/elasticsearch2/config
[root@localhost config]# grep -v  "^#" elasticsearch.yml 
cluster.name: my-application
node.name: node2
node.attr.rack: r1
node.master: false
node.max_local_storage_nodes: 3
path.data: /home/eladata
path.logs: /var/log/elk
http.cors.enabled: true
http.cors.allow-origin: "*"
network.host: 192.168.1.70
http.port: 9203
transport.tcp.port: 9304
discovery.zen.ping.unicast.hosts: ["192.168.1.70:9301"]
discovery.zen.minimum_master_nodes: 1
[root@localhost config]#

手動啟動命令

su elk -l -c '/home/elk/elasticsearch2/bin/elasticsearch2 -d'

啟動檔案 elasticsearch2.service

[root@localhost system]# pwd
/lib/systemd/system
[root@localhost system]# cat elasticsearch2.service 
[Unit]
Description=Elasticsearch
Documentation=http://www.elastic.co
Wants=network-online.target
After=network-online.target
[Service]
RuntimeDirectory=elasticsearch2
PrivateTmp=true
Environment=ES_HOME=/home/elk/elasticsearch2
Environment=ES_PATH_CONF=/home/elk/elasticsearch2/config
Environment=PID_DIR=/var/run/elasticsearch
EnvironmentFile=-/etc/sysconfig/elasticsearch
WorkingDirectory=/home/elk/elasticsearch2
User=elk
Group=elk
ExecStart=/home/elk/elasticsearch2/bin/elasticsearch -p ${PID_DIR}/elasticsearch.pid --quiet
StandardOutput=journal
StandardError=inherit
LimitNOFILE=65536
LimitNPROC=4096
LimitAS=infinity
LimitFSIZE=infinity
TimeoutStopSec=0
KillSignal=SIGTERM
KillMode=process
SendSIGKILL=no
SuccessExitStatus=143
[Install]
WantedBy=multi-user.target

[root@localhost system]#

下載logstash

目錄如下，預設配置即可

[root@localhost logstash]# pwd
/home/elk/logstash
[root@localhost logstash]#

手動啟動命令

./logstash -f ../dev.conf 
nohup ./logstash -f ../dev.conf &

下載kibana

配置檔案如下

[root@localhost config]# pwd
/home/elk/kibana/config
[root@localhost config]# grep -v  "^#" kibana.yml 
server.host: "192.168.1.70"
elasticsearch.hosts: ["http://192.168.1.70:9200"]
kibana.index: ".kibana"
i18n.locale: "zh-CN"

手動啟動命令

./kibana
nohup ./kibana &

kibana啟動檔案

[root@localhost system]# pwd
/lib/systemd/system
[root@localhost system]# cat kibana.service 
[Unit]
Description=Kibana  Server Manager
[Service]
ExecStart=/home/elk/kibana/bin/kibana
[Install]
WantedBy=multi-user.target
[root@localhost system]#

埠為：5601 訪問：192.168.1.70:5601

安裝Elasticsearch -head

yum install git npm
git clone https://github.com/mobz/elasticsearch-head.git 
[root@localhost elasticsearch-head]# pwd
/home/elk/elasticsearch-head
[root@localhost elasticsearch-head]#

啟動

npm install 
npm run start
nohup npm run start &

curl -XPUT '192.168.2.67:9100/book'

訪問192.168.2.67：9100 即可訪問

下載kafka

修改配置檔案如下

[root@localhost config]# pwd
/home/elk/kafka/config
[root@localhost config]# grep -v "^#" server.properties 
broker.id=0
listeners=PLAINTEXT://192.168.1.70:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/var/log/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=localhost:2181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0
delete.topic.enable=true
[root@localhost config]#

kafka配置啟動zookeeper

手動啟動方式

[root@localhost bin]# pwd
/home/elk/kafka/bin
[root@localhost bin]#
./zookeeper-server-start.sh ../config/zookeeper.properties

systemctl 啟動zookeeper

[root@localhost system]# pwd
/lib/systemd/system
[root@localhost system]# cat zookeeper.service 
[Service]
Type=forking
SyslogIdentifier=zookeeper
Restart=always
RestartSec=0s
ExecStart=/home/elk/kafka/bin/zookeeper-server-start.sh -daemon /home/elk/kafka/config/zookeeper.properties
ExecStop=/home/elk/kafka/bin/zookeeper-server-stop.sh
[root@localhost system]#

啟動kafka服務

手動啟動方式

./kafka-server-start.sh ../config/server.properties

systemctl 啟動kafka

[root@localhost system]# pwd
/lib/systemd/system
[root@localhost system]# cat kafka.service 
[Unit]
Description=Apache kafka
After=network.target
[Service]
Type=simple
Restart=always
RestartSec=0s
ExecStart=/home/elk/kafka/bin/kafka-server-start.sh  /home/elk/kafka/config/server.properties
ExecStop=/home/elk/kafka/bin/kafka-server-stop.sh
[root@localhost system]#

測試kafka

新建一個名字為test的topic

/kafka-topics.sh --create --zookeeper 192.168.1.70:2181 --replication-factor 1 --partitions 1 --topic test

檢視kafka中的topic

./kafka-topics.sh --list  --zookeeper 192.168.1.70:2181

往kafka topic為test中生產訊息

./kafka-console-producer.sh --broker-list 192.168.1.70:9092 --topic test

在kafka topic為test中消費訊息

bin/kafka-console-consumer.sh --bootstrap-server 192.168.1.70:9092 --topic test --from-beginning

生產的訊息，消費那邊接受到即是ok的

目標機器安裝filebeat

安裝6.5版本的

[root@localhost filebeat]# pwd
/usr/local/filebeat
[root@localhost filebeat]# cat filebeat.yml 
filebeat.prospectors:
- type: log
  paths:
    - /opt/logs/workphone-tcp/catalina.out
  fields:
     tag: 54_tcp_catalina_out
- type: log
  paths:
    - /opt/logs/workphone-webservice/catalina.out
  fields:
     tag: 54_web_catalina_out
name: 192.168.1.54
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 3
output.kafka:
  hosts: ["192.168.1.70:9092"]
  topic: "filebeat-log"
  partition.hash:
    reachable_only: true
  compression: gzip
  max_message_bytes: 1000000
  required_acks: 1

[root@localhost filebeat]#

安裝完成後去logstash編輯配置檔案

logstash操作

[root@localhost logstash]# pwd
/home/elk/logstash
[root@localhost logstash]# cat dev.conf 
input {
  kafka{
    bootstrap_servers => "192.168.1.70:9092"
    topics => ["filebeat-log"]
    codec => "json"
  }
}
filter {
        if [fields][tag]=="jpwebmap" {
            json{
                source => "message"
                remove_field => "message"
            }
            geoip {
            source => "client"
            target => "geoip"
            add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
            add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
            }
            mutate {
                convert => [ "[geoip][coordinates]", "float"]
                }
        }
    if [fields][tag] == "54_tcp_catalina_out"{
            grok {
                match => ["message", "%{TIMESTAMP_ISO8601:logdate}"]
            }
            date {
                match => ["logdate", "ISO8601"]
            }
            mutate {
                remove_field => [ "logdate" ]
            }
    }
    if [fields][tag] == "54_web_catalina_out"{
                grok {
                        match => ["message", "%{TIMESTAMP_ISO8601:logdate}"]
                }
                date {
                        match => ["logdate", "ISO8601"]
                }
                mutate {
                        remove_field => [ "logdate" ]
                }
        }
    if [fields][tag] == "55_tcp_catalina_out"{
                grok {
                        match => ["message", "%{TIMESTAMP_ISO8601:logdate}"]
                }
                date {
                        match => ["logdate", "ISO8601"]
                }
                mutate {
                        remove_field => [ "logdate" ]
                }
        }
        if [fields][tag] == "55_web_catalina_out"{
                grok {
                        match => ["message", "%{TIMESTAMP_ISO8601:logdate}"]
                }
                date {
                        match => ["logdate", "ISO8601"]
                }
                mutate {
                        remove_field => [ "logdate" ]
                }
        }
    if [fields][tag] == "51_nginx80_access_log" {
            mutate {
                add_field => { "spstr" => "%{[log][file][path]}" }
            }
            mutate {
                split => ["spstr" , "/"]
                # save the last element of the array as the api_method.
                add_field => ["src", "%{[spstr][-1]}" ]
            }
            mutate{
                remove_field => [ "friends", "ecs", "agent" , "spstr" ]
            }
            grok {
                match => { "message" => "%{IPORHOST:remote_addr} - %{DATA:remote_user} \[%{HTTPDATE:time}\] \"%{WORD:method} %{DATA:url} HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} %{NUMBER:body_sent:bytes} \"%{DATA:referrer}\" \"%{DATA:agent}\" \"%{DATA:x_forwarded_for}\" \"%{NUMBER:request_time}\" \"%{DATA:upstream_addr}\" \"%{DATA:upstream_status}\"" }
                remove_field => "message"
            }
            date {
                    match => ["time", "dd/MMM/yyyy:HH:mm:ss Z"]
                    target => "@timestamp"
            }
            geoip {
                source => "x_forwarded_for"
                target => "geoip"
                database => "/home/elk/logstash/GeoLite2-City.mmdb"
                add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
                add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
            }
            mutate {
                convert => [ "[geoip][coordinates]", "float"]
            }
    }
}
output {
if [fields][tag] == "wori"{
  elasticsearch {
   hosts => ["192.168.1.70:9200"]
   index => "zabbix"
       }
   }
if [fields][tag] == "54_tcp_catalina_out"{
  elasticsearch {
   hosts => ["192.168.1.70:9200"]
   index => "54_tcp_catalina_out"
       } 
   }
if [fields][tag] == "54_web_catalina_out"{
  elasticsearch {
   hosts => ["192.168.1.70:9200"]
   index => "54_web_catalina_out"
       } 
   }
if [fields][tag] == "55_tcp_catalina_out"{
  elasticsearch {
   hosts => ["192.168.1.70:9200"]
   index => "55_tcp_catalina_out"
       } 
   }   
if [fields][tag] == "55_web_catalina_out"{
  elasticsearch {
   hosts => ["192.168.1.70:9200"]
   index => "55_web_catalina_out"
       } 
   }
if [fields][tag] == "51_nginx80_access_log" {
       stdout{}
    elasticsearch {
    hosts => ["192.168.1.70:9200"]
    index => "51_nginx80_access_log"
    }
   }
}

其他的配置檔案

index.conf

filter {
    mutate {
        add_field => { "spstr" => "%{[log][file][path]}" }
    }
        mutate {
        split => ["spstr" , "/"]
        # save the last element of the array as the api_method.
        add_field => ["src", "%{[spstr][-1]}" ]
        }
        mutate{
    remove_field => [ "friends", "ecs", "agent" , "spstr" ]
    }
}

java.conf

filter {
if [fields][tag] == "java"{
    grok {
        match => ["message", "%{TIMESTAMP_ISO8601:logdate}"]
    }
    date {
        match => ["logdate", "ISO8601"]
    }
    mutate {
         remove_field => [ "logdate" ]
    }
  } #End if
}

kafkainput.conf

input {
  kafka{
    bootstrap_servers => "172.16.11.68:9092"
    #topics => ["ql-prod-tomcat" ]
    topics => ["ql-prod-dubbo","ql-prod-nginx","ql-prod-tomcat" ]
    codec => "json"
    consumer_threads => 5
    decorate_events => true
    #auto_offset_reset => "latest"
    group_id => "logstash"
    #client_id => ""
    ############################# HELK Optimizing Latency #############################
    fetch_min_bytes => "1"
    request_timeout_ms => "305000"
    ############################# HELK Optimizing Availability #############################
    session_timeout_ms => "10000"
    max_poll_records => "550"
    max_poll_interval_ms => "300000"
  }

}
#input {
#  kafka{
#    bootstrap_servers => "172.16.11.68:9092"
#    topics => ["ql-prod-java-dubbo","ql-prod","ql-prod-java" ]
#    codec => "json"
#    consumer_threads => 15
#    decorate_events => true
#    auto_offset_reset => "latest"
#    group_id => "logstash-1"
#    ############################# HELK Optimizing Latency #############################
#    fetch_min_bytes => "1"
#    request_timeout_ms => "305000"
#    ############################# HELK Optimizing Availability #############################
#    session_timeout_ms => "10000"
#    max_poll_records => "550"
#    max_poll_interval_ms => "300000"
#  }

#}

nginx.conf

filter {
if [fields][tag] == "nginx-access" {
        mutate {
        add_field => { "spstr" => "%{[log][file][path]}" }
        }
        mutate {
        split => ["spstr" , "/"]
        # save the last element of the array as the api_method.
        add_field => ["src", "%{[spstr][-1]}" ]
        }
        mutate{
        remove_field => [ "friends", "ecs", "agent" , "spstr" ]
        }

    grok {
        match => { "message" => "%{IPORHOST:remote_addr} - %{DATA:remote_user} \[%{HTTPDATE:time}\] \"%{WORD:method} %{DATA:url} HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} %{NUMBER:body_sent:bytes} \"%{DATA:referrer}\" \"%{DATA:agent}\" \"%{DATA:x_forwarded_for}\" \"%{NUMBER:request_time}\" \"%{DATA:upstream_addr}\" \"%{DATA:upstream_status}\"" }
        remove_field => "message"
    }
    date {
                match => ["time", "dd/MMM/yyyy:HH:mm:ss Z"]
                target => "@timestamp"
        }
    geoip {
        source => "x_forwarded_for"
        target => "geoip"
        database => "/opt/logstash-6.2.4/GeoLite2-City.mmdb"
        add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
        add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]

        }
    mutate {
        convert => [ "[geoip][coordinates]", "float"]
    }

  } #endif
}

ouput.conf

output{
  if [fields][tag] == "nginx-access" {
       stdout{}
    elasticsearch {
    user => elastic
    password => WR141bp2sveJuGFaD4oR
    hosts => ["172.16.11.67:9200"]
    index => "logstash-%{[fields][proname]}-%{+YYYY.MM.dd}"
    }
   }
       #stdout{}
   if [fields][tag] == "java" {
        elasticsearch {
        user => elastic
        password => WR141bp2sveJuGFaD4oR
        hosts => ["172.16.11.66:9200","172.16.11.68:9200"]
        index => "%{[host][name]}-%{[src]}"
        }
  }
}

ELK 日誌分析系統 ----------- 部署ElasticSearch群集
2020-10-29
Elasticsearch
ELK日誌分析系統
2020-11-18
elk 日誌分析系統
2015-06-15
【ELK】日誌分析系統
2016-06-06
ELK-日誌分析系統
2020-09-23
ELK+FileBeat日誌分析系統
2021-07-15
ELK日誌分析系統詳解
2020-11-29
企業級日誌分析系統——ELK
2020-11-19
ELK前端日誌分析、監控系統
2016-11-28
前端
ELK 日誌分析體系
2016-06-01
基於AWS-ELK部署系統日誌告警系統
2021-09-09
centos7 安裝elk日誌分析系統
2017-12-13
CentOS
使用Docker快速部署ELK分析Nginx日誌實踐
2018-08-25
DockerNginx
搭建ELK日誌分析系統（四）-kibana安裝和使用
2017-08-30
ELK實時日誌分析平臺環境部署--完整記錄
2016-09-29
ELK日誌系統之通用應用程式日誌接入方案
2018-09-05
ELK日誌分析系統超詳細！！理論+實操講解！！
2020-11-13
ELK日誌
2020-11-23
Windows系統搭建ELK日誌收集伺服器
2021-04-16
Windows伺服器
Kafka與ELK實現一個日誌系統
2022-02-28
Kafka
開源元件ELK日誌系統配置與管理
2018-03-16
元件
ELK日誌系統之使用Rsyslog快速方便的收集Nginx日誌
2018-08-29
Nginx
爬蟲日誌監控 -- Elastc Stack（ELK）部署
2020-09-26
爬蟲AST
ELK+kafka+Winlogbeat/FileBeat搭建統一日誌收集分析管理系統
2018-08-16
Kafka
Centos部署Loki日誌聚合系統
2022-03-07
CentOSLoki
日誌分析系統 - k8s部署ElasticSearch叢集
2022-01-05
K8SElasticsearch
日誌分析平臺ELK之日誌收集器filebeat
2020-10-04
【ELK】elastalert 日誌告警
2021-01-22
AST
部署Sentry日誌監控系統
2022-08-28
日誌分析平臺ELK之日誌收集器logstash
2020-10-02
go-kit 微服務日誌分析管理（ELK + Filebeat）
2020-03-16
Go微服務
微服務下，使用ELK做日誌收集及分析
2019-06-13
微服務
使用 ELK 搭建日記系統
2020-06-09
Centos7下ELK+Redis日誌分析平臺的叢集環境部署記錄
2018-05-29
CentOSRedis
ElasticSearch實戰系列九: ELK日誌系統介紹和安裝
2020-10-24
Elasticsearch
日誌系統ELK使用詳解(四)--kibana安裝和使用
2017-05-14
ELK實時分析之php的laravel專案日誌
2019-01-03
PHPLaravel
Splunk + Forwarder 收集分析Windows系統日誌
2017-11-20
ForwardWindows

最新Centos7.6 部署ELK日誌分析系統

下載elasticsearch

建立elk使用者並授權

主節點master

elasticsearch解壓,修改配置檔案

手動啟動命令

啟動檔案 elasticsearch.service

Node1節點

手動啟動命令

啟動檔案 elasticsearch1.service

Node2節點

手動啟動命令

啟動檔案 elasticsearch2.service

下載logstash

目錄如下，預設配置即可

手動啟動命令

下載kibana

配置檔案如下

手動啟動命令

kibana啟動檔案

埠為：5601 訪問：192.168.1.70:5601

安裝Elasticsearch -head

啟動

curl -XPUT '192.168.2.67:9100/book'

訪問192.168.2.67：9100 即可訪問

下載kafka

修改配置檔案如下

kafka配置啟動zookeeper

手動啟動方式

systemctl 啟動zookeeper

啟動kafka服務

手動啟動方式

systemctl 啟動kafka

測試kafka

新建一個名字為test的topic

檢視kafka中的topic

往kafka topic為test中 生產訊息

在kafka topic為test中 消費訊息

目標機器安裝filebeat

logstash操作

其他的配置檔案

index.conf

java.conf

kafkainput.conf

nginx.conf

ouput.conf

相關文章

往kafka topic為test中生產訊息

在kafka topic為test中消費訊息