07 . ELK Stack7.2一鍵多機部署指令碼

men發表於2020-08-02

一鍵部署指令碼

目錄結構

tree Log_Analysis_Platform_Document
Log_Analysis_Platform_Document
├── InstallES.sh
├── InstallFilebeat.sh
├── InstallKibana.sh
└── README.md

ES.sh

#!/usr/bin/env bash
# *************************************************************************************************************
# Author: ZhouJian
# Mail: 18621048481@163.com
# Data: 2019-9-7
# Describe: CentOS 7 AutoInstall Elasticsearchn-7.2 Deploy Script

# ****************************Elasticsearch Deplay Script******************************************************
clear
ESIP=`ip addr | grep "inet" | grep -v "127.0.0.1" | grep -v "inet6" | awk -F/ '{print $1}' | awk '{print $2}' `

echo -e "\033[32m ############################################################################# \033[0m"
echo -e "\033[32m #                           Auto Install ELK.                              ## \033[0m"
echo -e "\033[32m #                           Press Ctrl + C to cancel                       ## \033[0m"
echo -e "\033[32m #                           Any key to continue                            ## \033[0m"
echo -e "\033[32m # Softwae:elasticsearch-7.2.0/logstash-7.2.0/filebeat-7.2.0/kibana-7.2.0   ## \033[0m"
echo -e "\033[32m ############################################################################# \033[0m"

Read_Input() {
echo -e "\033[32m Please Input You Kibana Pass Key IP: \033[0m"
read -p "Please Input You HOST Pass Key IP:[192.168.244.55]" KibanaIP
read -p "Please Input You HOST Pass Key IP: Password:" KibanaPass

echo -e "\033[32m Please Input You Filebeat Pass Key IP: \033[0m"
read -p "Please Input You HOST Pass Key IP:[192.168.244.56]" FilebeatIP
read -p "Please Input You HOST Pass Key IP: Password:" FilebeatPass
}

Init_Yumsource() 
{
	if ! ping -c2 www.baidu.com &>/dev/null
	then
        	echo "您無法上外網，不能配置yum源"
        exit
	fi
		echo "配置yum源"
	if [ ! -d /etc/yum.repos.d/backup ];then
        	mkdir /etc/yum.repos.d/backup
		mv /etc/yum.repos.d/* /etc/yum.repos.d/backup 2>/dev/null	
		curl -o /etc/yum.repos.d/163.repo http://mirrors.163.com/.help/CentOS7-Base-163.repo &>/dev/null
    		curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo &>/dev/null
	fi
}

# *************************************************************************************************************
Init_Hostname() 
{
	hostnamectl set-hostname elk-1
	echo "$ESIP elk-1" >> /etc/hosts	
}


# *************************************************************************************************************
Init_SElinux() 
{
	echo "關閉防火牆"
	systemctl stop firewalld
	systemctl disable firewalld
	echo "關閉selinux"
	setenforce 0
	sed -ri '/^SELINUX=/ s/enforcing/disabled/'  /etc/selinux/config
	echo "解決sshd遠端連線慢的問題"
	sed -ri '/^GSSAPIAu/ s/yes/no/' /etc/ssh/sshd_config
	sed -ri '/^#UseDNS/ {s/^#//;s/yes/no/}' /etc/ssh/sshd_config
	systemctl enable sshd crond &> /dev/null
}

# **************************************************************************************************************
Create_UserLogFile() 
{
	groupadd elk
	useradd elk -g elk
	mkdir -pv /data/elk/{data,logs}
	chown -R elk:elk /data/
}
# **************************************************************************************************************

Unpackaged_Authorization() 
{
	yum -y install  ntpdate
	rpm -ivh  /root/InstallELKB-Shell/jdk-8u121-linux-x64.rpm
	tar xvf /root/InstallELKB-Shell/elasticsearch-7.2.0-linux-x86_64.tar.gz -C /opt/
	chown -R elk:elk /opt/elasticsearch-7.2.0/
	ntpdate -b ntp1.aliyun.com
}

# **************************************************************************************************************
Set_System_Parameter() 
{
cat >> /etc/security/limits.conf <<EOF
* soft nproc 2048
* hard nproc 4096
* soft nofile 65536
* hard nofile 131072
EOF

echo "vm.max_map_count = 262144" >> /etc/sysctl.conf && sysctl -p
cat >> /etc/profile <<EOF
export HISTTIMEFORMAT="%Y-%m-%d %H:%M:%S "
EOF
	source /etc/profile

cat >> /opt/elasticsearch-7.2.0/config/elasticsearch.yml <<EOF
cluster.name: elk
node.name: node-1
bootstrap.memory_lock: false
path.data: /data/elk/data
path.logs: /data/elk/logs
network.host: 0.0.0.0
http.port: 9200
discovery.seed_hosts: ["elk-1"]
cluster.initial_master_nodes: ["node-1"]
EOF
	runuser -l elk -c '/bin/bash /opt/elasticsearch-7.2.0/bin/elasticsearch ' &> /opt/elasticsearch.log  &
}

Test_Service() 
{
	esport=`ss -antp |grep :::9200 | awk -F::: '{print $2}'`
	if [ $esport -eq 9200 ];then
		echo -e  "\033[32m Elasticsearch is OK... \033[0m "
	fi
}

# **********************PublicKeyKibana******************************************************************************


PublicKeyKibana() 
{
if [ ! -f /usr/bin/expect ];then
	yum -y install expect
fi
sed -i 's/# *StrictHostKeyChecking *ask/StrictHostKeyChecking no/g' /etc/ssh/ssh_config
systemctl restart sshd

cd /root/.ssh/
ssh-keygen -t rsa -N '' -f id_rsa -q
if [ $? -eq 0 ];then
/usr/bin/expect <<-EOF
set timeout 10
spawn ssh-copy-id $KibanaIP
expect {
    "yes/no" { send "yes\r"; exp_continue }
    "password:" { send "$KibanaPass\r"}
}
expect eof
EOF
fi

}

# **********************Kibana Deploy Script********************************************************************
Install_Kibana() 
{
echo $ESIP > /root/InstallELKB-Shell/ESIP.txt
scp /root/InstallELKB-Shell/kibana-7.2.0-linux-x86_64.tar.gz $KibanaIP:
scp /root/InstallELKB-Shell/ESIP.txt $KibanaIP:
scp /root/InstallELKB-Shell/InstallKibana.sh $KibanaIP:
ssh root@$KibanaIP '
bash /root/InstallKibana.sh '
}



# *******************************************Filebeat Deploy Script***************************************************
PublicFilebeat() 
{
if [ ! -f /usr/bin/expect ];then
	yum -y install expect
fi
sed -i 's/# *StrictHostKeyChecking *ask/StrictHostKeyChecking no/g' /etc/ssh/ssh_config
systemctl restart sshd
cd /root/.ssh/
rm -rf /root/.ssh/*
ssh-keygen -t rsa -N '' -f id_rsa -q
if [ $? -eq 0 ];then
/usr/bin/expect <<-EOF
set timeout 10
spawn ssh-copy-id $FilebeatIP
expect {
    "yes/no" { send "yes\r"; exp_continue }
    "password:" { send "$FilebeatPass\r"}
}
expect eof
EOF
fi
}

Install_Filebeat() 
{
	scp /root/InstallELKB-Shell/filebeat-7.2.0-x86_64.rpm  $FilebeatIP:
	scp /root/InstallELKB-Shell/InstallFilebeat.sh $FilebeatIP:
	ssh root@$FilebeatIP 'bash /root/InstallFilebeat.sh'
	scp /root/InstallELKB-Shell/filebeat.yml $FilebeatIP:/etc/filebeat/ 
	ssh root@$FilebeatIP 'systemctl restart filebeat && systemctl disable filebeat && rm -rf /root/InstallFilebeat.sh' 
}


# ********************************************Logstash******************************************************************
Install_logstash() 
{
	tar xvf /root/InstallELKB-Shell/logstash-7.2.0.tar.gz  -C /opt/
	cp /root/InstallELKB-Shell/nginx.yml  /opt/logstash-7.2.0/
	/opt/logstash-7.2.0/bin/logstash -f /opt/logstash-7.2.0/nginx.yml   &>/opt/logstash.log &	
}


ES-StartUp_SelfStart() 
{
cat >> /etc/init.d/elasticsearch.sh <<EOF
nohup  runuser -l elk -c '/bin/bash /opt/elasticsearch-7.2.0/bin/elasticsearch' &
nohup /opt/logstash-7.2.0/bin/logstash -f /opt/nginx.yml &
EOF
	echo "/etc/init.d/elasticsearch.sh"  >> /etc/rc.d/rc.local
	chmod +x /etc/init.d/elasticsearch.sh
	chmod +x /etc/rc.d/rc.local 
}

main() {
#######Elasticsearch#######
Read_Input
Init_Yumsource
Init_Hostname
Init_SElinux
Create_UserLogFile
Unpackaged_Authorization
Set_System_Parameter
Test_Service
#########Kibana###########
PublicKeyKibana
Install_Kibana

########Filebeat#########
PublicFilebeat
Install_Filebeat

ES-StartUp_SelfStart
Kibana-StartUp_SelfStart
#######Logstash#########
Install_logstash
}
main

Kibana.sh

#!/usr/bin/env bash
# ***************************************************************************************************
# Author: ZhouJian
# MaiBox: 18621048481@163.com
# Data: 2019-9-7
# Describe: CentOS 7 Deploy Kibana Script

elastip=$(cat /root/ESIP.txt )
if [ ! -d /opt/kibana-7.2.0-linux-x86_64 ];then
	tar xvf /root/kibana-7.2.0-linux-x86_64.tar.gz -C /opt/
fi
# ***************************************************************************************************


init_yumsource() 
{
if ! ping -c2 www.baidu.com &>/dev/null
then
        echo "您無法上外網，不能配置yum源"
        exit
fi
echo "配置yum源"
if [ ! -d /etc/yum.repos.d/backup ];then
mkdir /etc/yum.repos.d/backup
	mv /etc/yum.repos.d/* /etc/yum.repos.d/backup 2>/dev/null
	curl -o /etc/yum.repos.d/163.repo http://mirrors.163.com/.help/CentOS7-Base-163.repo &>/dev/null
	curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo &>/dev/null
	yum -y install  ntpdate
    ntpdate -b ntp1.aliyun.com
fi
}

# ***************************************************************************************************
init_SElinux() 
{
	echo "關閉防火牆"
	systemctl stop firewalld
	systemctl disable firewalld
	echo "關閉selinux"
	setenforce 0
	sed -ri '/^SELINUX=/ s/enforcing/disabled/'  /etc/selinux/config
	echo "解決sshd遠端連線慢的問題"
	sed -ri '/^GSSAPIAu/ s/yes/no/' /etc/ssh/sshd_config
	sed -ri '/^#UseDNS/ {s/^#//;s/yes/no/}' /etc/ssh/sshd_config
	systemctl enable sshd crond &> /dev/null
}

# ***************************************************************************************************
SetKibanaParameter() 
{
cat >> /opt/kibana-7.2.0-linux-x86_64/config/kibana.yml <<EOF
server.host: "0.0.0.0"
server.port: 5601
elasticsearch.hosts: ["http://$elastip:9200"]
EOF
}

# ***************************************************************************************************
StartKibana() 
{
	/opt/kibana-7.2.0-linux-x86_64/bin/kibana --allow-root &>/opt/kibana.log &
}

Test_Service() 
{
	KibanaPort=` ss -antp | grep 5601 | awk '{print $4}' | awk -F*: '{print $NF}'`
    if [ $KibanaPort -eq 5601 ];then
    	echo -e  "\033[32m Kibana is OK... \033[0m "
    fi
}

DeleteUselessFiles() 
{
	rm -rf /root/kibana-7.2.0-linux-x86_64.tar.gz
	rm -rf /root/InstallKibana.sh
	rm -rf /root/ESIP.txt
}

Kibana-StartUp_SelfStart() 
{
        echo "nohup /opt/kibana-7.2.0-linux-x86_64/bin/kibana --allow-root  &" >> /etc/init.d/kibana.sh
        echo "/bin/bash /etc/init.d/kibana.sh" >> /etc/rc.local
        chmod +x /etc/init.d/kibana.sh
        chmod +x /etc/rc.local
}


init_SElinux
SetKibanaParameter
StartKibana
Test_Service
DeleteUselessFiles
Kibana-StartUp_SelfStart

Filebeat.sh

#!/usr/bin/env bash # *************************************************************
# Author: ZhouJian
# Mail: 18621048481@163.com
# Data: 2019-9-7
# Describe: CentOS 7 Deploy Filebeat7.2 Script

# *************************************************************

Init_Yumsource() 
{
	if ! ping -c2 www.baidu.com &>/dev/null
	then
		echo "您無法上外網，不能配置yum源"
		exit
	fi
		echo "配置yum源"
	if [ ! -d /etc/yum.repos.d/backup ];then
        mkdir /etc/yum.repos.d/backup
        mv /etc/yum.repos.d/* /etc/yum.repos.d/backup 2>/dev/null
        curl -o /etc/yum.repos.d/163.repo http://mirrors.163.com/.help/CentOS7-Base-163.repo &>/dev/null
        curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo &>/dev/null
	yum -y install ntpdate
	ntpdate -b ntp1.aliyun.com
	fi
}

Init_SElinux() 
{
	echo "關閉防火牆"
	systemctl stop firewalld
	systemctl disable firewalld
	echo "關閉selinux"
	setenforce 0
	sed -i '/^SELINUX=/ s/enforcing/disabled/'  /etc/selinux/config
	echo "解決sshd遠端連線慢的問題"
	sed -i '/^GSSAPIAu/ s/yes/no/' /etc/ssh/sshd_config
	sed -i '/^#UseDNS/ {s/^#//;s/yes/no/}' /etc/ssh/sshd_config
	systemctl enable sshd crond &> /dev/null
}


Install_Filebeat() 
{
	yum -y install ntpdate
	ntpdate -b ntp1.aliyun.com
	rpm -ivh /root/filebeat-7.2.0-x86_64.rpm
	rm -rf /root/filebeat-7.2.0-x86_64.rpm
}

Init_Yumsource
Init_SElinux
Install_Filebeat

README.md

環境要求：

# CentOS7
# Javaa 1.8

IP	hostname	軟體	記憶體要求
192.168.122.3	elk-1	Elasticsearch、Logstash	2G及以上
192.168.122.4	Kibana	Kibana	1G及以上
192.168.122.5	Filebeat	Filebeat	1G及以上

注意事項

# 1.一定要對時，時間校正，不然日誌出不來；
# 2.java包最好用openjdk；
# 3.啟動Elasticsearch必須切換成所建立的ELK使用者啟動，不然ES出於安全目的，會啟動報錯；
# 4.日誌從Filebeat到Logstash再到ES檢索到Kibana的讀取速度取決於機器配置，注意用
# cat  日誌檔案*  |  wc  -l   統計日誌數量，然後到Elasticsearch去看總數量，確保日誌都過來了在進行分析；

Elasticsearch安裝

初始化

curl -o /etc/yum.repos.d/163.repo http://mirrors.163.com/.help/CentOS7-Base-163.repo &>/dev/null
curl  -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo 
	yum -y install ntpdate
	ntpdate -b  ntp1.aliyun.com

設定Hostname解析

		hostnamectl set-hostname elk-1
		## 修改/etc/hosts 增加如下內容
		192.168.122.3     elk-1

java安裝

		# 安裝java 1.8
		yum -y install java-1.8.0-openjdk.x86_64

關閉防火牆，SeLinux

		setenforce 0
		sed -i '/^SELINUX=/ s/enforcing/disabled/'  /etc/selinux/config
		systemctl stop firewalld
		systemctl disable firewalld
		sed -i '/^GSSAPIAu/ s/yes/no/' /etc/ssh/sshd_config
		sed -i '/^#UseDNS/ {s/^#//;s/yes/no/}' /etc/ssh/sshd_config

建立使用者和組

		# create  user elk
		groupadd  elk
		useradd  elk  -g  elk

建立資料及日誌檔案並授權

		mkdir  -pv  /data/elk/{data,logs}
		chown  -R  elk:elk  /data/elk/

軟體包解壓、授權

# 上傳軟體包
#	通過scp 或者FTP方式上傳到/opt下
# 解壓軟體包到/opt目錄
		tar xvf elasticsearch-7.2.0-linux-x86_64.tar.gz -C  /opt/
# 授權
		chown  -R  elk:elk  # 軟體包名

elk-1配置檔案

		# 叢集名
		cluster.name:  elk
		# 節點名
		node.name: node-1
		# 儲存資料
		path.data:  /data/elk/data
		# 存放日誌
		path.logs:  /data/elk/logs
		# 鎖記憶體，儘量不使用交換記憶體
		bootstrap.memory_locak:  false
		# 網路地址
		network.host: 0.0.0.0
		http.port: 9200
		# 發現叢集hosts
		discovery.sead_hosts: ["elk-1"]
		# 設定叢集master節點
		cluster.inital_master_nodes: ["node-1"]

修改/etc/security/limits.conf

		# *號不是註釋
		* soft nofile 65536
		* hard nofile 131072
		* soft nproc 2048
		* hard nproc 4096

修改/etc/sysctl.conf

		echo "vm.max_map_count=262144" >> /etc/sysctl.conf
		sysctl -p

ES啟動

		nohup runuser -l elk -c '/bin/bash /opt/elasticsearch-7.2.0/bin/elasticsearch' &

檢查叢集健康狀態

		curl -XGET 'elk-1:9200/_cluster/health?pretty'

Kibana安裝使用

解壓Kibana安裝包

		tar xvf kibana-7.2.0-linux-x86_64.tar.gz  -C /opt/

修改Kibana配置檔案

		vim /opt/kibana-7.2.0-linux-x86_64/config/kibana.yml
			server.port:  5601   # Port
			server.host:  0.0.0.0   # 訪問限制
			elasticsearch.hosts: ["http://ESHostIP:9200"]

啟動命令

		/opt/kibana-7.2.0-linux-x86_64/bin/kibana --allow-root
		nohup  /opt/kibana-7.2.0-linux-x86_64/bin/kibana --allow-root  &  放入後臺使用
		tailf  nohup.out   # 實時檢視服務執行狀態

Filebeat 安裝使用

下載安裝

修改配置檔案(修改/etc/filebeat/filebeat.yml)

		filebeat.inputs:
		- type: log
		  enabled: true
		  paths:
		    - /var/log/*.log		# 抓取檔案日誌路徑
		    		# output.elasticsearch:
		#  hosts: ["ESHostIP:9200"]		# 輸出到ES

Filebeat到Lostash

Filebeat配置

（vim /etc/filebeat/filebeat.yml） shift + ：輸入set nu 顯示行號

			24:   enabled:  true				# 更改為true以啟用輸入配置
			28:   - /var/log/*.log           #   替換為要抓取的日誌檔案路徑
			73：  reload.enabled:  true	#		啟動Filebeat模組
			148： output.elasticsearch:   	  #  加上註釋；
			150:  hosts: ["localhost:9200"]   #   加上註釋；
			158： output.logstash:			#	去掉註釋；
			160： hosts: ["localhost:5044"]	#	去掉註釋，並修改localhost為logstash機器IP及對應埠號；

`測試配置檔案並啟動`

			filebeat  test  config  -e
			systemctl  start filebeat
			systemctl  enable filebeat

Logstash 安裝使用

解壓安裝

上傳包

tar xvf logstash-7.2.0.tar.gz -C /opt/

`啟動`

	/opt/logstash-7.2.0/bin/logstash -f /opt/配置檔名.yml
	## 後臺執行
	nohup  /opt/logstash-7.2.0/bin/logstash -f /opt/配置檔名.yml  &

Logstash到Elasticsearch

主要看配置檔案，配置檔案對了，直接按照上面命令啟動就可以了；

# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.

input {
  beats {
    port => 5044
  }
}

filter {
        grok {
                match => {
               "message" => " %{DATA:log_date} %{TIME:log_localtime} %{JAVAFILE:name_file} %{WORD:workd}\[%{WORD:ls}\]\: %{DATA:log_date2} %{TIME:log_localtime2} %{WORD:year_tmp}\: %{WORD:name_2}\: %{WORD:} %{WORD:}\, %{JAVAFILE:}\: %{JAVAFILE:app_id}\, %{WORD}\: %{IP:ip}\, %{WORD:}\: %{INT}\, %{WORD}\: %{USERNAME:device_id}"
                }
        }
}

output {
   elasticsearch {
      hosts => ["http://ElasticsearchHostIP:9200"]
      index => "nginx_log-%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
   }
}

一鍵部署指令碼
2019-03-29
指令碼
Docker 指令碼化一鍵部署
2019-05-06
Docker指令碼
一鍵部署lamp指令碼記錄
2017-11-27
LAMP指令碼
shell指令碼攻略--DNS正向解析一鍵部署
2020-12-30
指令碼DNS
k3s單機版安裝部署附一鍵安裝指令碼
2021-11-10
指令碼
輕鬆部署 Laravel 應用 | 《02. 一鍵指令碼》
2019-03-04
Laravel指令碼
ELK部署
2024-07-18
輕鬆部署 Laravel 應用 | 《03. 探尋一鍵指令碼》
2019-03-04
Laravel指令碼
shell指令碼：一鍵安裝LAMP、LNMP指令碼
2016-04-04
指令碼LAMPLNMP
LAMPorLNMP一鍵安裝指令碼
2017-11-08
LAMPLNMP指令碼
一次rman 指令碼的部署
2011-07-12
指令碼
ELK安裝部署
2024-03-29
shell指令碼一鍵安裝nginx
2018-12-05
指令碼Nginx
一鍵RAC環境配置指令碼
2016-05-09
指令碼
DSS+Linkis Ansible 單機一鍵安裝指令碼
2022-12-28
指令碼
一次 oracle rman 指令碼的部署
2011-07-12
Oracle指令碼
golang一鍵自動安裝指令碼
2019-06-11
Golang指令碼
LNMP一鍵自動安裝指令碼
2017-11-14
LNMP指令碼
mysql8.0原始碼一鍵安裝指令碼
2022-09-21
MySql原始碼指令碼
jenkins + supervisor + ansible 實現netcore程式的多機一鍵部署
2019-01-29
JenkinsNetCore
docker-compose部署ELK
2024-05-03
Docker
Shell指令碼 | 一鍵解除安裝安卓App
2019-02-22
指令碼安卓APP
centos7 mysql 一鍵安裝指令碼
2020-11-26
CentOSMySql指令碼
CentOS下LAMP一鍵yum安裝指令碼
2017-05-05
CentOSLAMP指令碼
cacti一鍵安裝指令碼V1.1
2017-11-14
指令碼
LNMP一鍵自動安裝指令碼薦
2013-01-06
LNMP指令碼
LNMP自動部署指令碼
2017-11-14
LNMP指令碼
通用程式部署shell指令碼
2024-05-31
指令碼
MyBatis07-(多對一、一對多)
2020-10-28
MyBatis
rsync客戶端一鍵安裝rsync指令碼(原始碼)
2020-05-10
客戶端指令碼原始碼
搬瓦工 VPS 效能一鍵測試指令碼
2018-12-19
指令碼
redis+lua實現指令碼一鍵查詢
2022-06-02
Redis指令碼
使用shell指令碼實現LANMP一鍵安裝
2016-12-18
指令碼
MySQL5.6一鍵編譯安裝指令碼
2016-01-08
MySql編譯指令碼
easy-hiphop一鍵安裝hiphop指令碼薦
2012-08-17
指令碼
shell指令碼一鍵安裝nginx-1.18.0
2024-04-23
指令碼Nginx
ELK詳解（七）——Kibana部署
2022-07-02
KafKa+Zookeeper+Flume部署指令碼
2017-12-19
Kafka指令碼