[日誌分析篇]-利用ELK分析jumpserver日誌-日誌拆分篇

領導下達了一個目標，需要監控堡壘機的使用情況和資料實時展示，考慮後採取ELK架構進行日誌處理和訊息通知
透過elk日誌分析平臺接收jumpserver日誌，對日誌進行過濾和拆分，並透過Grafana進行企業微信告警推送和大屏展示

1.系統介紹

名稱	軟體版本
jumpserver	jumpserver-3.10.13-tls
elasticsearch	elasticsearch-8.12.2
kibana	kibana-8.12.2
logstash	logstash-8.12.2
granfa	Grafana v11.0.0

2.jumpserver配置syslog

此處參考飛致雲syslog配置文件：https://kb.fit2cloud.com/?p=123#heading-11

• 在/opt/jumpserver/config/config.txt新增syslog配置

# 加入 syslog 相關設定
SYSLOG_ENABLE=True
SYSLOG_ADDR=10.22.3.12:5149

• 重啟jumpserver

# 重啟jumpserver
/opt/jumpserver-offline-release-v3.10.13-amd64/jmsctl.sh restart

3.jumpserver的日誌型別

jumpserver日誌一共有：登入日誌 、上傳檔案日誌、下載檔案日誌、操作日誌、改密日誌、會話日誌、命令日誌這幾種型別

4.logstash拆分jumpserver日誌

拆分log日誌是一個苦活，需要考慮引數的相容性和匹配性

# 定義接收jumpserver syslog的埠
input {
  udp {
    port => 5149
  }
}

# 定義拆分jumpserver syslog的規則
filter {
# 判斷日誌型別是否為session_command_log，特定grok規則來解決linux機器記錄w和top等命令日誌無法解析情況
# 對於非session_command_log日誌，將採用通用grok規則
  if [message] =~ /session_command_log/ {
    grok {
      match => { "message" => "<14>jumpserver: session_command_log - %{GREEDYDATA:reallogs}\u0000" }
      add_field => { "logtype" => "session_command_log" }
    }
   } else {
      grok {
      match => { "message" => "<%{NUMBER:priority}>%{GREEDYDATA:logsouce}: %{GREEDYDATA:logtype} - %{GREEDYDATA:reallogs}\u0000" }
    }
  }

# 利用json拆分實際記錄的日誌
  json {
    source => "reallogs"
    target => "manage"
  }

# 判斷日誌型別是否為login_log，用來獲取使用者賬戶名和使用者賬戶顯示名稱
  if [logtype] != "login_log" {
    mutate {
      gsub => ["[manage][user]", "\)$", ""]
      split => { "[manage][user]" => "(" }
    }
    mutate {
      add_field => {
        "[manage][user_name]" => "%{[manage][user][0]}"
        "[manage][user_account]" => "%{[manage][user][1]}"
      }
    }
  }

# 移除不需要的日誌
  mutate {
    remove_field => ["reallogs", "@version", "event", "logsouce", "priority", "[manage][user]", "message"]
  }

# 解析login_log日誌，用日誌內的真實時間來替換@timestamp，保證日誌時間的真確性
  if [logtype] == "login_log" {
    date {
      match => ["[manage][datetime]", "yyyy/MM/dd HH:mm:ss Z"]
      target => "@timestamp"
    }

  }

# 解析ftp_log，拆分資產名字和資產IP，拆分資產連線賬號和資產賬戶顯示名稱，用日誌內的真實時間來替換@timestamp，保證日誌時間的真確性
  if [logtype] == "ftp_log" {
    mutate {
      gsub => [
        "[manage][asset]", "\)$", "",
        "[manage][account]", "\)$", ""
      ]
      split => {
        "[manage][asset]" => "("
        "[manage][account]" => "("
      }
    }

    mutate {
      add_field => {
        "[manage][asset_name]" => "%{[manage][asset][0]}"
        "[manage][asset_ip]" => "%{[manage][asset][1]}"
        "[manage][asset_account_name]" => "%{[manage][account][0]}"
        "[manage][asset_account_user]" => "%{[manage][account][1]}"
      }
      remove_field => [ "[manage][asset]","[manage][account]" ]
    }

    date {
      match => ["[manage][date_start]", "yyyy/MM/dd HH:mm:ss Z"]
      target => "@timestamp"
    }

  }

# 解析operation_log日誌，用日誌內的真實時間來替換@timestamp，保證日誌時間的真確性
  if [logtype] == "operation_log" {
    date {
      match => ["[manage][datetime]", "yyyy/MM/dd HH:mm:ss Z"]
      target => "@timestamp"
    }

  }

# 解析password_change_log，操作人員名稱和顯示名，用日誌內的真實時間來替換@timestamp，保證日誌時間的真確性
  if [logtype] == "password_change_log" {
    mutate {
      gsub => [ "[manage][change_by]", "\)$", "" ]
      split => { "[manage][change_by]" => "(" }
    }

    mutate {
      add_field => {
        "[manage][changeby_user]" => "%{[manage][change_by][0]}"
        "[manage][changeby_account]" => "%{[manage][change_by][1]}"
      }
      remove_field => [ "[manage][change_by]" ]
    }

    date {
      match => ["[manage][datetime]", "yyyy/MM/dd HH:mm:ss Z"]
      target => "@timestamp"
    }

  }


# 解析host_session_log，拆分資產名字和資產IP，拆分資產連線賬號和資產賬戶顯示名稱，用日誌內的真實時間來替換@timestamp，保證日誌時間的真確性
  if [logtype] == "host_session_log" {
    mutate {
      gsub => [
        "[manage][asset]", "\)$", "",
        "[manage][account]", "\)$", ""
      ]
      split => {
        "[manage][asset]" => "("
        "[manage][account]" => "("
      }
    }

    mutate {
      add_field => {
        "[manage][asset_name]" => "%{[manage][asset][0]}"
        "[manage][asset_ip]" => "%{[manage][asset][1]}"
        "[manage][asset_account_name]" => "%{[manage][account][0]}"
        "[manage][asset_account_user]" => "%{[manage][account][1]}"
      }
      remove_field => [ "[manage][asset]","[manage][account]" ]
    }

    if [manage][date_end] {
      mutate {
        add_field => { "connect-time" => "%{[manage][duration]}" }
      }

      date {
        match => ["[manage][date_end]", "yyyy/MM/dd HH:mm:ss Z"]
        target => "@timestamp"
      }
    } else if [manage][date_start] {
      date {
        match => ["[manage][date_start]", "yyyy/MM/dd HH:mm:ss Z"]
        target => "@timestamp"
      }
    }

  }

# 解析session_command_log，拆分資產名字和資產IP，拆分資產連線賬號和資產賬戶顯示名稱，用日誌內的真實時間來替換@timestamp，保證日誌時間的真確性

  if [logtype] == "session_command_log" {
    mutate {
      gsub => [
        "[manage][asset]", "\)$", "",
        "[manage][account]", "\)$", ""
      ]
      split => {
        "[manage][asset]" => "("
        "[manage][account]" => "("
      }
    }

    mutate {
      add_field => {
        "[manage][asset_name]" => "%{[manage][asset][0]}"
        "[manage][asset_ip]" => "%{[manage][asset][1]}"
        "[manage][asset_account_name]" => "%{[manage][account][0]}"
        "[manage][asset_account_user]" => "%{[manage][account][1]}"
      }
      remove_field => [ "[manage][asset]","[manage][account]" ]
    }

    date {
      match => ["[manage][timestamp_display]", "yyyy/MM/dd HH:mm:ss Z"]
      target => "@timestamp"
    }

  }

# 日誌拆分完畢後，刪除不需要的日誌
  mutate {
    remove_field => ["[manage][id]","[manage][asset_id]","[manage][account_id]","[manage][org_id]","[manage][terminal_display]","[manage][terminal][id]","[manage][terminal][name]","[manage][user_id]","[manage][timestamp]", "[manage][session]"]
  }

}

# 建立索引
output {
  elasticsearch {
    hosts => ["http://localhost:9200"]
    index => "sh-blj-%{+YYYY.MM.dd}"
    action => "create"
    user => "elastic"
    password => "password"
  }
  stdout {
    codec => "rubydebug"
  }
}

5.解析後日志的顯示