logstash nginx error access 日誌處理

衣舞晨風發表於2017-04-08
NginxError
input {
   file {
    type => "nginx-access"  
    path => [ "/home/jiankunking/software/testdata/nginx/access.log" ]
    tags => [ "nginx","access"]
    start_position => beginning
   }
    file {
    type => "nginx-error" 
    path => [ "/home/jiankunking/software/testdata/nginx/error.log" ]
    tags => [ "nginx","error"]
    start_position => beginning
}
}
filter {
 if [type] == "nginx-access" {
        grok{
            match =>    ["message","%{IPORHOST:client_ip}\s{1,}\-\s\-\s\[%{HTTPDATE:time}\]\s{1,}\"(?:%{WORD:verb}\s{1,}%{NOTSPACE:request}(?:\s{1,}HTTP/%{NUMBER:http_version})?|-)\" %{NUMBER:response}\s{1,}(?:%{NUMBER:bytes}|-)\s{1,}%{QS:referrer}\s{1,}%{QS:agent}"]
        }
        date{
            match=>["time","dd/MMM/yyyy:HH:mm:ss Z"]
            target=>"logdate"
        }
        ruby{
            code => "event.set('logdateunix',event.get('logdate').to_i)"
        }
    } else if [type] == "nginx-error" { 
        grok {
        match => [
            "message", "(?<time>\d{4}/\d{2}/\d{2}\s{1,}\d{2}:\d{2}:\d{2})\s{1,}\[%{DATA:err_severity}\]\s{1,}(%{NUMBER:pid:int}#%{NUMBER}:\s{1,}\*%{NUMBER}|\*%{NUMBER}) %{DATA:err_message}(?:,\s{1,}client:\s{1,}(?<client_ip>%{IP}|%{HOSTNAME}))(?:,\s{1,}server:\s{1,}%{IPORHOST:server})(?:, request: %{QS:request})?(?:, host: %{QS:client_ip})?(?:, referrer: \"%{URI:referrer})?",
            "message", "(?<time>\d{4}/\d{2}/\d{2}\s{1,}\d{2}:\d{2}:\d{2})\s{1,}\[%{DATA:err_severity}\]\s{1,}%{GREEDYDATA:err_message}"]
        }
        date{
            match=>["time","yyyy/MM/dd HH:mm:ss"]
            target=>"logdate"
        }
        ruby{
            code => "event.set('logdateunix',event.get('logdate').to_i)"
        }
    }
}
output{
   elasticsearch{
        hosts => ["10.10.10.10:9200"]
        index => "logstash-nginx-%{+YYYY.MM.dd}"
    }
}

nginx access日誌格式：

10.10.10.10 - - [28/Mar/2017:13:21:04 +0800] "GET / HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36"
10.10.10.10 - - [28/Mar/2017:13:21:04 +0800] "GET /favicon.ico HTTP/1.1" 404 572 "http://10.11.11.11/" "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36"

拆分效果：

{
    "_index": "logstash-nginx-access-2017.04.07",
    "_type": "logs",
    "_id": "AVtHLpaYYixw4Fpnef2F",
    "_score": 1,
    "_source": {
        "request": "/favicon.ico",
        "agent": ""Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"",
        "verb": "GET",
        "http_version": "1.1",
        "message": "10.11.11.12 - - [28/Mar/2017:13:33:42 +0800] "GET /favicon.ico HTTP/1.1" 404 572 "http://log.c.haier.net/" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"",
        "tags": [
            "nginx"
            ,
            "access"
        ],
        "path": "/home/jiankunking/software/testdata/nginx/access.log",
        "referrer": ""http://log.c.haier.net/"",
        "@timestamp": "2017-04-07T06:51:26.135Z",
        "response": "404",
        "bytes": "572",
        "logdate": "2017-03-28T05:33:42.000Z",
        "@version": "1",
        "host": "ubuntu",
        "client_ip": "10.11.11.12",
        "logdateunix": 1490679222,
        "timestamp": "28/Mar/2017:13:33:42 +0800"
    }

}

nginx error日誌格式：

2017/03/28 13:34:21 [error] 17627#0: *8 open() "/usr/local/nginx/html/favicon.ico" failed (2: No such file or directory), client: 10.10.10.10, server: localhost, request: "GET /favicon.ico HTTP/1.1", host: "10.11.11.11"
2017/03/28 14:15:10 [notice] 20260#0: signal process started
2017/03/28 15:04:32 [emerg] 21321#0: invalid URL prefix in /usr/local/nginx/conf/nginx.conf:47

拆分效果：

{
    "_index": "logstash-nginx-error-2017.04.07",
    "_type": "logs",
    "_id": "AVtHEFkyYixw4FpnefNL",
    "_score": 1,
    "_source": {
        "path": "/home/jiankunking/software/testdata/nginx/error.log",
        "@timestamp": "2017-04-07T06:18:24.601Z",
        "err_severity": "notice",
        "logdate": "2017-03-28T06:15:10.000Z",
        "@version": "1",
        "host": "ubuntu",
        "time": "2017/03/28 14:15:10",
        "message": "2017/03/28 14:15:10 [notice] 20260#0: signal process started",
        "logdateunix": 1490681710,
        "tags": ["nginx",
        "error"],
        "err_message": "20260#0: signal process started"
    }
},
{
    "_index": "logstash-nginx-error-2017.04.07",
    "_type": "logs",
    "_id": "AVtHEFkyYixw4FpnefNK",
    "_score": 1,
    "_source": {
        "server": "localhost",
        "request": ""GET/favicon.icoHTTP/1.1"",
        "pid": 17627,
        "message": "2017/03/28 13:34:21 [error] 17627#0: *8 open() "/usr/local/nginx/html/favicon.ico" failed (2: No such file or directory), client: 10.11.11.11, server: localhost, request: "GET/favicon.icoHTTP/1.1", host: "10.11.11.12"",
        "tags": ["nginx",
        "error"],
        "err_message": "open() "/usr/local/nginx/html/favicon.ico" failed (2: No such file or directory)",
        "path": "/home/jiankunking/software/testdata/nginx/error.log",
        "@timestamp": "2017-04-07T06:18:24.595Z",
        "err_severity": "error",
        "logdate": "2017-03-28T05:34:21.000Z",
        "@version": "1",
        "host": "ubuntu",
        "client_ip": ["10.11.11.11",
        ""10.11.11.12""],
        "time": "2017/03/28 13:34:21",
        "logdateunix": 1490679261
    }
},
{
    "_index": "logstash-nginx-error-2017.04.07",
    "_type": "logs",
    "_id": "AVtHEFkyYixw4FpnefNM",
    "_score": 1,
    "_source": {
        "path": "/home/jiankunking/software/testdata/nginx/error.log",
        "@timestamp": "2017-04-07T06:18:24.610Z",
        "err_severity": "emerg",
        "logdate": "2017-03-28T07:04:32.000Z",
        "@version": "1",
        "host": "ubuntu",
        "time": "2017/03/28 15:04:32",
        "message": "2017/03/28 15:04:32 [emerg] 21321#0: invalid URL prefix in /usr/local/nginx/conf/nginx.conf:47",
        "logdateunix": 1490684672,
        "tags": ["nginx",
        "error"],
        "err_message": "21321#0: invalid URL prefix in /usr/local/nginx/conf/nginx.conf:47"
    }
}

本文參考：https://github.com/rooprob/logstash-various/blob/master/logstash-nginx-error.conf

作者：jiankunking 出處：http://blog.csdn.net/jiankunking

相關文章