使用Fluentd + Elasticsearch收集訪問日誌

<dependency>
    <groupId>ch.qos.logback</groupId>
    <artifactId>logback-access</artifactId>
    <version>1.2.3</version>
</dependency>`

<configuration>
    <appender name="FLUENCY" class="ch.qos.logback.more.appenders.FluencyLogbackAppender">
        <!-- Tag for Fluentd. Farther information: http://docs.fluentd.org/articles/config-file -->
        <tag>accesslog</tag>
        <!-- Host name/address and port number which Flentd placed -->
        <remoteHost>localhost</remoteHost>
        <port>20001</port>

        <!-- [Optional] Configurations to customize Fluency's behavior: https://github.com/komamitsu/fluencyusage  -->
        <ackResponseMode>false</ackResponseMode>
        <fileBackupDir>/tmp</fileBackupDir>
        <!-- Initial chunk buffer size is 1MB (by default)-->
        <bufferChunkInitialSize>2097152</bufferChunkInitialSize>
        <!--Threshold chunk buffer size to flush is 4MB (by default)-->
        <bufferChunkRetentionSize>16777216</bufferChunkRetentionSize>
        <!-- Max total buffer size is 512MB (by default)-->
        <maxBufferSize>268435456</maxBufferSize>
        <!-- Max wait until all buffers are flushed is 10 seconds (by default)-->
        <waitUntilBufferFlushed>30</waitUntilBufferFlushed>
        <!-- Max wait until the flusher is terminated is 10 seconds (by default) -->
        <waitUntilFlusherTerminated>40</waitUntilFlusherTerminated>
        <!-- Flush interval is 600ms (by default)-->
        <flushIntervalMillis>200</flushIntervalMillis>
        <!-- Max retry of sending events is 8 (by default) -->
        <senderMaxRetryCount>12</senderMaxRetryCount>
        <!-- [Optional] Enable/Disable use of EventTime to get sub second resolution of log event date-time -->
        <useEventTime>true</useEventTime>

        <encoder>
            <pattern><![CDATA[REQUEST FROM %remoteIP ON %date{yyyy-MM-dd HH:mm:ss,UTC} UTC // %responseHeader{X-UOW} // responseHeader{X-RequestId}    %n
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
%fullRequest
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
%fullResponse
                ]]>
            </pattern>
        </encoder>
    </appender>

    <appender-ref ref="FLUENCY"/>
</configuration>

<dependency>
    <groupId>org.komamitsu</groupId>
    <artifactId>fluency</artifactId>
    <version>1.8.1</version>
</dependency>
<dependency>
    <groupId>com.sndyuk</groupId>
    <artifactId>logback-more-appenders</artifactId>
    <version>1.5.0</version>
  </dependency>

<source>
    @type forward
    port 20001
</source>

<filter accesslog>
    @type parser
    key_name msg
    reserve_data false
    <parse>
        @type multiline
        format_firstline /^REQUEST FROM/
        format1 /REQUEST FROM (?<request.ip>[^ ]*) ON (?<time>\d{4}-\d{2}-\d{2} \d{2}\:\d{2}\:\d{2} [^ ]+) // (?<request.uow>[^ ]*) // (?<request.id>[^ ]*)\n/
        format2 />{49}\n/
        format3 /(?<request.method>[^ ]*) (?<request.path>[^ ]*) (?<request.protocol>[^ ]*)\n/
        format4 /(?<request.headers>(?:.|\n)*?)\n\n/
        format5 /(?<request.body>(?:.|\n)*?)\n/
        format6 /<{49}\n/
        format7 /(?<response.protocol>[^ ]*) (?<response.status.code>[^ ]*) (?<response.status.description>[^\n]*)\n/
        format8 /(?<response.headers>(?:.|\n)*?)\n\n/
        format9 /(?<response.body>(?:.|\n)*?)\n\Z/
    </parse>
</filter>

# Parse request.headers="Header: Value\n Header: Value\n" to become and Object request.headers={"Header": "Value", "Header": "Value"}
<filter accesslog>
  @type record_transformer
  enable_ruby true
  renew_record false
  auto_typecast true
  <record>
    hostname "#{Socket.gethostname}"
    request.headers ${Hash[record["request.headers"].each_line.map { |l| l.chomp.split(': ', 2) }]}
    response.headers ${Hash[record["response.headers"].each_line.map { |l| l.chomp.split(': ', 2) }]}
  </record>
</filter>

<match accesslog>
    @type forward
    send_timeout 5s
    recover_wait 10s
    hard_timeout 30s
    flush_interval 5s
    <server>
        name elastic-node-00
        host elastic-node-00
        port 24224
        weight 100
    </server>
</match>

<match **>
    @type file
      path /tmp/fluentd/output/messages
</match>

First filter
{ 
 ...
 "time": "2018-10-30 00:00:00 UTC"
 "request.ip": "192.168.0.1",
 "request.uow": "myapp-node-00-1540857599992",
 "request.id": "h5hSUaVHvr",
 "request.method": "POST",
 "request.path": "/my/app/path",
 "request.protocol": "HTTP/1.1",
 "request.headers": "X-Forwarded-Proto: https\nX-Forwarded-For: 69.28.94.231\nHost: my.company.com\nContent-Length: 30\nContent-Type: application/json",
 "request.body": "{\"message\": \"This is the body of the request\" }",
 "response.protocol": "HTTP/1.1",
 "response.status.code": "200",
 "response.status.description": "OK"
 "response.headers": "X-RequestId: h5hSUaVHvr\nX-UOW: myapp-node-00-1540857599992\nDate: Mon, 29 Oct 2018 23:59:59 GMT\nContent-Type: application/json; charset=UTF-8"
 "response.body": "{\"message\": \"This is the body of the response\", \"status\": \"Okey!\"}"
 ...
}
Second filter
{ 
 ...
 "time": "2018-10-30 00:00:00 UTC"
 "request.ip": "192.168.0.1",
 "request.uow": "myapp-node-00-1540857599992",
 "request.id": "h5hSUaVHvr",
 "request.method": "POST",
 "request.path": "/my/app/path",
 "request.protocol": "HTTP/1.1",
 "request.headers": { "X-Forwarded-Proto": "https", 
       "X-Forwarded-For": "69.28.94.231", 
       "Host":" my.company.com", 
       "Content-Length": "30", 
       "Content-Type": "application/json" 
      },
 "request.body": "{\"message\": \"This is the body of the request\" }",
 "response.protocol": "HTTP/1.1",
 "response.status.code": "200",
 "response.status.description": "OK"
 "response.headers": {
       "X-RequestId: "h5hSUaVHvr",
       "X-UOW": "myapp-node-00-1540857599992",
       "Date": "Mon, 29 Oct 2018 23:59:59 GMT",
       "Content-Type": "application/json; charset=UTF-8"
      }
 "response.body": "{\"message\": \"This is the body of the response\", \"status\": \"Okey!\"}"
 ...
}

<source>
  @type forward
  port 24224
  bind 0.0.0.0
</source>

<match accesslog>
  @type elasticsearch
  scheme http
  host localhost
  port 9200
  logstash_format true
  validate_client_version true
</match>