logstash产生时区替换@timestamp

在logstash中nginx配置一般分为两种格式：

1、nginx配置$time_local

log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"           "$proxy_add_x_forwarded_for" "$request_time"';

获取到的日志通常为

192.168.29.7 - - [26/Jun/2018:15:21:42 +0800] "GET /images/logo3.png HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:60.0) Gecko/20100101 Firefox/60.0" "-" "192.168.29.7" "0.000"

时间格式：26/Jun/2018:15:21:42 +0800

此时logstash配置如下

 1 input {
 2         file {
 3                 path => "/home/nginx/logs/main_logstashtest_access.log"
 4                 #codec => "json"
 5         }
 6 }
 7 
 8 filter {
 9         grok {
10                 match => ["message","\[%{HTTPDATE:request_time}\]"]
11         }
12         date {
13                 #locale => "en"
14                 match => ["request_time", "dd/MMM/yyyy:HH:mm:ss Z"]
15                 target => "@timestamp"
16         }
17 #       mutate {
18 #               remove_field => ["@timestamp"]
19 #       }
20 }
21 
22 output {
23         stdout {
24                 codec => "rubydebug"
25         }
26 }

此时访问nginx日志为：

{"@timestamp":"26/Jun/2018:15:39:56 +0800","@version":"1","host":"192.168.29.7","size":0,"reponsetime":0.000,"domain":"www.logstashtest.com","url":"/images/logo3.png","status":"304"}

logstash输出：

{
        "@version" => "1",
            "host" => "Sandos1",
      "@timestamp" => 2018-06-26T07:39:56.000Z,
         "message" => "192.168.29.7 - - [26/Jun/2018:15:39:56 +0800] \"GET /images/logo3.png HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:60.0) Gecko/20100101 Firefox/60.0\" \"-\" \"192.168.29.7\" \"0.000\"",
    "request_time" => "26/Jun/2018:15:39:56 +0800",
            "path" => "/home/nginx/logs/main_logstashtest_access.log"
}

2、nginx配置$time_iso8601

log_format json '{"@timestamp":"$time_iso8601",'
                        '"@version":"1",'
                        '"host":"$clientRealIp",'
                        '"size":$body_bytes_sent,'
                        '"reponsetime":$request_time,'
                        '"domain":"$host",'
                        '"url":"$uri",'
                        '"status":"$status"}';

获取到的日志通常为

{"@timestamp":"2018-06-26T15:39:56+08:00","@version":"1","host":"192.168.29.7","size":0,"reponsetime":0.000,"domain":"www.logstashtest.com","url":"/images/logo3.png","status":"304"}

时间格式：2018-06-26T15:39:56+08:00

此时logstash配置如下

 1 input { 
 2         file {  
 3                 path => "/home/nginx/logs/logstash_iso_test_access.log"
 4         }
 5 }
 6 
 7 filter {
 8         json {  
 9                 source => "message"
10         }
11         grok {  
12                 match => ["message","%{TIMESTAMP_ISO8601:isotime}"]
13         }
14 
15         date {  
16                 locale => "en"
17                 match => ["isotime", "ISO8601"]
18         }
19 #       mutate {
20 #               remove_field => ["@timestamp"]
21 #       }
22 }
23 
24 output {
25         stdout {
26                 codec => "rubydebug"
27         }
28 }

此时访问nginx日志为：

{"@timestamp":"2018-06-26T15:45:43+08:00","@version":"1","host":"192.168.29.7","size":0,"reponsetime":0.000,"domain":"www.logstashtest.com","url":"/images/logo3.png","status":"304"}

logstash输出：

{
        "@version" => "1",
            "host" => "Sandos1",
      "@timestamp" => 2018-06-26T07:45:43.000Z,
         "message" => "192.168.29.7 - - [26/Jun/2018:15:45:43 +0800] \"GET /images/logo3.png HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:60.0) Gecko/20100101 Firefox/60.0\" \"-\" \"192.168.29.7\" \"0.000\"",
    "request_time" => "26/Jun/2018:15:45:43 +0800",
            "path" => "/home/nginx/logs/main_logstashtest_access.log"
}

logstash产生时区替换@timestamp

推荐阅读