使用Logstash进行日志分析

LogStash主要用于数据收集和分析方面，配合Elasticsearch，Kibana用起来很方便，安装教程google出来很多。

推荐阅读

• Elasticsearch 权威指南
• 精通 Elasticsearch
• Kibana 中文指南
• The Logstash Book

目的

输入常规的Nginx日志，过滤成所需的字段，存入Elasticsearch里面。

日志样式：

115.182.31.11 - - [02/Aug/2013:08:35:10 +0800] "GET /v2/get?key=0b0c1c5523aa40c3a5dcde4402947693&appid=153&appname=%e6%96%97%e5%9c%b0%e4%b8%bb%e5%8d%95%e6%9c%ba%e7%89%88&uuid=861698005693444&client=1&operator=1&net=2&devicetype=1&adspacetype=1&category=2&ip=117.136.22.36&os_version=2.2.2&aw=320&ah=50&timestamp=1375403699&density=1.5&pw=800&ph=480&Device=ZTE-U%2bV880&sign=1f6fd0992ca09e8525b0f7165a928a2a HTTP/1.1" 200 76 "-" "-" -
117.135.137.180 - - [02/Aug/2013:08:35:10 +0800] "GET /v2/get?Format=json&Key=47378200063c41fe90eff85f11ca4d2f&AppId=324&AppName=%25E5%258D%2595%25E6%259C%25BA%25E6%2596%2597%25E5%259C%25B0%25E4%25B8%25BB&Uuid=b51d63a91da5a4111e6cc1fb2c2538d5&Client=1&Operator=1&Net=2&DeviceType=1&AdSpaceType=1&Category=28&Ip=117.136.7.111&Os_version=4.0.4&Aw=320&Ah=50&TimeStamp=1375403708&Sign=9a00b63a04c165deea70dedd6b747697 HTTP/1.0" 200 776 "-" "-" -
115.182.31.11 - - [02/Aug/2013:08:35:10 +0800] "GET /v2/get?key=0b0c1c5523aa40c3a5dcde4402947693&appid=153&appname=%e6%96%97%e5%9c%b0%e4%b8%bb%e5%8d%95%e6%9c%ba%e7%89%88&uuid=860173017274352&client=1&operator=2&net=3&devicetype=1&adspacetype=1&category=2&ip=120.7.195.5&os_version=2.3.5&aw=320&ah=50&timestamp=1375403700&density=1.5&long=39&lat=0699733%2c116&pw=854&ph=480&Device=MI-ONE%2bPlus&sign=f65a4b2b2681ac489f65acf49e3d8ebd HTTP/1.1" 200 76 "-" "-" -
115.182.31.12 - - [02/Aug/2013:08:35:10 +0800] "GET /v2/get?key=0b0c1c5523aa40c3a5dcde4402947693&appid=153&appname=%e6%96%97%e5%9c%b0%e4%b8%bb%e5%8d%95%e6%9c%ba%e7%89%88&uuid=863802017354171&client=1&operator=1&net=3&devicetype=1&adspacetype=1&category=2&ip=123.121.144.120&os_version=2.3.5&aw=320&ah=50&timestamp=1375403698&density=1.5&long=40&lat=11183975%2c116&pw=854&ph=480&Device=MI-ONE%2bPlus&sign=0c74cf53a4b6adfe5e218f4fab920da3 HTTP/1.1" 200 76 "-" "-" -
115.182.31.8 - - [02/Aug/2013:08:35:10 +0800] "GET /v2/get?key=0b0c1c5523aa40c3a5dcde4402947693&appid=153&appname=%e6%96%97%e5%9c%b0%e4%b8%bb%e5%8d%95%e6%9c%ba%e7%89%88&uuid=868247013598808&client=1&operator=4&net=2&devicetype=1&adspacetype=1&category=2&ip=117.136.20.88&os_version=2.3.5&aw=320&ah=50&timestamp=1375403707&density=1.5&pw=800&ph=480&Device=Lenovo%2bA520GRAY&sign=43d5260eb2b89f5984b513067e074f5e HTTP/1.1" 200 67 "-" "-" -

经过Logstash抽取收集后，每段会输出形式：

{
         "message" => "115.182.31.8 - - [02/Aug/2013:05:24:12 +0800] \"GET /v2/get?key=0b0c1c5523aa40c3a5dcde4402947693&appid=153&appname=斗地主单机版&uuid=355696050506936&client=1&operator=1&net=3&devicetype=1&adspacetype=1&category=2&ip=113.228.122.247&os_version=4.1.1&aw=320&ah=50&timestamp=1375392249&density=2&long=41&lat=917705,123&pw=1280&ph=720&Device=GT-N7100&sign=e9853bb1e8bd56874b647bc08e7ba576 HTTP/1.1\" 200 67 \"-\" \"-\" -",
        "@version" => "1",
      "@timestamp" => "2015-01-15T08:06:26.340Z",
            "host" => "vovo",
            "path" => "/home/vovo/access.log",
          "client" => "1",
           "ident" => "-",
            "auth" => "-",
       "timestamp" => "1375392249",
            "verb" => "GET",
         "request" => "/v2/get?key=0b0c1c5523aa40c3a5dcde4402947693&appid=153&appname=斗地主单机版&uuid=355696050506936&client=1&operator=1&net=3&devicetype=1&adspacetype=1&category=2&ip=113.228.122.247&os_version=4.1.1&aw=320&ah=50&timestamp=1375392249&density=2&long=41&lat=917705,123&pw=1280&ph=720&Device=GT-N7100&sign=e9853bb1e8bd56874b647bc08e7ba576",
    "http_version" => "1.1",
        "response" => "200",
           "bytes" => "67",
             "key" => "0b0c1c5523aa40c3a5dcde4402947693",
           "appid" => "153",
         "appname" => "斗地主单机版",
            "uuid" => "355696050506936",
        "operator" => "1",
             "net" => "3",
      "devicetype" => "1",
     "adspacetype" => "1",
        "category" => "2",
              "ip" => "113.228.122.247",
      "os_version" => "4.1.1",
              "aw" => "320",
              "ah" => "50",
         "density" => "2",
            "long" => "41",
             "lat" => "917705,123",
              "pw" => "1280",
              "ph" => "720",
          "Device" => "GT-N7100",
            "sign" => "e9853bb1e8bd56874b647bc08e7ba576"
}

为了方便理解和测试，我采用了Logstash样版配置文件进行设置。

sample.conf

当中包含实现urldecode和kv插件功能，需要运行 ./plugin install contrib 安装Logstash的默认插件。

input {
    file {
        path => "/home/vovo/access.log"  #指定日志目录或文件，也可以使用通配符*.log输入目录中的log文件。
        start_position => "beginning"
    }
}
filter {
    grok {
        match => ["message", "%{IPORHOST:client} (%{USER:ident}|-) (%{USER:auth}|-) \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version})?|-)\" %{NUMBER:response} %{NUMBER:bytes} \"(%{QS:referrer}|-)\" \"(%{QS:agent}|-)\""]　　　　#匹配模式 message是每段读进来的日志，IP、HTTPDATE、WORD、NOTSPACE、NUMBER都是patterns/grok-patterns中定义好的正则格式名称，对照上面的日志进行编写,冒号，(?:%{USER:ident}|-)这种形式是条件判断，相当于程序里面的二目运算。如果有双引号""或者[]号，需要在前面加\进行转义。
    }
    kv {
                source => "request"
                field_split => "&?"
                value_split => "="
        }　　#再单独将取得的URL、request字段取出来进行key-value值匹配，需要kv插件。提供字段分隔符"&?"，值键分隔符"="，则会自动将字段和值采集出来。
    urldecode {
        all_fields => true
    }　　#把所有字段进行urldecode（显示中文）
}

output {
    #elasticsearch {
    #    host => ‘localhost‘
    #    protocol => "http"
    #}　　#把采集的数据输出到elasticsearch里面。
    stdout {
        codec => rubydebug
    }　　#输出到屏幕上
}