修改nginx日志格式
log_format hanye ‘$proxy_add_x_forwarded_for $remote_user [$time_local] "$request" $http_host‘
‘[$body_bytes_sent] $request_body "$http_referer" "$http_user_agent" [$ssl_protocol] [$ssl_cipher]‘
‘[$request_time] [$status] [$upstream_status] [$upstream_response_time] [$upstream_addr]‘;
server {
listen 80;
server_name _;
access_log /data/wwwlogs/access_nginx.log hanye;
}
生成nginx访问日志
[email protected]:~# ab -c 20 -n 20 http://192.168.1.252/
收集nginx日志
[[email protected] wwwlogs]#cat /data/elk-services/logstash/patterns.d/nginx
NGUSERNAME [a-zA-Z\.\@\-\+_%]+
NGUSER %{NGUSERNAME}
NGINXACCESS %{IP:clent_ip} (?:-|%{USER:ident}) \[%{HTTPDATE:log_date}\] \"%{WORD:http_verb} (?:%{PATH:baseurl}\?%{NOTSPACE:params}(?: HTTP/%{NUMBER:http_version})?|%{DATA:raw_http_request})\" (%{IPORHOST:url_domain}|%{URIHOST:ur_domain}|-)\[(%{BASE16FLOAT:request_time}|-)\] %{NOTSPACE:request_body} %{QS:referrer_rul} %{GREEDYDATA:User_Agent} \[%{GREEDYDATA:ssl_protocol}\] \[(?:%{GREEDYDATA:ssl_cipher}|-)\]\[%{NUMBER:time_duration}\] \[%{NUMBER:http_status_code}\] \[(%{BASE10NUM:upstream_status}|-)\] \[(%{NUMBER:upstream_response_time}|-)\] \[(%{URIHOST:upstream_addr}|-)\]
[[email protected] wwwlogs]# cat /data/elk-services/logstash/config/nginx_geoip.yml
input {
file {
path => "/data/wwwlogs/access_nginx.log"
type => "252nginx-access"
start_position => "beginning"
}
}
filter {
if [type] == "252nginx-access" {
grok {
patterns_dir => [ "/data/elk-services/logstash/patterns.d" ]
match => { "message" => "%{NGINXACCESS}" }
overwrite => [ "message" ]
}
geoip {
source => "clent_ip"
target => "geoip"
database => "/data/soft/GeoLite2-City_20190409/GeoLite2-City.mmdb"
}
useragent {
source => "User_Agent"
target => "userAgent"
}
urldecode {
all_fields => true
}
mutate {
gsub => ["User_Agent","[\"]",""] #将user_agent中的 " 换成空
convert => [ "response","integer" ]
convert => [ "body_bytes_sent","integer" ]
convert => [ "bytes_sent","integer" ]
convert => [ "upstream_response_time","float" ]
convert => [ "upstream_status","integer" ]
convert => [ "request_time","float" ]
convert => [ "port","integer" ]
}
date {
match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
}
}
}
output {
if [type] == "252nginx-access" {
elasticsearch {
hosts => ["192.168.1.252:9200"]
index => "logstash-nginx-access-252-%{+YYYY.MM.dd}"
}
}
}
安装geoip插件
[[email protected] elasticsearch]#./bin/elasticsearch-plugin install ingest-geoip
[[email protected] elasticsearch]# ./bin/elasticsearch-plugin install ingest-user-agent
启动检查logstash和生成nginx日志
[[email protected] config]# ../bin/logstash -f nginx_geoip.yml
[[email protected] config]# ab -c 20 -n 20 http://192.168.1.252/
elasticsearch-head 检查日志生成
kibana查看
创建索引
####查看索引数据
原文地址:https://blog.51cto.com/9025736/2377352
时间: 2024-10-08 22:41:42