在工作中需要在elk中展示tomcat的日志，对其进行分析，报错排查及其它定制需求；
下面为catalina.out日志样例，以此为样本进行字段的解析；注：可以看到此段日志有规律的，由此可以对其进行解析；
2018-08-08 00:00:16,728 INFO [com.staples.adv.action.BaseShopAction] - <*********entry method getUserGroupList***********> 2018-08-08 00:00:16,728 INFO [com.staples.adv.action.BaseShopAction] - <*********exit method getUserGroupList***********> 2018-08-08 00:11:10,463 WARN [org.springframework.web.servlet.PageNotFound] - <Request method 'POST' not supported> 2018-08-08 00:11:10,647 WARN [org.springframework.web.servlet.PageNotFound] - <No mapping found for HTTP request with URI [/news/2016/gou.php] in DispatcherServlet with name 'springmvc'> 2018-08-08 00:18:13,061 ERROR [org.vancebox.exception.ExceptionResolver] - <ClientAbortException: java.net.SocketException: Broken pipe at org.apache.catalina.connector.OutputBuffer.realWriteBytes(OutputBuffer.java:388) at org.apache.tomcat.util.buf.ByteChunk.flushBuffer(ByteChunk.java:462) at org.apache.tomcat.util.buf.ByteChunk.append(ByteChunk.java:366) at org.apache.catalina.connector.OutputBuffer.writeBytes(OutputBuffer.java:413) at org.apache.catalina.connector.OutputBuffer.write(OutputBuffer.java:401) at org.apache.catalina.connector.CoyoteOutputStream.write(CoyoteOutputStream.java:91) at org.springframework.util.StreamUtils.copy(StreamUtils.java:125) at org.springframework.web.servlet.resource.ResourceHttpRequestHandler.writeContent(ResourceHttpRequestHandler.java:252) at org.springframework.web.servlet.resource.ResourceHttpRequestHandler.handleRequest(ResourceHttpRequestHandler.java:153) at org.springframework.web.servlet.mvc.HttpRequestHandlerAdapter.handle(HttpRequestHandlerAdapter.java:49) at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:925) at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:856) at org.springframework.web.servlet.FrameworkServlet.proce***equest(FrameworkServlet.java:953) at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:844) at javax.servlet.http.HttpServlet.service(HttpServlet.java:621) at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:829) at javax.servlet.http.HttpServlet.service(HttpServlet.java:722) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) at org.vancebox.filter.LocaleFilter.doFilterInternal(LocaleFilter.java:54) at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) at org.springframework.web.filter.HiddenHttpMethodFilter.doFilterInternal(HiddenHttpMethodFilter.java:77) at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:88) at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:225) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:169) at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:168) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:98) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:407) at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:999) at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:565) at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:307) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) Caused by: java.net.SocketException: Broken pipe at java.net.SocketOutputStream.socketWrite0(Native Method) at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:92) at java.net.SocketOutputStream.write(SocketOutputStream.java:136) at com.sun.net.ssl.internal.ssl.OutputRecord.writeBuffer(OutputRecord.java:297) at com.sun.net.ssl.internal.ssl.OutputRecord.write(OutputRecord.java:286) at com.sun.net.ssl.internal.ssl.SSLSocketImpl.writeRecordInternal(SSLSocketImpl.java:748) at com.sun.net.ssl.internal.ssl.SSLSocketImpl.writeRecord(SSLSocketImpl.java:736) at com.sun.net.ssl.internal.ssl.AppOutputStream.write(AppOutputStream.java:89) at org.apache.coyote.http11.InternalOutputBuffer.realWriteBytes(InternalOutputBuffer.java:215) at org.apache.tomcat.util.buf.ByteChunk.flushBuffer(ByteChunk.java:462) at org.apache.tomcat.util.buf.ByteChunk.append(ByteChunk.java:366) at org.apache.coyote.http11.InternalOutputBuffer$OutputStreamOutputBuffer.doWrite(InternalOutputBuffer.java:240) at org.apache.coyote.http11.filters.IdentityOutputFilter.doWrite(IdentityOutputFilter.java:84) at org.apache.coyote.http11.AbstractOutputBuffer.doWrite(AbstractOutputBuffer.java:192) at org.apache.coyote.Response.doWrite(Response.java:504) at org.apache.catalina.connector.OutputBuffer.realWriteBytes(OutputBuffer.java:383) ... 43 more > Aug 8, 2018 12:18:13 AM org.apache.catalina.core.StandardWrapperValve invoke SEVERE: Servlet.service() for servlet [springmvc] in context with path [] threw exception [Request processing failed; nested exception is java.lang.IllegalStateException: getOutputStream() has already been called for this response] with root cause java.lang.IllegalStateException: getOutputStream() has already been called for this response at org.apache.catalina.connector.Response.getWriter(Response.java:637) at org.apache.catalina.connector.ResponseFacade.getWriter(ResponseFacade.java:214) at org.springframework.web.servlet.view.freemarker.FreeMarkerView.processTemplate(FreeMarkerView.java:366) at org.springframework.web.servlet.view.freemarker.FreeMarkerView.doRender(FreeMarkerView.java:283) at org.springframework.web.servlet.view.freemarker.FreeMarkerView.renderMergedTemplateModel(FreeMarkerView.java:233) at org.springframework.web.servlet.view.AbstractTemplateView.renderMergedOutputModel(AbstractTemplateView.java:167) at org.springframework.web.servlet.view.AbstractView.render(AbstractView.java:264) at org.springframework.web.servlet.DispatcherServlet.render(DispatcherServlet.java:1208) at org.springframework.web.servlet.DispatcherServlet.processDispatchResult(DispatcherServlet.java:992) at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:939) at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:856) at org.springframework.web.servlet.FrameworkServlet.proce***equest(FrameworkServlet.java:953) at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:844) at javax.servlet.http.HttpServlet.service(HttpServlet.java:621) at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:829) at javax.servlet.http.HttpServlet.service(HttpServlet.java:722) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) at org.vancebox.filter.LocaleFilter.doFilterInternal(LocaleFilter.java:54) at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) at org.springframework.web.filter.HiddenHttpMethodFilter.doFilterInternal(HiddenHttpMethodFilter.java:77) at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:88) at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:225) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:169) at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:168) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:98) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:407) at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:999) at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:565) at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:307) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) 2018-08-08 00:19:30,178 INFO [com.staples.adv.action.BaseShopAction] - <*********entry method getUserGroupList***********> 2018-08-08 00:19:30,178 INFO [com.staples.adv.action.BaseShopAction] - <*********exit method getUserGroupList***********>
通过在grokdebug上面进行调试，日志字段可以解析为：
%{TIMESTAMP_ISO8601:access_time} %{LOGLEVEL:loglevel} \[%{DATA:exception_info}\] - \<%{MESSAGE:message}\>
注1：将catalina.out的日志解析为：日志时间、日志级别、异常提示、异常内容
注2：MESSAGE是自己定义的匹配规则，用于匹配任何数据（包括单行和多行）：MESSAGE [\s\S]*
filebeat先读取catalina.out的日志，写入redis中：
filebeat.inputs: - type: log paths: - /data/sadv/10.78.3.129/test5.out tags: ["sadv"] fields: type: sadv fields_under_root: true multiline: pattern: '^\d+-\d+-\d+ \d+:\d+:\d+' negate: true match: after output.redis: hosts: ["10.78.1.181"] key: "sadv" datatype: list
logstash向redis读取数据，解析过滤之后写入elastic中：
input {
redis {
host => "10.78.1.181"
port => 6379
db => "0"
data_type => "list"
key => "sadv"
type => "sadv"
}
}
filter {
mutate {
remove_field => ["@version","prospector","input","beat","source","offset"]
}
grok {
match => {
"message" => "%{TIMESTAMP_ISO8601:access_time} %{LOGLEVEL:loglevel} \[%{DATA:exception_info}\] - \<%{MESSAGE:message}\>"
}
pattern_definitions => {
"MESSAGE" => "[\s\S]*"
}
}
date {
match => [ "access_time","yyyy-MM-dd HH:mm:ss,SSS" ]
}
mutate {
remove_field => ["access_time","[message][0]"]
}
}
output {
if [type] == "sadv" {
if [tags][0] == "sadv" {
elasticsearch {
hosts => ["10.78.1.184:9200","10.78.1.185:9200","10.78.1.188:9200"]
index => "%{type}-%{+YYYY.MM.dd}"
}
stdout { codec => rubydebug }
}
}
}
注：output中的stdout为调试所用，生产建议删除
原文地址：http://blog.51cto.com/liuzhengwei521/2156303
时间： 2024-10-09 14:36:23