logstash是java应用，依赖JDK，首先需要安装JDK，在安装jdk过程中，logstash-2.3.4使用JDK-1.7版本有bug，使用JDK-1.8版本正常，因此我们安装JDK-1.8版本。
安装JDK
官网地址：http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html
# rpm -ivh jdk-8u101-linux-x64.rpm # echo "export JAVA_HOME=/usr/java/latest" >> /etc/profile # echo "export PATH=$PATH:$JAVA_HOME/bin" >> /etc/profile # source /etc/profile # java -version java version "1.8.0_101" Java(TM) SE Runtime Environment (build 1.8.0_101-b13) Java HotSpot(TM) 64-Bit Server VM (build 25.101-b13, mixed mode)
安装logstash
官网地址：https://www.elastic.co/products/logstash
# tar xf logstash-2.3.4.tar.gz -C /usr/local/app/ # ln -sv /usr/local/app/logstash-2.3.4 /usr/local/logstash # cd /usr/local/logstash# mkdir patterns
NGUSERNAME [a-zA-Z\.\@\-\+_%]+
NGUSER %{NGUSERNAME}
NGINXACCESS %{IPORHOST:server_name} %{IPORHOST:server_ip} %{IPORHOST:client_ip} %{NUMBER:client_port} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response} (?:%{NUMBER:body_bytes_sent}|-) (?:"(?:%{URI:referrer}|-)"|%{QS:referrer}) %{QS:agent} %{NUMBER:request_time} (?:%{NUMBER:upstream_response_time}|-)
patterns/nginx
SYSLOGBASE2 (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
SYSLOGPAMSESSION %{SYSLOGBASE} (?=%{GREEDYDATA:message})%{WORD:pam_module}\(%{DATA:pam_caller}\): session %{WORD:pam_session_state} for user %{USERNAME:username}(?: by %{GREEDYDATA:pam_by})?
CRON_ACTION [A-Z ]+
CRONLOG %{SYSLOGBASE} \(%{USER:user}\) %{CRON_ACTION:action} \(%{DATA:message}\)
SYSLOGLINE %{SYSLOGBASE2} %{GREEDYDATA:message}
# IETF 5424 syslog(8) format (see http://www.rfc-editor.org/info/rfc5424)
SYSLOG5424PRI <%{NONNEGINT:syslog5424_pri}>
SYSLOG5424SD \[%{DATA}\]+
SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{HOSTNAME:syslog5424_host}|-) +(?:%{WORD:syslog5424_app}|-) +(?:%{WORD:syslog5424_proc}|-) +(?:%{WORD:syslog5424_msgid}|-) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|)
SYSLOG5424LINE %{SYSLOG5424BASE} +%{GREEDYDATA:syslog5424_msg}
patterns/syslog
编写配置文件
配置文件编写是一个难点，这里有一些示例供参考：https://github.com/logstash-plugins/logstash-patterns-core/tree/master/patterns
input {
beats {
port => 5048
host => "10.80.2.181"
}
}
filter {
if [type] == "51-nginxaccesslog" {
grok {
patterns_dir => ["./patterns"]
match => { "message" => "%{IPORHOST:server_name} %{IPORHOST:server_ip} %{IPORHOST:clientip} %{NUMBER:clientport} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:status} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} %{NUMBER:request_time:float} (?:%{NUMBER:upstream_time:float}|-)" }
remove_field => ["message"]
}
date {
match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z"]
}
} else if [type] == "51-nginxerrorlog" {
grok {
patterns_dir => ["./patterns"]
match => { "message" => "%{DATESTAMP} %{SYSLOG5424SD:nginx_error_level} %{GREEDYDATA:nginx_error_msg}"}
remove_field => ["message"]
}
date {
match => [ "timestamp", "YYYY/MMM/dd HH:mm:ss"]
}
} else if [type] == "51-phperrorlog" {
grok {
patterns_dir => ["./patterns"]
match => { "message" => "%{SYSLOG5424SD} (?:%{DATA:php_error_level}\:) %{GREEDYDATA:error_msg}" }
remove_field => ["message"]
}
date {
match => [ "timestamp", "dd-MMM-YYYY HH:mm:ss Z"]
}
}
}
output {
if "_grokparsefailure" in [tags] {
file { path => "/var/log/logstash/grokparsefailure-%{[type]}-%{+YYYY.MM.dd}.log" }
}
elasticsearch {
hosts => ["10.80.2.83:9200","10.80.2.84:9200"]
sniffing => true
manage_template => false
template_overwrite => true
index => "%{[type]}-%{+YYYY.MM.dd}"
document_type => "%{[type]}"
}
}
conf.d/logstash.conf
编写启动脚本
#!/bin/sh
# Init script for logstash
# Maintained by Elasticsearch
# Generated by pleaserun.
# Implemented based on LSB Core 3.1:
# * Sections: 20.2, 20.3
#
### BEGIN INIT INFO
# Provides: logstash
# Required-Start: $remote_fs $syslog
# Required-Stop: $remote_fs $syslog
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description:
# Description: Starts Logstash as a daemon.
### END INIT INFO
PATH=/sbin:/usr/sbin:/bin:/usr/bin
export PATH
if [ `id -u` -ne 0 ]; then
echo "You need root privileges to run this script"
exit 1
fi
name=logstash
pidfile="/usr/local/logstash/$name.pid"
LS_USER=nobody
LS_GROUP=nobody
LS_HOME=/usr/local/logstash
#LS_HOME=/home/logstash
LS_HEAP_SIZE="12g"
LS_LOG_DIR=/data/logstash/log
LS_LOG_FILE="${LS_LOG_DIR}/$name.log"
LS_CONF_DIR=/usr/local/logstash/conf.d
LS_OPEN_FILES=65535
LS_NICE=-20
LS_THREADS=8
KILL_ON_STOP_TIMEOUT=${KILL_ON_STOP_TIMEOUT-0} #default value is zero to this variable but could be updated by user request
LS_OPTS=""
[ -r /etc/default/$name ] && . /etc/default/$name
[ -r /etc/sysconfig/$name ] && . /etc/sysconfig/$name
program=/usr/local/logstash/bin/logstash
args="agent -f ${LS_CONF_DIR} -w ${LS_THREADS} -l ${LS_LOG_FILE} ${LS_OPTS}"
quiet() {
"[email protected]" > /dev/null 2>&1
return $?
}
start() {
LS_JAVA_OPTS="${LS_JAVA_OPTS} -Djava.io.tmpdir=${LS_HOME}"
HOME=${LS_HOME}
export PATH HOME LS_HEAP_SIZE LS_JAVA_OPTS LS_USE_GC_LOGGING LS_GC_LOG_FILE
# chown doesn‘t grab the suplimental groups when setting the user:group - so we have to do it for it.
# Boy, I hope we‘re root here.
SGROUPS=$(id -Gn "$LS_USER" | tr " " "," | sed ‘s/,$//‘; echo ‘‘)
if [ ! -z $SGROUPS ]
then
EXTRA_GROUPS="--groups $SGROUPS"
fi
# set ulimit as (root, presumably) first, before we drop privileges
ulimit -n ${LS_OPEN_FILES}
# Run the program!
nice -n ${LS_NICE} chroot --userspec $LS_USER:$LS_GROUP $EXTRA_GROUPS / sh -c "
cd $LS_HOME
ulimit -n ${LS_OPEN_FILES}
exec \"$program\" $args
" > "${LS_LOG_DIR}/$name.stdout" 2> "${LS_LOG_DIR}/$name.err" &
# Generate the pidfile from here. If we instead made the forked process
# generate it there will be a race condition between the pidfile writing
# and a process possibly asking for status.
echo $! > $pidfile
echo "$name started."
return 0
}
stop() {
# Try a few times to kill TERM the program
if status ; then
pid=`cat "$pidfile"`
echo "Killing $name (pid $pid) with SIGTERM"
kill -TERM $pid
# Wait for it to exit.
for i in 1 2 3 4 5 6 7 8 9 ; do
echo "Waiting $name (pid $pid) to die..."
status || break
sleep 1
done
if status ; then
if [ $KILL_ON_STOP_TIMEOUT -eq 1 ] ; then
echo "Timeout reached. Killing $name (pid $pid) with SIGKILL. This may result in data loss."
kill -KILL $pid
echo "$name killed with SIGKILL."
else
echo "$name stop failed; still running."
return 1 # stop timed out and not forced
fi
else
echo "$name stopped."
fi
fi
}
status() {
if [ -f "$pidfile" ] ; then
pid=`cat "$pidfile"`
if kill -0 $pid > /dev/null 2> /dev/null ; then
# process by this pid is running.
# It may not be our pid, but that‘s what you get with just pidfiles.
# TODO(sissel): Check if this process seems to be the same as the one we
# expect. It‘d be nice to use flock here, but flock uses fork, not exec,
# so it makes it quite awkward to use in this case.
return 0
else
return 2 # program is dead but pid file exists
fi
else
return 3 # program is not running
fi
}
reload() {
if status ; then
kill -HUP `cat "$pidfile"`
fi
}
force_stop() {
if status ; then
stop
status && kill -KILL `cat "$pidfile"`
fi
}
configtest() {
# Check if a config file exists
if [ ! "$(ls -A ${LS_CONF_DIR}/* 2> /dev/null)" ]; then
echo "There aren‘t any configuration files in ${LS_CONF_DIR}"
return 1
fi
HOME=${LS_HOME}
export PATH HOME
test_args="--configtest -f ${LS_CONF_DIR} ${LS_OPTS}"
$program ${test_args}
[ $? -eq 0 ] && return 0
# Program not configured
return 6
}
case "$1" in
start)
status
code=$?
if [ $code -eq 0 ]; then
echo "$name is already running"
else
start
code=$?
fi
exit $code
;;
stop) stop ;;
force-stop) force_stop ;;
status)
status
code=$?
if [ $code -eq 0 ] ; then
echo "$name is running"
else
echo "$name is not running"
fi
exit $code
;;
reload) reload ;;
restart)
quiet configtest
RET=$?
if [ ${RET} -ne 0 ]; then
echo "Configuration error. Not restarting. Re-run with configtest parameter for details"
exit ${RET}
fi
stop && start
;;
configtest)
configtest
exit $?
;;
*)
echo "Usage: $SCRIPTNAME {start|stop|force-stop|status|reload|restart|configtest}" >&2
exit 3
;;
esac
exit $?
/etc/init.d/logstash
# chomd +x /etc/init.d/logstash # chown –R nobody.nobody /usr/local/logstash # chkconfig --add logstash
时间： 2025-01-07 16:00:44