CentOS Install kafka

一.zookeeper集群安装

# 环境

192.168.190.152
192.168.190.153
192.168.190.154

Install jdk

# tar zxvf jdk-8u65-linux-x64.tar.gz
# mv jdk1.8.0_65 /usr/local/
# vi /etc/profile.d/jdk.sh
JAVA_HOME=/usr/local/jdk1.8.0_65
export JAVA_HOME
PATH=$JAVA_HOME/bin:$PATH
export PATH

# source /etc/profile

Install zookeeper

# wget http://www-eu.apache.org/dist/zookeeper/zookeeper-3.4.9/zookeeper-3.4.9.tar.gz

# tar zxvf zookeeper-3.4.9.tar.gz
# mv zookeeper-3.4.9 /usr/local/
# useradd zookeeper
# chown -R zookeeper.zookeeper /usr/local/zookeeper-3.4.9

# cd /usr/local/zookeeper-3.4.9/conf/
# cp zoo_sample.cfg zoo.cfg
# mkdir /etc/zookeeper
# ln -sf /usr/local/zookeeper-3.4.9/conf/zoo.cfg /etc/zookeeper/
# mkdir -p /var/lib/zookeeper
# chown -R zookeeper:zookeeper /var/lib/zookeeper

# 用来标识主机
#echo 1 > /var/lib/zookeeper/myid
#echo 2 > /var/lib/zookeeper/myid
#echo 3 > /var/lib/zookeeper/myid
# vi /etc/zookeeper/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/var/lib/zookeeper
clientPort=2181
maxClientCnxns=0
server.1=192.168.190.152:2888:3888
server.2=192.168.190.153:2888:3888
server.3=192.168.190.154:2888:3888
[[email protected] ~]# vi /etc/init.d/zookeeper
#! /bin/sh
#
# chkconfig: 2345 90 10
# description: zookeeper daemon

. /etc/init.d/functions

# You will probably want to change only two following lines.
BASEDIR="/usr/local/zookeeper-3.4.9"
USER="zookeeper"

PROG="zookeeper"
CMD="bin/zkServer.sh"
RETVAL=0

start () {
echo -n $"Starting ${PROG}: "
runuser ${USER} -c "cd ${BASEDIR}
${CMD} start > /dev/null &"
echo
}

stop () {
echo -n $"Stopping ${PROG}: "
runuser ${USER} -c "cd ${BASEDIR}
${CMD} stop > /dev/null &"
echo
}

restart () {
stop
start
}

# See how we were called.
case "$1" in
start)
start
;;
stop)
stop
;;
restart)
restart
;;
*)
echo $"Usage: $0 {start|stop|status}"
RETVAL=2
;;
esac

exit $RETVAL

# chmod 755 /etc/init.d/zookeeper
# /etc/init.d/zookeeper start
# chkconfig zookeeper on
# ./zkCli.sh

# 查看当前节点列表
[zk: localhost:2181(CONNECTED) 6] ls /
[zookeeper]

# 创建节点
[zk: localhost:2181(CONNECTED) 7] create /test "test"
Created /test
[zk: localhost:2181(CONNECTED) 8] ls /
[zookeeper, test]

# 查看节点数据
[zk: localhost:2181(CONNECTED) 9] get /test
test
cZxid = 0x100000004
ctime = Thu Oct 13 08:35:04 CST 2016
mZxid = 0x100000004
mtime = Thu Oct 13 08:35:04 CST 2016
pZxid = 0x100000004
cversion = 0
dataVersion = 0
aclVersion = 0
ephemeralOwner = 0x0
dataLength = 4
numChildren = 0

# 设置节点数据
[zk: localhost:2181(CONNECTED) 10] set /test "111111"
cZxid = 0x100000004
ctime = Thu Oct 13 08:35:04 CST 2016
mZxid = 0x100000005
mtime = Thu Oct 13 08:35:41 CST 2016
pZxid = 0x100000004
cversion = 0
dataVersion = 1
aclVersion = 0
ephemeralOwner = 0x0
dataLength = 6
numChildren = 0

# 删除节点数据
[zk: localhost:2181(CONNECTED) 11] delete /test
[zk: localhost:2181(CONNECTED) 12] ls /
[zookeeper]

二.kafka集群安装

# 环境

192.168.190.152
192.168.190.153
192.168.190.154
wget http://mirrors.cnnic.cn/apache/kafka/0.10.0.1/kafka_2.11-0.10.0.1.tgz

# tar zxvf kafka_2.11-0.10.0.1.tgz
# mv kafka_2.11-0.10.0.1 /usr/local/
# cd /usr/local/kafka_2.11-0.10.0.1/
# vi config/server.properties
broker.id=0 # id唯一 0 1 2
port=9092
host.name=192.168.190.152 #192.168.190.153 192.168.190.154
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
num.partitions=1
default.replication.factor=2
num.recovery.threads.per.data.dir=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.190.152:2181,192.168.190.153:2181,192.168.190.154:2181
zookeeper.connection.timeout.ms=6000
# useradd kafka
# touch /etc/init.d/kafka
# chmod 755 /etc/init.d/kafka
# vi /etc/init.d/kafka
#!/bin/sh
#
# chkconfig: 345 99 01
# description: Kafka
#
# File : Kafka
#
# Description: Starts and stops the Kafka server
#

source /etc/rc.d/init.d/functions

KAFKA_HOME=/usr/local/kafka_2.11-0.10.0.1
KAFKA_USER=kafka
# See how we were called.
case "$1" in

start)
echo -n "Starting Kafka:"
/sbin/runuser $KAFKA_USER -c "nohup $KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties > /var/log/kafka/server.out 2> /var/log/kafka/server.err &"
echo " done."
exit 0
;;

stop)
echo -n "Stopping Kafka: "
/sbin/runuser $KAFKA_USER -c "ps -ef | grep kafka.Kafka | grep -v grep | awk ‘{print \$2}‘ | xargs kill"
echo " done."
exit 0
;;
hardstop)
echo -n "Stopping (hard) Kafka: "
/sbin/runuser $KAFKA_USER -c "ps -ef | grep kafka.Kafka | grep -v grep | awk ‘{print \$2}‘ | xargs kill -9"
echo " done."
exit 0
;;

status)
c_pid=`ps -ef | grep kafka.Kafka | grep -v grep | awk ‘{print $2}‘`
if [ "$c_pid" = "" ] ; then
echo "Stopped"
exit 3
else
echo "Running $c_pid"
exit 0
fi
;;

restart)
stop
start
;;

*)
echo "Usage: cassandra {start|stop|hardstop|status|restart}"
exit 1
;;

esac
# mkdir -p /var/log/kafka
# chown -R kafka.kafka /var/log/kafka
# chown -R kafka.kafka /usr/local/kafka_2.11-0.10.0.1/logs

# 启动服务
# /etc/init.d/kafka start

# 创建topic
# bin/kafka-topics.sh --create --zookeeper 192.168.190.152:2181,192.168.190.153:2181,192.168.190.154:2181 --replication-factor 2 --partitions 1 --topic test333

# 删除
# bin/kafka-topics.sh --delete --zookeeper 192.168.190.152:2181,192.168.190.153:2181,192.168.190.154:2181 --topic test333

# 查看topic
# bin/kafka-topics.sh --list --zookeeper localhost:2181

# 开启生产端
# bin/kafka-console-producer.sh --broker-list 192.168.190.152:9092,192.168.190.153:9092,192.168.190.154:9092 --topic test333

# 开启消费端
# bin/kafka-console-consumer.sh --zookeeper 192.168.190.152:2181,192.168.190.153:2181,192.168.190.154:2181 --topic test333 --from-beginning

# bin/kafka-topics.sh --describe --zookeeper 192.168.190.152:2181,192.168.190.153:2181,192.168.190.154:2181 --topic test333

# 错误处理
[2016-10-14 06:36:18,401] WARN Error while fetching metadata with correlation id 0 : {test333=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)
[2016-10-14 06:36:19,543] WARN Error while fetching metadata with correlation id 1 : {test333=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)
[2016-10-14 06:36:19,680] WARN Error while fetching metadata with correlation id 2 : {test333=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)
[2016-10-14 06:36:19,908] WARN Error while fetching metadata with correlation id 3 : {test333=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)
[2016-10-14 06:36:20,116] WARN Error while fetching metadata with correlation id 4 : {test333=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)
[2016-10-14 06:36:20,334] WARN Error while fetching metadata with correlation id 5 : {test333=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)
[2016-10-14 06:36:20,505] WARN Error while fetching metadata with correlation id 6 : {test333=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)
[2016-10-14 06:36:20,757] WARN Error while fetching metadata with correlation id 7 : {test333=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)

config/server.properties kafka必须配置hostname

时间： 2024-08-05 20:38:23

CentOS Install kafka的相关文章

CentOS install Desktop and Remotely access 远程管理Centos桌面 TigerVNC

Centos 安装完毕,但是通过windows 远程管理桌面: 当然我们可以通过secureCRT工具连接命令行,但是我希望通过windows 连接他们的远程桌面: 首先我们的保证 X windows等桌面环境安装成功: yum groupinstall -y "Desktop"   "Desktop Platform"   "Desktop Platform Development" "Fonts" "Genera

centos install shutter (How to enable Nux Dextop repository on CentOS or RHEL)

http://ask.xmodulo.com/enable-nux-dextop-repository-centos-rhel.html Question: I would like to install a RPM package which is available only in Nux Dextop repository. How can I set up Nux Dextop repository on CentOS or RHEL? Nux Dextop is a third-par

centos install jdk

方法一:手动解压JDK的压缩包,然后设置环境变量 1.在/usr/目录下创建java目录  [[email protected] ~]# mkdir/usr/java [[email protected] ~]# cd /usr/java 2.下载,然后解压 [[email protected] java]# curl -O http://download.oracle.com/otn-pub/java/jdk/7u79-b15/jdk-7u79-linux-x64.tar.gz [[email

Kafka学习之：Centos 下Kafka集群的安装

kafka是LinkedIn开发并开源的一个分布式MQ系统,现在是Apache的一个孵化项目.在它的主页描述kafka为一个高吞吐量的分布式(能将消息分散到不同的节点上)MQ.在这片博文中,作者简单提到了开发kafka而不选择已有MQ系统的原因.两个原因:性能和扩展性.Kafka仅仅由7000行Scala编写,据了解,Kafka每秒可以生产约25万消息(50 MB),每秒处理55万消息(110 MB). 安装准备 版本 Kafka版本:kafka_2.10-0.8.2.0 Zookeeper版本

centos Install Docker

安装必备软件 yum -y install iptables  iptables-services net-tools vim wget 安装Dockeryum -y install docker 下载centos镜像docker pull centos 查看centos 镜像docker images centos 运行一个Docker容器docker run -i -t centos /bin/bash 搜索Ubuntu镜像docker search Ubuntu 显示正在运行的容器(需要先

centos install redmine (项目管理工具)

安装环境:Centos.mysql.Ruby.Apache.Redmine yum update yum -y groupinstall "Development Tools" yum -y install ntp zlib zlib-devel sqlite-devel httpd mysql-server mysql-devel curl curl-devel httpd-devel apr-devel apr-util-devel mlocate manlibxml2-devel

Centos install Parosid

SELinux and Parosid To run the VisualEditor and Parsoid on a CentOS7-Server, you need to do the following. The requirement is a working MediaWiki (min. V1.25.x) setup. (Crate a LAMP setup on CentOS 7, RHEL 7, Fedora 22, or Scientific Linux 7) 1) Beco

docker centos install

1. yum update 2. yum install -y yum-utils device-mapper-persistent-data lvm2 3. yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo(阿里仓库) 4 yum list docker-ce 5. yum install docker-ce 6. systemctl start docke

Centos install Python2.7

#get python2.7wget https://www.python.org/ftp/python/2.7.9/Python-2.7.9.tgz #install pythontar zxvf Python-2.7.9.tgzcd Python-2.7.9./configuremakemake installcd ..rm -rf Python*