flume + kafka 基本配置

apache-flume1.6 sink默认支持kafka

• [FLUME-2242] - Flume Sink and Source for Apache Kafka

官方给的例子很贴心，可以直接运行=，=，详细配置之后慢慢看。

a1.channels = channel1
a1.sources = src-1
a1.sinks = k1

a1.sources.src-1.type = spooldir
a1.sources.src-1.channels = channel1
a1.sources.src-1.spoolDir = /opt/flumeSpool/
a1.sources.src-1.fileHeader = false

#a1.sources.src-1.type = exec
#a1.sources.src-1.command = tail -F /opt/flumeSpool/abc.log
#a1.sources.src-1.channels = channel1

a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.topic = testlog
a1.sinks.k1.brokerList = 1.1.1.11:9092,1.1.1.12:9092
a1.sinks.k1.requiredAcks = 1
a1.sinks.k1.batchSize = 20
a1.sinks.k1.channel = channel1

a1.channels.channel1.type   = org.apache.flume.channel.kafka.KafkaChannel
a1.channels.channel1.capacity = 10000
a1.channels.channel1.transactionCapacity = 1000
a1.channels.channel1.brokerList=1.1.1.11:9092,1.1.1.12:9092
a1.channels.channel1.topic=testlog
a1.channels.channel1.zookeeperConnect=1.1.1.13:2181