【大数据平台】大数据平台 HDP2.5.3遇到的坑：配置篇

Kafka

Flume:

sourceAgent.channels = logCh kafkaCh

sourceAgent.sources = sourceDataSource

sourceAgent.sinks = logSink kafkaSink

sourceAgent.channels.kafkaCh.type = memory

sourceAgent.channels.kafkaCh.capacity = 1000000

sourceAgent.channels.kafkaCh.transactionCapacity = 1000000

sourceAgent.channels.kafkaCh.keep-alive = 10

sourceAgent.channels.logCh.type = memory

sourceAgent.channels.logCh.capacity = 1000000

sourceAgent.channels.logCh.transactionCapacity = 1000000

sourceAgent.channels.logCh.keep-alive = 10

sourceAgent.sources.sourceDataSource.type = org.apache.flume.source.http.HTTPSource

sourceAgent.sources.sourceDataSource.bind= 172.21.9.74

sourceAgent.sources.sourceDataSource.port = 30001

sourceAgent.sources.sourceDataSource.channels = logCh kafkaCh

sourceAgent.sinks.logSink.channel = logCh

sourceAgent.sinks.logSink.type = logger

sourceAgent.sinks.kafkaSink.channel = kafkaCh

sourceAgent.sinks.kafkaSink.type = org.apache.flume.sink.kafka.KafkaSink

sourceAgent.sinks.kafkaSink.topic = icestone-gateway-event-topic

sourceAgent.sinks.kafkaSink.brokerList =172.21.9.70:6667,172.21.9.72:6667,172.21.9.73:6667,172.21.9.74:6667

sourceAgent.sinks.kafkaSink.flumeBatchSize = 20

sourceAgent.sinks.kafkaSink.producer.acks = 1

sourceAgent.sinks.kafkaSink.producer.linger.ms = 1

sourceAgent.sinks.kafkaSink.producer.compression.type = none

HDFS

YARN

Hbase

Oozie

Spark

MR2