LABORATORY
[ Bigdata&Machine Learning ]
Kafka는 대용량 데이터를 위한 분산형 스트리밍 플랫폼입니다.
Kafka clustring Test를 수행해 봅니다.
전체Archtecture
#1 ip setting
3대의 Centos 7.3 준비
server.1=192.168.0.145:2888:3888
server.2=192.168.0.174:2888:3888
server.3=192.168.0.175:2888:3888
#2 setting zookeeper & kafka
wget http://mirror.navercorp.com/apache/kafka/2.2.0/kafka_2.12-2.2.0.tgz tar zxvf kafka_2.12-2.2.0.tgz mv kafka_2.12-2.2.0 kafka
configure zookeeper (1,2,3 server)[root@kafka1 min]# vi kafka/config/zookeeper.properties
# zookeeper servers
server.1=192.168.0.145:2888:3888
server.2=192.168.0.174:2888:3888
server.3=192.168.0.175:2888:3888
### configure kafka
[root@kafka1 min]# vi kafka/config/server.properties
broker.id=1 ## id를 server별로 설정
listeners=PLAINTEXT://:9092
zookeeper.connect=192.168.0.145:2181,192.168.0.174:2181,192.168.0.175:2181
delete.topic.enable=true
#3 start zookeeper & kafka
#### start zookeeper server and kafka server
[root@kafka1 min]# kafka/bin/zookeeper-server-start.sh -daemon ./kafka/config/zookeeper.properties
[root@kafka1 min]# kafka/bin/kafka-server-start.sh -daemon ./kafka/config/server.properties
#4 Verify all brokers
[root@kafka1 min]# ./kafka/bin/zookeeper-shell.sh 192.168.0.145:2181 ls /brokers/ids
Connecting to 192.168.0.145:2181
WATCHER::
WatchedEvent state:SyncConnected type:None path:null
[1, 2, 3]
#5 Create topic
### Create topic
kafka/bin/kafka-topics.sh –create –zookeeper 192.168.0.145:2181,192.168.0.175:2181,192.168.0.175:2181 –partitions 3 –replication-factor 3 –topic suicides
### List topic
kafka/bin/kafka-topics.sh –list –zookeeper 192.168.0.145:2181,192.168.0.175:2181,192.168.0.175:2181
### descirbe topic
[root@kafka1 min]# kafka/bin/kafka-topics.sh –describe –zookeeper 192.168.0.145:2181,192.168.0.175:2181,192.168.0.175:2181
Topic:suicides PartitionCount:3 ReplicationFactor:3 Configs:
Topic: suicides Partition: 0 Leader: 3 Replicas: 3,1,2 Isr: 3,1,2
Topic: suicides Partition: 1 Leader: 1 Replicas: 1,2,3 Isr: 1,2,3
Topic: suicides Partition: 2 Leader: 2 Replicas: 2,3,1 Isr: 2,3,1
#6 produce & consume
### produce
kafka/bin/kafka-console-producer.sh –broker-list 192.168.0.145:9092 –topic suicides
### consume
kafka/bin/kafka-console-consumer.sh –bootstrap-server 192.168.0.145:9092 –topic suicides –from-beginning
