#APACHE KAFKA 데이타마이그래이션 작업

 

> cd /home1/irteamsu/kafka2/kafka_2.11-1.1.0/

 

> base

> bin/kafka-server-stop.sh

> bin/zookeeper-server-stop.sh

 

########################cp -R /tmp/kafka-logs/ /home1/irteamsu/kafkadata/

 

cp /tmp/kafka-logs/20180416-0/* /home1/irteamsu/kafkadata/kafka-logs/20180416-0/

cp /tmp/kafka-logs/recovery-point-offset-checkpoint /home1/irteamsu/kafkadata/kafka-logs/

cp /tmp/kafka-logs/log-start-offset-checkpoint /home1/irteamsu/kafkadata/kafka-logs/

cp /tmp/kafka-logs/replication-offset-checkpoint /home1/irteamsu/kafkadata/kafka-logs/

 

 

> /home1/irteamsu/kafka2/kafka_2.11-1.1.0/config/server.properties 변경

 

> bin/zookeeper-server-start.sh config/zookeeper.properties &

> bin/kafka-server-start.sh config/server.properties &

 

 

#APACHE KAFKA 데이타마이그래이션 작업 확인

 

> bin/kafka-topics.sh --list --zookeeper localhost:2181

 

[irteamsu@cdb021.imgr kafka_2.11-1.1.0]$ netstat -an | grep 2181

tcp        0      0 0.0.0.0:2181            0.0.0.0:*               LISTEN

tcp        0      0 127.0.0.1:49964         127.0.0.1:2181          TIME_WAIT

tcp        0      0 127.0.0.1:35802         127.0.0.1:2181          ESTABLISHED

tcp        0      0 127.0.0.1:2181          127.0.0.1:35802         ESTABLISHED

[irteamsu@cdb021.imgr kafka_2.11-1.1.0]$

[irteamsu@cdb021.imgr kafka_2.11-1.1.0]$

 

[irteamsu@cdb021.imgr kafka_2.11-1.1.0]$

[irteamsu@cdb021.imgr 20180509-0]$ netstat -an | grep 9092

tcp        0      0 0.0.0.0:9092            0.0.0.0:*               LISTEN

tcp        0     52 10.113.68.29:9092       10.113.68.28:59222      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59274      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59254      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59268      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59278      ESTABLISHED

tcp        0      0 127.0.0.1:9092          127.0.0.1:42516         ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59276      ESTABLISHED

tcp        0     44 10.113.68.29:9092       10.113.68.28:59246      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59272      ESTABLISHED

tcp        0     72 10.113.68.29:52474      10.113.68.29:9092       ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59234      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59270      ESTABLISHED

tcp        0     44 10.113.68.29:9092       10.113.68.28:59240      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59266      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59262      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59284      ESTABLISHED

tcp        0     52 10.113.68.29:9092       10.113.68.28:59227      ESTABLISHED

tcp        0     52 10.113.68.29:9092       10.113.68.28:59226      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.29:52474      ESTABLISHED

tcp        0     44 10.113.68.29:9092       10.113.68.28:59242      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59250      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59248      ESTABLISHED

tcp        0     52 10.113.68.29:9092       10.113.68.28:59252      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59446      ESTABLISHED

tcp        0     52 10.113.68.29:9092       10.113.68.28:59230      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59224      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59286      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59280      ESTABLISHED

tcp        1      0 10.113.68.29:52470      10.113.68.29:9092       CLOSE_WAIT

tcp        0      0 10.113.68.29:9092       10.113.68.28:59258      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59256      ESTABLISHED

tcp        0     44 10.113.68.29:9092       10.113.68.28:59244      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59232      ESTABLISHED

tcp        0     52 10.113.68.29:9092       10.113.68.28:59236      ESTABLISHED

tcp        0      0 127.0.0.1:42516         127.0.0.1:9092          ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59220      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59448      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59264      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:59238      ESTABLISHED

[irteamsu@cdb021.imgr 20180509-0]$

 

 

 

 

 

[irteamsu@cdb021.imgr kafka_2.11-1.1.0]$

[irteamsu@cdb021.imgr kafka_2.11-1.1.0]$ bin/kafka-topics.sh --list --zookeeper localhost:2181

[2018-04-16 18:14:17,021] INFO Accepted socket connection from /127.0.0.1:54720 (org.apache.zookeeper.server.NIOServerCnxnFactory)

[2018-04-16 18:14:17,024] INFO Client attempting to establish new session at /127.0.0.1:54720 (org.apache.zookeeper.server.ZooKeeperServer)

[2018-04-16 18:14:17,025] INFO Established session 0x162cdbb20e00001 with negotiated timeout 30000 for client /127.0.0.1:54720 (org.apache.zookeeper.server.ZooKeeperServer)

20180411

20180412

20180413

20180414

20180415

20180416

20180417

20180418

20180419

20180420

20180421

20180422

20180423

20180424

20180425

20180426

20180427

20180428

20180429

20180430

20180431

 

[2018-04-16 18:14:17,066] INFO Processed session termination for sessionid: 0x162cdbb20e00001 (org.apache.zookeeper.server.PrepRequestProcessor)

[2018-04-16 18:14:17,067] INFO Closed socket connection for client /127.0.0.1:54720 which had sessionid 0x162cdbb20e00001 (org.apache.zookeeper.server.NIOServerCnxn)

[irteamsu@cdb021.imgr kafka_2.11-1.1.0]$

[irteamsu@cdb021.imgr kafka_2.11-1.1.0]$ netstat -an | grep 2181

tcp        0      0 0.0.0.0:2181            0.0.0.0:*               LISTEN

tcp        0      0 127.0.0.1:54646         127.0.0.1:2181          ESTABLISHED

tcp        0      0 127.0.0.1:54720         127.0.0.1:2181          TIME_WAIT

tcp        0      0 127.0.0.1:2181          127.0.0.1:54646         ESTABLISHED

[irteamsu@cdb021.imgr kafka_2.11-1.1.0]$ netstat -an | grep 9092

tcp        0      0 0.0.0.0:9092            0.0.0.0:*               LISTEN

tcp        0      0 10.113.68.29:9092       10.113.68.28:45470      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45450      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45414      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45476      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45424      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45440      ESTABLISHED

tcp        0      0 127.0.0.1:9092          127.0.0.1:48314         ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45426      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45442      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.29:58272      ESTABLISHED

tcp        0      0 127.0.0.1:48314         127.0.0.1:9092          ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45462      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45436      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45432      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45458      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45474      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45446      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45434      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45472      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.29:58268      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45460      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45420      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45464      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45444      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45438      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45422      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45452      ESTABLISHED

tcp        0      0 10.113.68.29:58268      10.113.68.29:9092       ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45428      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45448      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45418      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45478      ESTABLISHED

tcp        0      0 10.113.68.29:58272      10.113.68.29:9092       ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45480      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45454      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45466      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45430      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.10:53490      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45456      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45468      ESTABLISHED

tcp        0      0 10.113.68.29:9092       10.113.68.28:45416      ESTABLISHED

[irteamsu@cdb021.imgr kafka_2.11-1.1.0]$

'KAFKA' 카테고리의 다른 글

Apache Kafka ( 아파치 카프카 ) 2.11-0.9.0.0 소개 및 설치하기  (0) 2019.10.31
KAFKA 설치및 The Apache Kafka C/C++ library 설치  (0) 2019.10.31

Apache Kafka ( 아파치 카프카 ) 2.11-0.9.0.0 소개 및 설치하기

Apache Kafka ( 아파치 카프카 ) 
2.11-0.9.0.0 소개 및 설치하기

아파치 카프카(Apache Kafka)는 아파치 소프트웨어 재단이 스칼라로 개발한 오픈 소스 메시지 브로커 프로젝트이다. 
이 프로젝트는 실시간 데이터 피드를 관리하기 위해 통일된, 높은 스루풋의 낮은 레티어신을 지닌 플랫폼을 제공하는 것이 목표이다. 
요컨대 분산 트랜잭션 로그로 구성된, 상당히 확장 가능한 pub/sub 메시지 큐로 정의할 수 있으며, 스트리밍 데이터를 처리하기 위한 기업 인프라를 위한 고부가 가치 기능이다.
디자인은 트랜잭션 로그에 많은 영향을 받았다
아파치 카프카는 원래 링크드인이 개발한 것으로, 2011년 초에 최종적으로 오픈 소스화되었다. 2012년 10월 23일에는 아파치 인큐베이터로부터 완전히 빠져나왔다. 
2014년 11월 링크드인에서 카프카를 만들던 일부 엔지니어들이 카프카에 집중하기 위해 Confluent라는 새로운 회사를 창립하였다

카프카 설치는 간단합니다.

아파치 카프카 다운로드 링크
https://kafka.apache.org/downloads


이후 설치할 공간에서 압출을 푼 뒤

$tar zvxf kafka_2.11-0.9.0.0.tgz
$cd kafka_2.11-0.9.0.0

환경 설정을 진행합니다.
환경설정 파일은 config 폴더에 위치해 있습니다.

vi config/server.properties

그 후 사용할 Datadir 폴더와 
DataLogdir 폴더를 사용할 위치에 만들어 줍니다.
데이터가 큰 경우에는 OS에 구성하면 안되고 Data 영역에 사용하여야 합니다.

log.dirs 설정
만들어 준 위치를 넣어줍니다.

broker.id 를 설정해 줍니다.
유니크한 숫자를 브로커 마다 각각 입력하시면 됩니다.
주키퍼를 설치한 서버정보를 입력합니다.


설치를 안했을 경우 주키퍼를 실행하기 위해서
vi config/zookeeper.properties
zookeeper.connect=localhost:2181 (주키퍼를 설치한 서버)

주키퍼 설정을 해줍시다.

bin/zookeeper-server-start.sh config/zookeeper.properties
(주키퍼 실행 미리 설치 안했을 경우)

bin/kafka-server-start.sh config/server.properties 
(카프카 실행)

'KAFKA' 카테고리의 다른 글

APACHE KAFKA 데이타마이그래이션 작업(102)  (0) 2019.10.31
KAFKA 설치및 The Apache Kafka C/C++ library 설치  (0) 2019.10.31

READY FILE

wget http://apache.mirror.cdnetworks.com/kafka/1.1.0/kafka_2.11-1.1.0.tgz 
wget http://apache.mirror.cdnetworks.com/zookeeper/zookeeper-3.4.11/zookeeper-3.4.11.tar.gz


KAFKA INSTALL(JAVA VERSION) - https://kafka.apache.org/quicksta
1. Download the 1.1.0 release and un-tar it. 
> tar -xzf kafka_2.11-1.1.0.tgz
> cd kafka_2.11-1.1.0


2. Start the server
2.1 Kafka uses ZooKeeper so you need to first start a ZooKeeper server if you don't already have one. You can use the convenience script packaged with kafka to get a quick-and-dirty single-node ZooKeeper instance.

> bin/zookeeper-server-start.sh config/zookeeper.properties

3. Step 3: Create a topic
3.1 Let's create a topic named "test" with a single partition and only one replica:

> bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
> bin/kafka-server-start.sh config/server.properties
> for ii in `seq 20180411 20180431`; do bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic $ii; done
> cd /home1/irteamsu/kafka2/kafka_2.11-1.1.0
> bin/kafka-server-start.sh config/server.properties
> bin/kafka-topics.sh --list --zookeeper localhost:2181

4. Send some messages
4.1 Kafka comes with a command line client that will take input from a file or from standard input and send it out as messages to the Kafka cluster. By default, each line will be sent as a separate message.

Run the producer and then type a few messages into the console to send to the server.

> bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test
 
5. Start a consumer
5.1 Kafka also has a command line consumer that will dump out messages to standard output.

> bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning
> bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic 20180412 --from-beginning
 

[irteamsu@cdb021.imgr kafka_2.11-1.1.0]$ netstat -an | grep 2181
tcp        0      0 0.0.0.0:2181            0.0.0.0:*               LISTEN
tcp        0      0 127.0.0.1:49964         127.0.0.1:2181          TIME_WAIT
tcp        0      0 127.0.0.1:35802         127.0.0.1:2181          ESTABLISHED
tcp        0      0 127.0.0.1:2181          127.0.0.1:35802         ESTABLISHED
[irteamsu@cdb021.imgr kafka_2.11-1.1.0]$ netstat -an | grep 9092
tcp        0      0 0.0.0.0:9092            0.0.0.0:*               LISTEN
tcp        0      0 10.113.68.29:9092       10.113.68.28:50494      ESTABLISHED
tcp        0      0 10.113.68.29:9092       10.113.68.28:50516      ESTABLISHED
tcp        0      0 10.113.68.29:9092       10.113.68.28:50502      ESTABLISHED
tcp        0      0 10.113.68.29:9092       10.113.68.28:50504      ESTABLISHED
tcp        0      0 10.113.68.29:9092       10.113.68.28:50480      ESTABLISHED
tcp        0      0 127.0.0.1:43490         127.0.0.1:9092          ESTABLISHED
tcp        0      0 127.0.0.1:9092          127.0.0.1:43490         ESTABLISHED
tcp        0      0 10.113.68.29:9092       10.113.68.28:50520      ESTABLISHED
tcp        0      0 10.113.68.29:9092       10.113.68.28:50490      ESTABLISHED
tcp        0      0 10.113.68.29:9092       10.113.68.28:50481      ESTABLISHED
tcp        0     72 10.113.68.29:53448      10.113.68.29:9092       ESTABLISHED
tcp        0      0 10.113.68.29:9092       10.113.68.28:50508      ESTABLISHED
[irteamsu@cdb021.imgr kafka_2.11-1.1.0]$

/*------------------------------------------------------------
 * The Apache Kafka C/C++ library 
 *-----------------------------------------------------------/

[irteamsu@cdb021.imgr librdkafka]$ history
    1  2018-04-04_17:44:32\ ls
    2  2018-04-04_17:44:40\ cd
    3  2018-04-04_17:44:40\ ls
    4  2018-04-04_17:44:45\ cd install
    5  2018-04-04_17:44:49\ mkdir install
    6  2018-04-04_17:44:51\ cd install/
    7  2018-04-04_17:44:51\ ls
    8  2018-04-04_17:45:04\ mkdir kafka
    9  2018-04-04_17:45:05\ cd kafka/
   10  2018-04-04_17:45:06\ ls
   11  2018-04-04_17:45:10\ git clone https://github.com/edenhill/librdkafka
   12  2018-04-04_17:45:14\ ls
   13  2018-04-04_17:45:16\ cd librdkafka/
   14  2018-04-04_17:45:16\ ls
   15  2018-04-04_17:45:23\ ./configure
   16  2018-04-04_17:45:26\ make
   17  2018-04-04_17:46:08\ vi README.
   18  2018-04-04_17:46:13\ vi README.md
   19  2018-04-04_17:46:26\ sudo make install
   20  2018-04-04_17:49:19\ pwd
   21  2018-04-04_17:49:21\ history
[irteamsu@cdb021.imgr librdkafka]$ pwd
/home1/irteamsu/install/kafka/librdkafka

#include:/usr/local/include/librdkafka/
#lib:/usr/local/lib


ReadQueueApacheData: error while loading shared libraries: librdkafka.so.1: cannot open shared object file: No such file or directory
ReadQueueApacheData: error while loading shared libraries: librdkafka.so.1: cannot open shared object file: No such file or directory
ReadQueueApacheData: error while loading shared libraries: librdkafka.so.1: cannot open shared object file: No such file or directory

#프로파일에 저장
export LD_LIBRARY_PATH=/usr/local/lib

 

https://github.com/edenhill/librdkafka/blob/master/examples/rdkafka_simple_producer.c
https://github.com/edenhill/librdkafka/blob/master/examples/rdkafka_consumer_example.c

 

 

'KAFKA' 카테고리의 다른 글

APACHE KAFKA 데이타마이그래이션 작업(102)  (0) 2019.10.31
Apache Kafka ( 아파치 카프카 ) 2.11-0.9.0.0 소개 및 설치하기  (0) 2019.10.31

+ Recent posts

티스토리툴바