Bigdata-Kafka-node模块实现调用js发送数据

Posted by Yancy on 2017-08-01

mongodb写到kafka 指定topic消费。为了保证数据稳定可靠性。
配合大数据在countly 使用开源Kafka-node是一个Node.js客户端 写js程序让countly三台集群分别数据到kafka 做新的topic主题备份。

1
npm install kafka-node

进入kafka-node目录: vim kafka_test.js

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
var kafka = require('kafka-node'),
HighLevelProducer = kafka.HighLevelProducer,
//Producer = kafka.Producer,
client = new kafka.Client('169.44.62.139:2281,169.44.59.138:2281,169.44.62.137:2281'),
//producer = new Producer(client);
producer = new HighLevelProducer(client);
console.log('连接kafka中');
var argv = {
topic: "test1"
};
var topic = argv.topic || 'test1';
var p = argv.p || 0;
var a = argv.a || 0;
var producer = new HighLevelProducer(client, {
requireAcks: 1,
partitionerType: 3
});
console.log(producer);
producer.on('ready', function() {
var args = {
appid: '222-wx238c28839a133d0e',
createTime: '222-ddd',
toUserName: '222-wx238c28839a133d0e',
fromUserName: '222-wx238c28839a133d0e'
};
producer.send([{
topic: topic,
partition: p,
messages: [JSON.stringify(args)],
attributes: a
}], function(err, result) {
console.log(err || result);
process.exit();
});
console.log(args);
});

官网地址:https://www.npmjs.com/package/kafka-node#install-kafka

交流学习:

🐧 Linux shell_高级运维派: 459096184 圈子 (系统运维-应用运维-自动化运维-虚拟化技术研究欢迎加入)
🐧 BigData-Exchange School : 521621407 圈子(大数据运维)(Hadoop开发人员)(大数据研究爱好者) 欢迎加入

相应Bidata有内部微信交流群互相学习,加入QQ群有链接。