gfwleak/galaxy-tsg-olap-storm-log-stream-completion
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

提交线上使用版本

Browse Source
This commit is contained in:
qidaijie
2020-12-25 17:32:54 +08:00
parent 5438144b07
commit e6c602a154
43 changed files with 1446 additions and 2299 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/main/java/cn/ac/iie/spout/CustomizedKafkaSpout.java
View File

@@ -1,6 +1,7 @@
package cn.ac.iie.spout;
import cn.ac.iie.common.FlowWriteConfig;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
@@ -31,14 +32,45 @@ public class CustomizedKafkaSpout extends BaseRichSpout {
private static Properties createConsumerConfig() {
Properties props = new Properties();
props.put("bootstrap.servers", FlowWriteConfig.BOOTSTRAP_SERVERS);
props.put("bootstrap.servers", FlowWriteConfig.INPUT_KAFKA_SERVERS);
props.put("group.id", FlowWriteConfig.GROUP_ID);
props.put("session.timeout.ms", "60000");
props.put("max.poll.records", 3000);
props.put("max.partition.fetch.bytes", 31457280);
props.put("auto.offset.reset", FlowWriteConfig.AUTO_OFFSET_RESET);
// switch (FlowWriteConfig.KAFKA_TOPIC) {
// case "PROXY-EVENT-LOG":
// props.put("client.id", "proxy");
// break;
// case "RADIUS-RECORD-LOG":
// props.put("client.id", "radius");
// break;
// case "CONNECTION-RECORD-LOG":
// props.put("client.id", "connection");
// break;
// case "SECURITY-EVENT-LOG":
// props.put("client.id", "security");
// break;
// case "CONNECTION-SKETCH":
// props.put("client.id", "sketch");
// break;
// case "ACTIVE-DEFENCE-EVENT-LOG":
// props.put("client.id", "active");
// break;
// case "SYS-PACKET-CAPTURE-LOG":
// props.put("client.id", "packet");
// break;
//
// default:
// }
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
/**
* kafka限流配置-20201117
*/
props.put(ConsumerConfig.CLIENT_ID_CONFIG, FlowWriteConfig.CONSUMER_CLIENT_ID);
return props;
}