gfwleak/galaxy-tsg-olap-flume-interceptor-flume-http-avro-gtpc
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

福建gtpc项目，线上flume拦截器配置

Browse Source
This commit is contained in:
wangchengcheng
2022-09-27 11:38:15 +08:00
commit 5c76b457b6
2 changed files with 86 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
http-avro-gtpc-f.properties Normal file
View File

@@ -0,0 +1,43 @@
#为source channel sink起名
plgtpc.sources = httpsource1
plgtpc.channels = channel1
plgtpc.sinks = kafkasink1
#指定source收集到的数据发送到哪个管道
#指定source数据收集策略
plgtpc.sources.httpsource1.type = http
plgtpc.sources.httpsource1.bind = 172.16.60.21
plgtpc.sources.httpsource1.port = 7701
plgtpc.sources.httpsource1.handler = org.apache.flume.sink.solr.morphline.BlobHandler
#plgtpc.sources.httpsource1.handler = org.apache.flume.source.http.JSONHandler
plgtpc.sources.httpsource1.channels = channel1
#指定channel为memory,即表示所有的数据都装进memory当中
plgtpc.channels.channel1.type = memory
plgtpc.channels.channel1.capacity = 10000
plgtpc.channels.channel1.transactionCapacity = 1000
plgtpc.channels.channel1.byteCapacityBufferPercentage = 40
plgtpc.channels.channel1.byteCapacity = 2684354560
#原始kafka-sink,直接将json数据写入kafka
plgtpc.sinks.kafkasink1.channel = channel1
plgtpc.sinks.kafkasink1.type = org.apache.flume.sink.kafka.KafkaSink
plgtpc.sinks.kafkasink1.kafka.topic = GTPC-RECORD
plgtpc.sinks.kafkasink1.allowTopicOverride = false
plgtpc.sinks.kafkasink1.kafka.bootstrap.servers = 192.168.8.32:9094
plgtpc.sinks.kafkasink1.kafka.flumeBatchSize = 262144
plgtpc.sinks.kafkasink1.kafka.producer.max.request.size = 5242880
plgtpc.sinks.kafkasink1.kafka.producer.acks = 1
plgtpc.sinks.kafkasink1.kafka.producer.linger.ms = 1
plgtpc.sinks.kafkasink1.kafka.producer.security.protocol= SASL_PLAINTEXT
plgtpc.sinks.kafkasink1.kafka.producer.sasl.mechanism= PLAIN
plgtpc.sinks.kafkasink1.kafka.producer.sasl.jaas.config= org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019";
#拦截器配置
plgtpc.sources.httpsource1.interceptors = i1
plgtpc.sources.httpsource1.interceptors.i1.type = com.zdjizhi.Avro2Gtpc$FlumeAvroAppBuilder
plgtpc.sources.httpsource1.interceptors.i1.kSchemaDir = /home/olap/flume-1.9.0/conf/k_schem.avsc
plgtpc.sources.httpsource1.interceptors.i1.fSchemaDir = /home/olap/flume-1.9.0/conf/f_schem.avsc
plgtpc.sources.httpsource1.interceptors.i1.schemaType = t_cdr_f
plgtpc.sources.httpsource1.interceptors.i1.commonVsysId=1
plgtpc.sources.httpsource1.interceptors.i1.gtpVersion=v2

43
http-avro-gtpc-k.properties Normal file
View File

@@ -0,0 +1,43 @@
#为source channel sink起名
plgtpc.sources = httpsource1
plgtpc.channels = channel1
plgtpc.sinks = kafkasink1
#指定source收集到的数据发送到哪个管道
#指定source数据收集策略
plgtpc.sources.httpsource1.type = http
plgtpc.sources.httpsource1.bind = 172.16.60.21
plgtpc.sources.httpsource1.port = 7702
plgtpc.sources.httpsource1.handler = org.apache.flume.sink.solr.morphline.BlobHandler
#plgtpc.sources.httpsource1.handler = org.apache.flume.source.http.JSONHandler
plgtpc.sources.httpsource1.channels = channel1
#指定channel为memory,即表示所有的数据都装进memory当中
plgtpc.channels.channel1.type = memory
plgtpc.channels.channel1.capacity = 10000
plgtpc.channels.channel1.transactionCapacity = 1000
plgtpc.channels.channel1.byteCapacityBufferPercentage = 40
plgtpc.channels.channel1.byteCapacity = 2684354560
#原始kafka-sink,直接将json数据写入kafka
plgtpc.sinks.kafkasink1.channel = channel1
plgtpc.sinks.kafkasink1.type = org.apache.flume.sink.kafka.KafkaSink
plgtpc.sinks.kafkasink1.kafka.topic = GTPC-RECORD
plgtpc.sinks.kafkasink1.allowTopicOverride = false
plgtpc.sinks.kafkasink1.kafka.bootstrap.servers = 192.168.8.32:9094
plgtpc.sinks.kafkasink1.kafka.flumeBatchSize = 262144
plgtpc.sinks.kafkasink1.kafka.producer.max.request.size = 5242880
plgtpc.sinks.kafkasink1.kafka.producer.acks = 1
plgtpc.sinks.kafkasink1.kafka.producer.linger.ms = 1
plgtpc.sinks.kafkasink1.kafka.producer.security.protocol= SASL_PLAINTEXT
plgtpc.sinks.kafkasink1.kafka.producer.sasl.mechanism= PLAIN
plgtpc.sinks.kafkasink1.kafka.producer.sasl.jaas.config= org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019";
#拦截器配置
plgtpc.sources.httpsource1.interceptors = i1
plgtpc.sources.httpsource1.interceptors.i1.type = com.zdjizhi.FlumeAvroApp$FlumeAvroAppBuilder
plgtpc.sources.httpsource1.interceptors.i1.kSchemaDir = /home/olap/flume-1.9.0/conf/k_schem.avsc
plgtpc.sources.httpsource1.interceptors.i1.fSchemaDir = /home/olap/flume-1.9.0/conf/f_schem.avsc
plgtpc.sources.httpsource1.interceptors.i1.schemaType = t_cdr_k
plgtpc.sources.httpsource1.interceptors.i1.commonVsysId=1
plgtpc.sources.httpsource1.interceptors.i1.gtpVersion=v2