gfwleak/galaxy-tsg-olap-log-stream-doublewrite
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

针对p19项目研发的双写程序

Browse Source
This commit is contained in:
‘wangchengcheng’
2021-10-22 19:41:52 +08:00
parent a862f38b6d
commit bca1d5f40c
244 changed files with 6538 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
target/classes/com/zdjizhi/bean/FileMeta.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/bean/SourceList.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/common/FlowWriteConfig.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/topology/LogFlowWriteTopology.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/app/AppUtils$1.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/app/AppUtils.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/exception/FlowWriteException.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/functions/DealFileProcessFunction$1.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/functions/DealFileProcessFunction.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/functions/FilterNullFunction.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/functions/MapCompletedFunction.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/general/FileEdit.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/general/SnowflakeId.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/general/TransFormMap.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/general/TransFormObject.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/general/TransFormTypeMap.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/general/TransFunction.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/hbase/HBaseUtils$1.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/hbase/HBaseUtils.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/http/HttpClientUtil.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/json/JsonParseUtil.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/json/JsonTypeUtils.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/json/TypeUtils.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/kafka/CertUtils.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/kafka/Consumer.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/kafka/Producer.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/ordinary/MD5Utils.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/system/FlowWriteConfigurations.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/zookeeper/DistributedLock$LockException.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/zookeeper/DistributedLock.class Normal file
View File

Binary file not shown.

BIN
target/classes/com/zdjizhi/utils/zookeeper/ZookeeperUtils.class Normal file
View File

Binary file not shown.

50
target/classes/default_config.properties Normal file
View File

@@ -0,0 +1,50 @@
#producer<65><72><EFBFBD>ԵĴ<D4B5><C4B4><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
retries=0
#<23><><EFBFBD>ĺ<EFBFBD><C4BA><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>˵һ<CBB5><D2BB>Batch<63><68><EFBFBD><EFBFBD><EFBFBD><EFBFBD>֮<EFBFBD><D6AE><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ã<EFBFBD><C3A3><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Batch<63><68>û<EFBFBD><C3BB>д<EFBFBD><D0B4><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>뷢<EFBFBD>ͳ<EFBFBD>ȥ<EFBFBD><C8A5>
linger.ms=10
#<23><><EFBFBD><EFBFBD><EFBFBD>ڳ<EFBFBD>ʱ֮ǰδ<C7B0>յ<EFBFBD><D5B5><EFBFBD>Ӧ<EFBFBD><D3A6><EFBFBD>ͻ<EFBFBD><CDBB>˽<EFBFBD><CBBD>ڱ<EFBFBD>Ҫʱ<D2AA><CAB1><EFBFBD>·<EFBFBD><C2B7><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
request.timeout.ms=30000
#producer<65><72><EFBFBD>ǰ<EFBFBD><C7B0><EFBFBD>batch<63><68><EFBFBD>з<EFBFBD><D0B7>͵<EFBFBD>,<2C><><EFBFBD>δ<EFBFBD>С<EFBFBD><D0A1>Ĭ<EFBFBD><C4AC>:16384
batch.size=262144
#Producer<65><72><EFBFBD><EFBFBD><EFBFBD>ڻ<EFBFBD><DABB><EFBFBD><EFBFBD><EFBFBD>Ϣ<EFBFBD>Ļ<EFBFBD><C4BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>С
#64M
#buffer.memory=67108864
#128M
buffer.memory=134217728
#<23><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ÿ<EFBFBD>η<EFBFBD><CEB7>͸<EFBFBD>Kafka<6B><61><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>С,Ĭ<><C4AC>1048576
#5M
#max.request.size=5242880
#10M
max.request.size=10485760
#kafka SASL<53><4C>֤<EFBFBD>û<EFBFBD><C3BB><EFBFBD>
kafka.user=admin
#kafka SASL<53><4C>SSL<53><4C>֤<EFBFBD><D6A4><EFBFBD><EFBFBD>
kafka.pin=galaxy2019
#kafka source connection timeout
session.timeout.ms=60000
#kafka source poll
max.poll.records=3000
#kafka source poll bytes
max.partition.fetch.bytes=31457280
#hbase table name
hbase.table.name=subscriber_info
#<23>ʼ<EFBFBD>Ĭ<EFBFBD>ϱ<EFBFBD><CFB1><EFBFBD>
mail.default.charset=UTF-8
#kafka source protocol; SSL or SASL
kafka.source.protocol=SASL
#kafka sink protocol; SSL or SASL
kafka.sink.protocol=

72
target/classes/service_flow_config.properties Normal file
View File

@@ -0,0 +1,72 @@
#--------------------------------地址配置------------------------------#
#管理kafka地址
input.kafka.servers=10.3.60.3:9094
#管理输出kafka地址
output.kafka.servers=10.3.45.126:6667,10.3.45.127:6667,10.3.45.128:6667
#zookeeper 地址 用于配置log_id
zookeeper.servers=10.3.60.3:2181
#hbase zookeeper地址 用于连接HBase
hbase.zookeeper.servers=10.3.60.3:2181
#oos地址
oos.servers=10.3.45.124:8057
#--------------------------------HTTP/定位库------------------------------#
#定位库地址
#tools.library=/opt/dat/
tools.library=D:/dingweiku/dat/
#网关的schema位置
schema.http=http://10.3.60.3:9999/metadata/schema/v1/fields/proxy_event
#网关APP_ID 获取接口
app.id.http=http://10.3.60.3:9999/open-api/appDicList
#--------------------------------Kafka消费组信息------------------------------#
#kafka 接收数据topic
#input.kafka.topic=SESSION-RECORD
input.kafka.topic=PROXY-EVENT
#文件源数据topic
output.kafka.topic=TRAFFIC-FILE-METADATA
percent.kafka.topic=PROXY-EVENT
#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据；
group.id=session-record-log-20211018-A
#生产者压缩模式 none or snappy
producer.kafka.compression.type=none
#生产者ack
producer.ack=1
#--------------------------------topology配置------------------------------#
#consumer 并行度
consumer.parallelism=1
#转换函数并行度
transform.parallelism=1
#数据中心，取值范围(0-63)
data.center.id.num=0
#hbase 更新时间，如填写0则不更新缓存
hbase.tick.tuple.freq.secs=180
#app_id 更新时间，如填写0则不更新缓存
app.tick.tuple.freq.secs=0
#--------------------------------默认值配置------------------------------#
#邮件默认编码
mail.default.charset=UTF-8
#0不需要补全原样输出日志，1需要补全
log.need.complete=1