gfwleak/zhanghongqing-knowledge-log
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1.过滤异常数据 2.优化sink写入代码 3.优化clickhouse配置

Browse Source
This commit is contained in:
zhanghongqing
2022-09-23 17:10:05 +08:00
parent 25e5b51766
commit 9cdfe060cf
31 changed files with 396 additions and 1001 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
properties/default_config.properties
View File

@@ -45,3 +45,15 @@ log.transform.type=1
#\u4E24\u4E2A\u8F93\u51FA\u4E4B\u95F4\u7684\u6700\u5927\u65F6\u95F4(\u5355\u4F4Dmilliseconds)
buffer.timeout=15000
### datasource-pool
spring.datasource.hikari.minimum-idle=1
spring.datasource.hikari.maximum-pool-size=10
spring.datasource.hikari.auto-commit=true
spring.datasource.hikari.idle-timeout=10000
spring.datasource.hikari.max-lifetime=30000
spring.datasource.hikari.connection-timeout=30000
spring.datasource.hikari.connection-test-query=SELECT 1
ck.max.threads=20
#\u5355\u4F4Dsecond\uFF0C\u5065\u5EB7\u68C0\u67E5\u95F4\u9694
ck.schedule.actualization=20

23
properties/service_flow_config.properties
View File

@@ -1,14 +1,14 @@
#--------------------------------\u5730\u5740\u914D\u7F6E------------------------------#
#\u7BA1\u7406kafka\u5730\u5740,\u591A\u53F0\u9017\u53F7\u8FDE\u63A5ip1:9094,ip2:9094
source.kafka.servers=192.168.44.12:9094
source.kafka.servers=192.168.44.85:9094
#\u7BA1\u7406\u8F93\u51FAkafka\u5730\u5740
sink.kafka.servers=192.168.44.12:9094
sink.kafka.servers=192.168.44.85:9094
#--------------------------------HTTP/\u5B9A\u4F4D\u5E93/ssl------------------------------#
tools.library=
#--------------------------------Kafka\u6D88\u8D39/\u751F\u4EA7\u914D\u7F6E------------------------------#
#\u8BFB\u53D6topic,\u5B58\u50A8\u8BE5spout id\u7684\u6D88\u8D39offset\u4FE1\u606F\uFF0C\u53EF\u901A\u8FC7\u8BE5\u62D3\u6251\u547D\u540D;\u5177\u4F53\u5B58\u50A8offset\u7684\u4F4D\u7F6E\uFF0C\u786E\u5B9A\u4E0B\u6B21\u8BFB\u53D6\u4E0D\u91CD\u590D\u7684\u6570\u636E\uFF1B
group.id=KNOWLEDGE-GROUPtest
group.id=KNOWLEDGE-GROUP-20220905
#--------------------------------topology\u914D\u7F6E------------------------------#
#consumer \u5E76\u884C\u5EA6
source.parallelism=1
@@ -47,7 +47,7 @@ sink.arangodb.table.r.nx.domain2domain=R_NX_DOMAIN2DOMAIN
#\u4F7F\u7528flink\u5165\u5E93\u539F\u59CB\u65E5\u5FD70\uFF1A\u5426\uFF0C1\uFF1A\u662F
sink.ck.raw.log.insert.open=1
#clickhouse\u914D\u7F6E\uFF0C\u591A\u4E2A\u9017\u53F7\u8FDE\u63A5 ip1:8123,ip2:8123
ck.hosts=192.168.44.12:8123
ck.hosts=192.168.44.85:8123
# ,192.168.44.86:8123,192.168.44.87:8123
ck.database=tsg_galaxy_v3
ck.username=tsg_insert
@@ -56,19 +56,19 @@ ck.pin=galaxy2019
ck.connection.timeout=10000
ck.socket.timeout=600000
#clickhouse\u5165\u5E93\u6279\u91CF\u5355\u4F4D\u6761
ck.batch=20000
ck.batch=100000
#clickhouse\u5165\u5E93\u524D\u79EF\u7D2F\u6279\u91CF\u65F6\u95F4\u5355\u4F4D\u6BEB\u79D2
sink.ck.batch.delay.time=3000
sink.ck.batch.delay.time=30000
#flink \u65E5\u5FD7\u5EF6\u8FDF\u8D85\u65F6\u65F6\u95F4
flink.watermark.max.delay.time=60
#ck relation\u7EDF\u8BA1\u65F6\u95F4\u95F4\u9694 \u5355\u4F4Ds
log.aggregate.duration=10
log.aggregate.duration=5
#arangodb \u7EDF\u8BA1\u65F6\u95F4\u95F4\u9694 \u5355\u4F4Ds
log.aggregate.duration.graph=30
log.aggregate.duration.graph=10
#arangoDB\u53C2\u6570\u914D\u7F6E
arangodb.host=192.168.44.12
arangodb.host=192.168.44.83
arangodb.port=8529
arangodb.user=root
arangodb.password=galaxy_2019
@@ -79,4 +79,7 @@ arangodb.thread.pool.number=10
#\u6279\u91CF\u7D2F\u8BA1\u65F6\u95F4\u5355\u4F4D\u6BEB\u79D2ms
sink.arangodb.batch.delay.time=1000
aggregate.max.value.length=18
aggregate.max.value.length=18
#\u662F\u5426\u5165ip2ip\u8868 1:\u662F
sink.arangodb.raw.log.insert.open=0