增加24.09相关初始化sql及模版配置文件
This commit is contained in:
doufenghu
@@ -0,0 +1,50 @@
|
||||
flink.job.name=agg_traffic_eml_file_chunk_combiner
|
||||
|
||||
#kafka source配置
|
||||
#9092为无验证 9095为ssl 9094为sasl
|
||||
source.kafka.broker=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094
|
||||
source.kafka.topic=TRAFFIC-EML-FILE-STREAM-RECORD
|
||||
source.kafka.group.id=agg_traffic_eml_file_chunk_combiner_1
|
||||
#earliest从头开始 latest最新
|
||||
source.kafka.auto.offset.reset=latest
|
||||
source.kafka.session.timeout.ms=60000
|
||||
#每次拉取操作从分区中获取的最大记录数
|
||||
source.kafka.max.poll.records=1000
|
||||
#消费者从单个分区中一次性获取的最大字节数
|
||||
source.kafka.max.partition.fetch.bytes=31457280
|
||||
source.kafka.enable.auto.commit=true
|
||||
#kafka SASL验证用户名
|
||||
source.kafka.user=olap
|
||||
#kafka SASL及SSL验证密码
|
||||
source.kafka.pin=galaxy2024
|
||||
#SSL需要
|
||||
source.kafka.tools.library=/opt/tsg/olap/topology/data/
|
||||
|
||||
map.filter.expression=FileChunk.offset <= 1073741824
|
||||
|
||||
#窗口相关配置
|
||||
combiner.window.parallelism=1
|
||||
#窗口大小，单位秒
|
||||
combiner.window.size=10
|
||||
|
||||
#sink相关参数
|
||||
sink.parallelism=1
|
||||
#可选hos、oss、hbase
|
||||
sink.type=hos
|
||||
sink.async=false
|
||||
|
||||
#hos sink相关配置
|
||||
#访问nginx或单个hos配置为ip:port；访问多个hos，配置为ip1:port,ip2:port...
|
||||
sink.hos.endpoint=192.168.44.11:8186,192.168.44.14:8186
|
||||
sink.hos.bucket=traffic_eml_file_bucket
|
||||
sink.hos.token=c21f969b5f03d33d43e04f8f136e7682
|
||||
sink.hos.batch.size=1048576
|
||||
sink.hos.batch.interval.ms=10000
|
||||
|
||||
#http相关配置
|
||||
sink.http.client.retries.number=3
|
||||
sink.http.client.max.total=20
|
||||
sink.http.client.max.per.route=10
|
||||
sink.http.client.connect.timeout.ms=10000
|
||||
sink.http.client.request.timeout.ms=10000
|
||||
sink.http.client.socket.timeout.ms=60000
|
||||
@@ -0,0 +1,50 @@
|
||||
flink.job.name=agg_traffic_http_file_chunk_combiner
|
||||
|
||||
#kafka source配置
|
||||
#9092为无验证 9095为ssl 9094为sasl
|
||||
source.kafka.broker=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094
|
||||
source.kafka.topic=TRAFFIC-HTTP-FILE-STREAM-RECORD
|
||||
source.kafka.group.id=agg_traffic_http_file_chunk_combiner_1
|
||||
#earliest从头开始 latest最新
|
||||
source.kafka.auto.offset.reset=latest
|
||||
source.kafka.session.timeout.ms=60000
|
||||
#每次拉取操作从分区中获取的最大记录数
|
||||
source.kafka.max.poll.records=1000
|
||||
#消费者从单个分区中一次性获取的最大字节数
|
||||
source.kafka.max.partition.fetch.bytes=31457280
|
||||
source.kafka.enable.auto.commit=true
|
||||
#kafka SASL验证用户名
|
||||
source.kafka.user=olap
|
||||
#kafka SASL及SSL验证密码
|
||||
source.kafka.pin=galaxy2024
|
||||
#SSL需要
|
||||
source.kafka.tools.library=/opt/tsg/olap/topology/data/
|
||||
|
||||
map.filter.expression=FileChunk.offset <= 1073741824
|
||||
|
||||
#窗口相关配置
|
||||
combiner.window.parallelism=3
|
||||
#窗口大小，单位秒
|
||||
combiner.window.size=10
|
||||
|
||||
#sink相关参数
|
||||
sink.parallelism=3
|
||||
#可选hos、oss、hbase
|
||||
sink.type=hos
|
||||
sink.async=false
|
||||
|
||||
#hos sink相关配置
|
||||
#访问nginx或单个hos配置为ip:port；访问多个hos，配置为ip1:port,ip2:port...
|
||||
sink.hos.endpoint=192.168.44.11:8186,192.168.44.14:8186
|
||||
sink.hos.bucket=traffic_http_file_bucket
|
||||
sink.hos.token=c21f969b5f03d33d43e04f8f136e7682
|
||||
sink.hos.batch.size=1048576
|
||||
sink.hos.batch.interval.ms=10000
|
||||
|
||||
#http相关配置
|
||||
sink.http.client.retries.number=3
|
||||
sink.http.client.max.total=20
|
||||
sink.http.client.max.per.route=10
|
||||
sink.http.client.connect.timeout.ms=10000
|
||||
sink.http.client.request.timeout.ms=10000
|
||||
sink.http.client.socket.timeout.ms=60000
|
||||
@@ -0,0 +1,50 @@
|
||||
flink.job.name=agg_traffic_policy_capture_file_chunk_combiner
|
||||
|
||||
#kafka source配置
|
||||
#9092为无验证 9095为ssl 9094为sasl
|
||||
source.kafka.broker=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094
|
||||
source.kafka.topic=TRAFFIC-POLICY-CAPTURE-FILE-STREAM-RECORD
|
||||
source.kafka.group.id=agg_traffic_policy_capture_file_chunk_combiner_1
|
||||
#earliest从头开始 latest最新
|
||||
source.kafka.auto.offset.reset=latest
|
||||
source.kafka.session.timeout.ms=60000
|
||||
#每次拉取操作从分区中获取的最大记录数
|
||||
source.kafka.max.poll.records=1000
|
||||
#消费者从单个分区中一次性获取的最大字节数
|
||||
source.kafka.max.partition.fetch.bytes=31457280
|
||||
source.kafka.enable.auto.commit=true
|
||||
#kafka SASL验证用户名
|
||||
source.kafka.user=olap
|
||||
#kafka SASL及SSL验证密码
|
||||
source.kafka.pin=galaxy2024
|
||||
#SSL需要
|
||||
source.kafka.tools.library=/opt/tsg/olap/topology/data/
|
||||
|
||||
map.filter.expression=FileChunk.offset <= 1073741824
|
||||
|
||||
#窗口相关配置
|
||||
combiner.window.parallelism=3
|
||||
#窗口大小，单位秒
|
||||
combiner.window.size=10
|
||||
|
||||
#sink相关参数
|
||||
sink.parallelism=3
|
||||
#可选hos、oss、hbase
|
||||
sink.type=hos
|
||||
sink.async=false
|
||||
|
||||
#hos sink相关配置
|
||||
#访问nginx或单个hos配置为ip:port；访问多个hos，配置为ip1:port,ip2:port...
|
||||
sink.hos.endpoint=192.168.44.11:8186,192.168.44.14:8186
|
||||
sink.hos.bucket=traffic_policy_capture_file_bucket
|
||||
sink.hos.token=c21f969b5f03d33d43e04f8f136e7682
|
||||
sink.hos.batch.size=1048576
|
||||
sink.hos.batch.interval.ms=10000
|
||||
|
||||
#http相关配置
|
||||
sink.http.client.retries.number=3
|
||||
sink.http.client.max.total=20
|
||||
sink.http.client.max.per.route=10
|
||||
sink.http.client.connect.timeout.ms=10000
|
||||
sink.http.client.request.timeout.ms=10000
|
||||
sink.http.client.socket.timeout.ms=60000
|
||||
@@ -0,0 +1,50 @@
|
||||
flink.job.name=agg_traffic_rtp_file_chunk_combiner
|
||||
|
||||
#kafka source配置
|
||||
#9092为无验证 9095为ssl 9094为sasl
|
||||
source.kafka.broker=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094
|
||||
source.kafka.topic=TRAFFIC-RTP-FILE-STREAM-RECORD
|
||||
source.kafka.group.id=agg_traffic_rtp_file_chunk_combiner_1
|
||||
#earliest从头开始 latest最新
|
||||
source.kafka.auto.offset.reset=latest
|
||||
source.kafka.session.timeout.ms=60000
|
||||
#每次拉取操作从分区中获取的最大记录数
|
||||
source.kafka.max.poll.records=1000
|
||||
#消费者从单个分区中一次性获取的最大字节数
|
||||
source.kafka.max.partition.fetch.bytes=31457280
|
||||
source.kafka.enable.auto.commit=true
|
||||
#kafka SASL验证用户名
|
||||
source.kafka.user=olap
|
||||
#kafka SASL及SSL验证密码
|
||||
source.kafka.pin=galaxy2024
|
||||
#SSL需要
|
||||
source.kafka.tools.library=/opt/tsg/olap/topology/data/
|
||||
|
||||
map.filter.expression=FileChunk.offset <= 1073741824
|
||||
|
||||
#窗口相关配置
|
||||
combiner.window.parallelism=3
|
||||
#窗口大小，单位秒
|
||||
combiner.window.size=10
|
||||
|
||||
#sink相关参数
|
||||
sink.parallelism=3
|
||||
#可选hos、oss、hbase
|
||||
sink.type=hos
|
||||
sink.async=false
|
||||
|
||||
#hos sink相关配置
|
||||
#访问nginx或单个hos配置为ip:port；访问多个hos，配置为ip1:port,ip2:port...
|
||||
sink.hos.endpoint=192.168.44.11:8186,192.168.44.14:8186
|
||||
sink.hos.bucket=traffic_rtp_file_bucket
|
||||
sink.hos.token=c21f969b5f03d33d43e04f8f136e7682
|
||||
sink.hos.batch.size=1048576
|
||||
sink.hos.batch.interval.ms=10000
|
||||
|
||||
#http相关配置
|
||||
sink.http.client.retries.number=3
|
||||
sink.http.client.max.total=20
|
||||
sink.http.client.max.per.route=10
|
||||
sink.http.client.connect.timeout.ms=10000
|
||||
sink.http.client.request.timeout.ms=10000
|
||||
sink.http.client.socket.timeout.ms=60000
|
||||
14
file-chunk-combiner/cluster/env/agg_traffic_eml_file_chunk_combiner.sh
vendored
Normal file
14
file-chunk-combiner/cluster/env/agg_traffic_eml_file_chunk_combiner.sh
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
export MAIN_CLASS="com.zdjizhi.FileChunkCombiner"
|
||||
export PARALLELISM="1"
|
||||
export TASK_MODE="yarn-per-job"
|
||||
|
||||
export FLINK_JOB_OPTS="
|
||||
-Djobmanager.memory.process.size=1024m
|
||||
-Dtaskmanager.memory.process.size=1024m
|
||||
-Dtaskmanager.numberOfTaskSlots=1
|
||||
-Dtaskmanager.memory.framework.off-heap.size=256m
|
||||
-Dtaskmanager.memory.jvm-metaspace.size=128m
|
||||
"
|
||||
|
||||
14
file-chunk-combiner/cluster/env/agg_traffic_http_file_chunk_combiner.sh
vendored
Normal file
14
file-chunk-combiner/cluster/env/agg_traffic_http_file_chunk_combiner.sh
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
export MAIN_CLASS="com.zdjizhi.FileChunkCombiner"
|
||||
export PARALLELISM="3"
|
||||
export TASK_MODE="yarn-per-job"
|
||||
|
||||
export FLINK_JOB_OPTS="
|
||||
-Djobmanager.memory.process.size=1024m
|
||||
-Dtaskmanager.memory.process.size=2048m
|
||||
-Dtaskmanager.numberOfTaskSlots=3
|
||||
-Dtaskmanager.memory.framework.off-heap.size=256m
|
||||
-Dtaskmanager.memory.jvm-metaspace.size=128m
|
||||
"
|
||||
|
||||
14
file-chunk-combiner/cluster/env/agg_traffic_policy_capture_file_chunk_combiner.sh
vendored
Normal file
14
file-chunk-combiner/cluster/env/agg_traffic_policy_capture_file_chunk_combiner.sh
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
export MAIN_CLASS="com.zdjizhi.FileChunkCombiner"
|
||||
export PARALLELISM="3"
|
||||
export TASK_MODE="yarn-per-job"
|
||||
|
||||
export FLINK_JOB_OPTS="
|
||||
-Djobmanager.memory.process.size=1024m
|
||||
-Dtaskmanager.memory.process.size=2048m
|
||||
-Dtaskmanager.numberOfTaskSlots=3
|
||||
-Dtaskmanager.memory.framework.off-heap.size=256m
|
||||
-Dtaskmanager.memory.jvm-metaspace.size=128m
|
||||
"
|
||||
|
||||
14
file-chunk-combiner/cluster/env/agg_traffic_rtp_file_chunk_combiner.sh
vendored
Normal file
14
file-chunk-combiner/cluster/env/agg_traffic_rtp_file_chunk_combiner.sh
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
export MAIN_CLASS="com.zdjizhi.FileChunkCombiner"
|
||||
export PARALLELISM="3"
|
||||
export TASK_MODE="yarn-per-job"
|
||||
|
||||
export FLINK_JOB_OPTS="
|
||||
-Djobmanager.memory.process.size=1024m
|
||||
-Dtaskmanager.memory.process.size=2048m
|
||||
-Dtaskmanager.numberOfTaskSlots=3
|
||||
-Dtaskmanager.memory.framework.off-heap.size=256m
|
||||
-Dtaskmanager.memory.jvm-metaspace.size=128m
|
||||
"
|
||||
|
||||
Reference in New Issue
Block a user