diff --git a/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/config/agg_traffic_eml_file_chunk_combiner b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/config/agg_traffic_eml_file_chunk_combiner new file mode 100644 index 0000000..7dcd279 --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/config/agg_traffic_eml_file_chunk_combiner @@ -0,0 +1,50 @@ +flink.job.name=agg_traffic_eml_file_chunk_combiner + +#kafka source配置 +#9092为无验证 9095为ssl 9094为sasl +source.kafka.broker=192.168.44.12:9092 +source.kafka.topic=TRAFFIC-EML-FILE-STREAM-RECORD +source.kafka.group.id=agg_traffic_eml_file_chunk_combiner_1 +#earliest从头开始 latest最新 +source.kafka.auto.offset.reset=latest +source.kafka.session.timeout.ms=60000 +#每次拉取操作从分区中获取的最大记录数 +source.kafka.max.poll.records=1000 +#消费者从单个分区中一次性获取的最大字节数 +source.kafka.max.partition.fetch.bytes=31457280 +source.kafka.enable.auto.commit=true +#kafka SASL验证用户名 +source.kafka.user=olap +#kafka SASL及SSL验证密码 +source.kafka.pin=galaxy2024 +#SSL需要 +source.kafka.tools.library=/opt/tsg/olap/topology/data/ + +map.filter.expression=FileChunk.offset <= 1073741824 + +#窗口相关配置 +combiner.window.parallelism=1 +#窗口大小,单位秒 +combiner.window.size=10 + +#sink相关参数 +sink.parallelism=1 +#可选hos、oss、hbase +sink.type=hos +sink.async=false + +#hos sink相关配置 +#访问nginx或单个hos配置为ip:port;访问多个hos,配置为ip1:port,ip2:port... +sink.hos.endpoint=192.168.44.12:8186 +sink.hos.bucket=traffic_eml_file_bucket +sink.hos.token=c21f969b5f03d33d43e04f8f136e7682 +sink.hos.batch.size=1048576 +sink.hos.batch.interval.ms=10000 + +#http相关配置 +sink.http.client.retries.number=3 +sink.http.client.max.total=20 +sink.http.client.max.per.route=10 +sink.http.client.connect.timeout.ms=10000 +sink.http.client.request.timeout.ms=10000 +sink.http.client.socket.timeout.ms=60000 \ No newline at end of file diff --git a/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/config/agg_traffic_http_file_chunk_combiner b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/config/agg_traffic_http_file_chunk_combiner new file mode 100644 index 0000000..b1a4daf --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/config/agg_traffic_http_file_chunk_combiner @@ -0,0 +1,50 @@ +flink.job.name=agg_traffic_http_file_chunk_combiner + +#kafka source配置 +#9092为无验证 9095为ssl 9094为sasl +source.kafka.broker=192.168.44.12:9092 +source.kafka.topic=TRAFFIC-HTTP-FILE-STREAM-RECORD +source.kafka.group.id=agg_traffic_http_file_chunk_combiner_1 +#earliest从头开始 latest最新 +source.kafka.auto.offset.reset=latest +source.kafka.session.timeout.ms=60000 +#每次拉取操作从分区中获取的最大记录数 +source.kafka.max.poll.records=1000 +#消费者从单个分区中一次性获取的最大字节数 +source.kafka.max.partition.fetch.bytes=31457280 +source.kafka.enable.auto.commit=true +#kafka SASL验证用户名 +source.kafka.user=olap +#kafka SASL及SSL验证密码 +source.kafka.pin=galaxy2024 +#SSL需要 +source.kafka.tools.library=/opt/tsg/olap/topology/data/ + +map.filter.expression=FileChunk.offset <= 1073741824 + +#窗口相关配置 +combiner.window.parallelism=3 +#窗口大小,单位秒 +combiner.window.size=10 + +#sink相关参数 +sink.parallelism=3 +#可选hos、oss、hbase +sink.type=hos +sink.async=false + +#hos sink相关配置 +#访问nginx或单个hos配置为ip:port;访问多个hos,配置为ip1:port,ip2:port... +sink.hos.endpoint=192.168.44.12:8186 +sink.hos.bucket=traffic_http_file_bucket +sink.hos.token=c21f969b5f03d33d43e04f8f136e7682 +sink.hos.batch.size=1048576 +sink.hos.batch.interval.ms=10000 + +#http相关配置 +sink.http.client.retries.number=3 +sink.http.client.max.total=20 +sink.http.client.max.per.route=10 +sink.http.client.connect.timeout.ms=10000 +sink.http.client.request.timeout.ms=10000 +sink.http.client.socket.timeout.ms=60000 \ No newline at end of file diff --git a/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/config/agg_traffic_policy_capture_file_chunk_combiner b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/config/agg_traffic_policy_capture_file_chunk_combiner new file mode 100644 index 0000000..11c9aef --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/config/agg_traffic_policy_capture_file_chunk_combiner @@ -0,0 +1,50 @@ +flink.job.name=agg_traffic_policy_capture_file_chunk_combiner + +#kafka source配置 +#9092为无验证 9095为ssl 9094为sasl +source.kafka.broker=192.168.44.12:9092 +source.kafka.topic=TRAFFIC-POLICY-CAPTURE-FILE-STREAM-RECORD +source.kafka.group.id=agg_traffic_policy_capture_file_chunk_combiner_1 +#earliest从头开始 latest最新 +source.kafka.auto.offset.reset=latest +source.kafka.session.timeout.ms=60000 +#每次拉取操作从分区中获取的最大记录数 +source.kafka.max.poll.records=1000 +#消费者从单个分区中一次性获取的最大字节数 +source.kafka.max.partition.fetch.bytes=31457280 +source.kafka.enable.auto.commit=true +#kafka SASL验证用户名 +source.kafka.user=olap +#kafka SASL及SSL验证密码 +source.kafka.pin=galaxy2024 +#SSL需要 +source.kafka.tools.library=/opt/tsg/olap/topology/data/ + +map.filter.expression=FileChunk.offset <= 1073741824 + +#窗口相关配置 +combiner.window.parallelism=3 +#窗口大小,单位秒 +combiner.window.size=10 + +#sink相关参数 +sink.parallelism=3 +#可选hos、oss、hbase +sink.type=hos +sink.async=false + +#hos sink相关配置 +#访问nginx或单个hos配置为ip:port;访问多个hos,配置为ip1:port,ip2:port... +sink.hos.endpoint=192.168.44.12:8186 +sink.hos.bucket=traffic_policy_capture_file_bucket +sink.hos.token=c21f969b5f03d33d43e04f8f136e7682 +sink.hos.batch.size=1048576 +sink.hos.batch.interval.ms=10000 + +#http相关配置 +sink.http.client.retries.number=3 +sink.http.client.max.total=20 +sink.http.client.max.per.route=10 +sink.http.client.connect.timeout.ms=10000 +sink.http.client.request.timeout.ms=10000 +sink.http.client.socket.timeout.ms=60000 \ No newline at end of file diff --git a/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/config/agg_traffic_rtp_file_chunk_combiner b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/config/agg_traffic_rtp_file_chunk_combiner new file mode 100644 index 0000000..58e7774 --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/config/agg_traffic_rtp_file_chunk_combiner @@ -0,0 +1,50 @@ +flink.job.name=agg_traffic_rtp_file_chunk_combiner + +#kafka source配置 +#9092为无验证 9095为ssl 9094为sasl +source.kafka.broker=192.168.44.12:9092 +source.kafka.topic=TRAFFIC-RTP-FILE-STREAM-RECORD +source.kafka.group.id=agg_traffic_rtp_file_chunk_combiner_1 +#earliest从头开始 latest最新 +source.kafka.auto.offset.reset=latest +source.kafka.session.timeout.ms=60000 +#每次拉取操作从分区中获取的最大记录数 +source.kafka.max.poll.records=1000 +#消费者从单个分区中一次性获取的最大字节数 +source.kafka.max.partition.fetch.bytes=31457280 +source.kafka.enable.auto.commit=true +#kafka SASL验证用户名 +source.kafka.user=olap +#kafka SASL及SSL验证密码 +source.kafka.pin=galaxy2024 +#SSL需要 +source.kafka.tools.library=/opt/tsg/olap/topology/data/ + +map.filter.expression=FileChunk.offset <= 1073741824 + +#窗口相关配置 +combiner.window.parallelism=3 +#窗口大小,单位秒 +combiner.window.size=10 + +#sink相关参数 +sink.parallelism=3 +#可选hos、oss、hbase +sink.type=hos +sink.async=false + +#hos sink相关配置 +#访问nginx或单个hos配置为ip:port;访问多个hos,配置为ip1:port,ip2:port... +sink.hos.endpoint=192.168.44.12:8186 +sink.hos.bucket=traffic_rtp_file_bucket +sink.hos.token=c21f969b5f03d33d43e04f8f136e7682 +sink.hos.batch.size=1048576 +sink.hos.batch.interval.ms=10000 + +#http相关配置 +sink.http.client.retries.number=3 +sink.http.client.max.total=20 +sink.http.client.max.per.route=10 +sink.http.client.connect.timeout.ms=10000 +sink.http.client.request.timeout.ms=10000 +sink.http.client.socket.timeout.ms=60000 \ No newline at end of file diff --git a/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/env/agg_traffic_eml_file_chunk_combiner.sh b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/env/agg_traffic_eml_file_chunk_combiner.sh new file mode 100644 index 0000000..a60f9fc --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/env/agg_traffic_eml_file_chunk_combiner.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +export MAIN_CLASS="com.zdjizhi.FileChunkCombiner" +export PARALLELISM="1" +export TASK_MODE="yarn-session" + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +-Dtaskmanager.memory.process.size=1024m +-Dtaskmanager.numberOfTaskSlots=1 +-Dtaskmanager.memory.framework.off-heap.size=256m +-Dtaskmanager.memory.jvm-metaspace.size=128m +" + diff --git a/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/env/agg_traffic_http_file_chunk_combiner.sh b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/env/agg_traffic_http_file_chunk_combiner.sh new file mode 100644 index 0000000..ff68f57 --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/env/agg_traffic_http_file_chunk_combiner.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +export MAIN_CLASS="com.zdjizhi.FileChunkCombiner" +export PARALLELISM="3" +export TASK_MODE="yarn-session" + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +-Dtaskmanager.memory.process.size=2048m +-Dtaskmanager.numberOfTaskSlots=3 +-Dtaskmanager.memory.framework.off-heap.size=256m +-Dtaskmanager.memory.jvm-metaspace.size=128m +" + diff --git a/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/env/agg_traffic_policy_capture_file_chunk_combiner.sh b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/env/agg_traffic_policy_capture_file_chunk_combiner.sh new file mode 100644 index 0000000..ff68f57 --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/env/agg_traffic_policy_capture_file_chunk_combiner.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +export MAIN_CLASS="com.zdjizhi.FileChunkCombiner" +export PARALLELISM="3" +export TASK_MODE="yarn-session" + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +-Dtaskmanager.memory.process.size=2048m +-Dtaskmanager.numberOfTaskSlots=3 +-Dtaskmanager.memory.framework.off-heap.size=256m +-Dtaskmanager.memory.jvm-metaspace.size=128m +" + diff --git a/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/env/agg_traffic_rtp_file_chunk_combiner.sh b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/env/agg_traffic_rtp_file_chunk_combiner.sh new file mode 100644 index 0000000..ff68f57 --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/env/agg_traffic_rtp_file_chunk_combiner.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +export MAIN_CLASS="com.zdjizhi.FileChunkCombiner" +export PARALLELISM="3" +export TASK_MODE="yarn-session" + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +-Dtaskmanager.memory.process.size=2048m +-Dtaskmanager.numberOfTaskSlots=3 +-Dtaskmanager.memory.framework.off-heap.size=256m +-Dtaskmanager.memory.jvm-metaspace.size=128m +" + diff --git a/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/config/agg_traffic_eml_file_chunk_combiner b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/config/agg_traffic_eml_file_chunk_combiner new file mode 100644 index 0000000..05053c4 --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/config/agg_traffic_eml_file_chunk_combiner @@ -0,0 +1,50 @@ +flink.job.name=agg_traffic_eml_file_chunk_combiner + +#kafka source配置 +#9092为无验证 9095为ssl 9094为sasl +source.kafka.broker=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094 +source.kafka.topic=TRAFFIC-EML-FILE-STREAM-RECORD +source.kafka.group.id=agg_traffic_eml_file_chunk_combiner_1 +#earliest从头开始 latest最新 +source.kafka.auto.offset.reset=latest +source.kafka.session.timeout.ms=60000 +#每次拉取操作从分区中获取的最大记录数 +source.kafka.max.poll.records=1000 +#消费者从单个分区中一次性获取的最大字节数 +source.kafka.max.partition.fetch.bytes=31457280 +source.kafka.enable.auto.commit=true +#kafka SASL验证用户名 +source.kafka.user=olap +#kafka SASL及SSL验证密码 +source.kafka.pin=galaxy2024 +#SSL需要 +source.kafka.tools.library=/opt/tsg/olap/topology/data/ + +map.filter.expression=FileChunk.offset <= 1073741824 + +#窗口相关配置 +combiner.window.parallelism=1 +#窗口大小,单位秒 +combiner.window.size=10 + +#sink相关参数 +sink.parallelism=1 +#可选hos、oss、hbase +sink.type=hos +sink.async=false + +#hos sink相关配置 +#访问nginx或单个hos配置为ip:port;访问多个hos,配置为ip1:port,ip2:port... +sink.hos.endpoint=192.168.44.11:8186,192.168.44.14:8186 +sink.hos.bucket=traffic_eml_file_bucket +sink.hos.token=c21f969b5f03d33d43e04f8f136e7682 +sink.hos.batch.size=1048576 +sink.hos.batch.interval.ms=10000 + +#http相关配置 +sink.http.client.retries.number=3 +sink.http.client.max.total=20 +sink.http.client.max.per.route=10 +sink.http.client.connect.timeout.ms=10000 +sink.http.client.request.timeout.ms=10000 +sink.http.client.socket.timeout.ms=60000 \ No newline at end of file diff --git a/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/config/agg_traffic_http_file_chunk_combiner b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/config/agg_traffic_http_file_chunk_combiner new file mode 100644 index 0000000..fdd496a --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/config/agg_traffic_http_file_chunk_combiner @@ -0,0 +1,50 @@ +flink.job.name=agg_traffic_http_file_chunk_combiner + +#kafka source配置 +#9092为无验证 9095为ssl 9094为sasl +source.kafka.broker=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094 +source.kafka.topic=TRAFFIC-HTTP-FILE-STREAM-RECORD +source.kafka.group.id=agg_traffic_http_file_chunk_combiner_1 +#earliest从头开始 latest最新 +source.kafka.auto.offset.reset=latest +source.kafka.session.timeout.ms=60000 +#每次拉取操作从分区中获取的最大记录数 +source.kafka.max.poll.records=1000 +#消费者从单个分区中一次性获取的最大字节数 +source.kafka.max.partition.fetch.bytes=31457280 +source.kafka.enable.auto.commit=true +#kafka SASL验证用户名 +source.kafka.user=olap +#kafka SASL及SSL验证密码 +source.kafka.pin=galaxy2024 +#SSL需要 +source.kafka.tools.library=/opt/tsg/olap/topology/data/ + +map.filter.expression=FileChunk.offset <= 1073741824 + +#窗口相关配置 +combiner.window.parallelism=3 +#窗口大小,单位秒 +combiner.window.size=10 + +#sink相关参数 +sink.parallelism=3 +#可选hos、oss、hbase +sink.type=hos +sink.async=false + +#hos sink相关配置 +#访问nginx或单个hos配置为ip:port;访问多个hos,配置为ip1:port,ip2:port... +sink.hos.endpoint=192.168.44.11:8186,192.168.44.14:8186 +sink.hos.bucket=traffic_http_file_bucket +sink.hos.token=c21f969b5f03d33d43e04f8f136e7682 +sink.hos.batch.size=1048576 +sink.hos.batch.interval.ms=10000 + +#http相关配置 +sink.http.client.retries.number=3 +sink.http.client.max.total=20 +sink.http.client.max.per.route=10 +sink.http.client.connect.timeout.ms=10000 +sink.http.client.request.timeout.ms=10000 +sink.http.client.socket.timeout.ms=60000 \ No newline at end of file diff --git a/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/config/agg_traffic_policy_capture_file_chunk_combiner b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/config/agg_traffic_policy_capture_file_chunk_combiner new file mode 100644 index 0000000..98cfc68 --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/config/agg_traffic_policy_capture_file_chunk_combiner @@ -0,0 +1,50 @@ +flink.job.name=agg_traffic_policy_capture_file_chunk_combiner + +#kafka source配置 +#9092为无验证 9095为ssl 9094为sasl +source.kafka.broker=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094 +source.kafka.topic=TRAFFIC-POLICY-CAPTURE-FILE-STREAM-RECORD +source.kafka.group.id=agg_traffic_policy_capture_file_chunk_combiner_1 +#earliest从头开始 latest最新 +source.kafka.auto.offset.reset=latest +source.kafka.session.timeout.ms=60000 +#每次拉取操作从分区中获取的最大记录数 +source.kafka.max.poll.records=1000 +#消费者从单个分区中一次性获取的最大字节数 +source.kafka.max.partition.fetch.bytes=31457280 +source.kafka.enable.auto.commit=true +#kafka SASL验证用户名 +source.kafka.user=olap +#kafka SASL及SSL验证密码 +source.kafka.pin=galaxy2024 +#SSL需要 +source.kafka.tools.library=/opt/tsg/olap/topology/data/ + +map.filter.expression=FileChunk.offset <= 1073741824 + +#窗口相关配置 +combiner.window.parallelism=3 +#窗口大小,单位秒 +combiner.window.size=10 + +#sink相关参数 +sink.parallelism=3 +#可选hos、oss、hbase +sink.type=hos +sink.async=false + +#hos sink相关配置 +#访问nginx或单个hos配置为ip:port;访问多个hos,配置为ip1:port,ip2:port... +sink.hos.endpoint=192.168.44.11:8186,192.168.44.14:8186 +sink.hos.bucket=traffic_policy_capture_file_bucket +sink.hos.token=c21f969b5f03d33d43e04f8f136e7682 +sink.hos.batch.size=1048576 +sink.hos.batch.interval.ms=10000 + +#http相关配置 +sink.http.client.retries.number=3 +sink.http.client.max.total=20 +sink.http.client.max.per.route=10 +sink.http.client.connect.timeout.ms=10000 +sink.http.client.request.timeout.ms=10000 +sink.http.client.socket.timeout.ms=60000 \ No newline at end of file diff --git a/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/config/agg_traffic_rtp_file_chunk_combiner b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/config/agg_traffic_rtp_file_chunk_combiner new file mode 100644 index 0000000..b63382c --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/config/agg_traffic_rtp_file_chunk_combiner @@ -0,0 +1,50 @@ +flink.job.name=agg_traffic_rtp_file_chunk_combiner + +#kafka source配置 +#9092为无验证 9095为ssl 9094为sasl +source.kafka.broker=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094 +source.kafka.topic=TRAFFIC-RTP-FILE-STREAM-RECORD +source.kafka.group.id=agg_traffic_rtp_file_chunk_combiner_1 +#earliest从头开始 latest最新 +source.kafka.auto.offset.reset=latest +source.kafka.session.timeout.ms=60000 +#每次拉取操作从分区中获取的最大记录数 +source.kafka.max.poll.records=1000 +#消费者从单个分区中一次性获取的最大字节数 +source.kafka.max.partition.fetch.bytes=31457280 +source.kafka.enable.auto.commit=true +#kafka SASL验证用户名 +source.kafka.user=olap +#kafka SASL及SSL验证密码 +source.kafka.pin=galaxy2024 +#SSL需要 +source.kafka.tools.library=/opt/tsg/olap/topology/data/ + +map.filter.expression=FileChunk.offset <= 1073741824 + +#窗口相关配置 +combiner.window.parallelism=3 +#窗口大小,单位秒 +combiner.window.size=10 + +#sink相关参数 +sink.parallelism=3 +#可选hos、oss、hbase +sink.type=hos +sink.async=false + +#hos sink相关配置 +#访问nginx或单个hos配置为ip:port;访问多个hos,配置为ip1:port,ip2:port... +sink.hos.endpoint=192.168.44.11:8186,192.168.44.14:8186 +sink.hos.bucket=traffic_rtp_file_bucket +sink.hos.token=c21f969b5f03d33d43e04f8f136e7682 +sink.hos.batch.size=1048576 +sink.hos.batch.interval.ms=10000 + +#http相关配置 +sink.http.client.retries.number=3 +sink.http.client.max.total=20 +sink.http.client.max.per.route=10 +sink.http.client.connect.timeout.ms=10000 +sink.http.client.request.timeout.ms=10000 +sink.http.client.socket.timeout.ms=60000 \ No newline at end of file diff --git a/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/env/agg_traffic_eml_file_chunk_combiner.sh b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/env/agg_traffic_eml_file_chunk_combiner.sh new file mode 100644 index 0000000..d4913fc --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/env/agg_traffic_eml_file_chunk_combiner.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +export MAIN_CLASS="com.zdjizhi.FileChunkCombiner" +export PARALLELISM="1" +export TASK_MODE="yarn-per-job" + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +-Dtaskmanager.memory.process.size=1024m +-Dtaskmanager.numberOfTaskSlots=1 +-Dtaskmanager.memory.framework.off-heap.size=256m +-Dtaskmanager.memory.jvm-metaspace.size=128m +" + diff --git a/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/env/agg_traffic_http_file_chunk_combiner.sh b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/env/agg_traffic_http_file_chunk_combiner.sh new file mode 100644 index 0000000..c01287f --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/env/agg_traffic_http_file_chunk_combiner.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +export MAIN_CLASS="com.zdjizhi.FileChunkCombiner" +export PARALLELISM="3" +export TASK_MODE="yarn-per-job" + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +-Dtaskmanager.memory.process.size=2048m +-Dtaskmanager.numberOfTaskSlots=3 +-Dtaskmanager.memory.framework.off-heap.size=256m +-Dtaskmanager.memory.jvm-metaspace.size=128m +" + diff --git a/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/env/agg_traffic_policy_capture_file_chunk_combiner.sh b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/env/agg_traffic_policy_capture_file_chunk_combiner.sh new file mode 100644 index 0000000..c01287f --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/env/agg_traffic_policy_capture_file_chunk_combiner.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +export MAIN_CLASS="com.zdjizhi.FileChunkCombiner" +export PARALLELISM="3" +export TASK_MODE="yarn-per-job" + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +-Dtaskmanager.memory.process.size=2048m +-Dtaskmanager.numberOfTaskSlots=3 +-Dtaskmanager.memory.framework.off-heap.size=256m +-Dtaskmanager.memory.jvm-metaspace.size=128m +" + diff --git a/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/env/agg_traffic_rtp_file_chunk_combiner.sh b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/env/agg_traffic_rtp_file_chunk_combiner.sh new file mode 100644 index 0000000..c01287f --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/env/agg_traffic_rtp_file_chunk_combiner.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +export MAIN_CLASS="com.zdjizhi.FileChunkCombiner" +export PARALLELISM="3" +export TASK_MODE="yarn-per-job" + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +-Dtaskmanager.memory.process.size=2048m +-Dtaskmanager.numberOfTaskSlots=3 +-Dtaskmanager.memory.framework.off-heap.size=256m +-Dtaskmanager.memory.jvm-metaspace.size=128m +" + diff --git a/tsg_olap/upgrade/TSG-24.09/hbase/update_hbase_24.09.sh b/tsg_olap/upgrade/TSG-24.09/hbase/update_hbase_24.09.sh index 706f127..54bfaae 100644 --- a/tsg_olap/upgrade/TSG-24.09/hbase/update_hbase_24.09.sh +++ b/tsg_olap/upgrade/TSG-24.09/hbase/update_hbase_24.09.sh @@ -1,19 +1,11 @@ +docker exec -i HMaster hbase shell < 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} alter 'index_time_troubleshooting_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} alter 'index_filename_troubleshooting_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} alter 'index_partfile_troubleshooting_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} - alter 'assessment_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} alter 'index_time_assessment_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} alter 'index_filename_assessment_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} alter 'index_partfile_assessment_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} -alter 'knowledge_base_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}} -alter 'index_time_knowledge_base_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}} -alter 'index_filename_knowledge_base_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}} -alter 'index_partfile_knowledge_base_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}} - -alter 'report_snapshot_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}} -alter 'index_time_report_snapshot_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}} -alter 'index_filename_report_snapshot_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}} -alter 'index_partfile_report_snapshot_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}} \ No newline at end of file +EOF diff --git a/tsg_olap/upgrade/TSG-24.09/hos/create_bucket_24.09.sh b/tsg_olap/upgrade/TSG-24.09/hos/create_bucket_24.09.sh index ad37488..0473632 100644 --- a/tsg_olap/upgrade/TSG-24.09/hos/create_bucket_24.09.sh +++ b/tsg_olap/upgrade/TSG-24.09/hos/create_bucket_24.09.sh @@ -4,5 +4,5 @@ curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_ht curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_eml_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/troubleshooting_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/assessment_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' -curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/knowledge_base_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-replication:2' -curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/report_snapshot_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-replication:2' \ No newline at end of file +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/knowledge_base_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-replication:单机为1集群为2' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/report_snapshot_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-replication:单机为1集群为2' \ No newline at end of file