整理groot任务模板格式

2024-10-28 10:57:34 +08:00
parent 3ec7b28d6a
commit c866701894
5 changed files with 95 additions and 12 deletions
--- a/tsg_olap/installation/configuration/tsg-olap.yml
+++ b/tsg_olap/installation/configuration/tsg-olap.yml
@@ -2,7 +2,7 @@ config_namespace: "prod"
  
 # Name of the data center
 data_center_name: xxg
-data_center_id_num: 2
+data_center_id_num: 1

 galaxy_qgw_service:
  # Running memory of the Galaxy-qgw-service.
@@ -87,7 +87,12 @@ default_init:
 flink:
  app-protocol-stat-traffic-merge:
    agg_app_protocol_traffic:
-      env: "{{ default_init.flink.env }}"
+      env:
+        parallelism: 1
+        taskmanager.memory.process.size: 2048m
+        taskmanager.memory.jvm-metaspace.size: 256m
+        taskmanager.numberOfTaskSlots: 1
+        taskmanager.memory.framework.off-heap.size: 512m
      config:
        template: agg_app_protocol_traffic

@@ -96,19 +101,47 @@ flink:
      env: "{{ default_init.flink.env }}"
      config:
        template: detection_dos_attack
-
+#---------------------------------------------file chunk
  file-chunk-combiner:
-    agg_traffic_file_chunk_combine:
+    agg_traffic_eml_file_chunk_combiner:
      env: "{{ default_init.flink.env }}"
      config:
-        template: agg_traffic_file_chunk_combine
+        template: agg_traffic_file_chunk_combiner
+        kafka_source_topic: TRAFFIC-EML-FILE-STREAM-RECORD
+        hos_sink_bucket: traffic_eml_file_bucket
+        combiner_window_parallelism: 1
+        hos_sink_parallelism: 1
+    agg_traffic_http_file_chunk_combiner:
+      env: "{{ default_init.flink.env }}"
+      config:
+        template: agg_traffic_file_chunk_combiner
+        kafka_source_topic: TRAFFIC-HTTP-FILE-STREAM-RECORD
+        hos_sink_bucket: traffic_http_file_bucket
+        combiner_window_parallelism: 1
+        hos_sink_parallelism: 1
+    agg_traffic_policy_capture_file_chunk_combiner:
+      env: "{{ default_init.flink.env }}"
+      config:
+        template: agg_traffic_file_chunk_combiner
+        kafka_source_topic: TRAFFIC-POLICY-CAPTURE-FILE-STREAM-RECORD
+        hos_sink_bucket: traffic_policy_capture_file_bucket
+        combiner_window_parallelism: 1
+        hos_sink_parallelism: 1
+    agg_traffic_rtp_file_chunk_combiner:
+      env: "{{ default_init.flink.env }}"
+      config:
+        template: agg_traffic_file_chunk_combiner
+        kafka_source_topic: TRAFFIC-RTP-FILE-STREAM-RECORD
+        hos_sink_bucket: traffic_rtp_file_bucket
+        combiner_window_parallelism: 1
+        hos_sink_parallelism: 1

  sip-rtp-correlation:
    correlation_sip_rtp_session:
      env: "{{ default_init.flink.env }}"
      config:
        template: correlation_sip_rtp_session
-
+#---------------------------------------------------------------grootstream
  groot-stream:
    dos_event_kafka_to_clickhouse:
      env: "{{ default_init.flink.env }}"
--- a/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/templates/agg_traffic_file_chunk_combiner
+++ b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/templates/agg_traffic_file_chunk_combiner
@@ -0,0 +1,50 @@
+flink.job.name={{ job_name }}
+
+#kafka source配置
+#9092为无验证 9095为ssl 9094为sasl
+source.kafka.broker={{ kafka_source_servers }}
+source.kafka.topic={{ kafka_source_topic }}
+source.kafka.group.id={{ kafka_source_group_id }}
+#earliest从头开始 latest最新
+source.kafka.auto.offset.reset=latest
+source.kafka.session.timeout.ms=60000
+#每次拉取操作从分区中获取的最大记录数
+source.kafka.max.poll.records=1000
+#消费者从单个分区中一次性获取的最大字节数
+source.kafka.max.partition.fetch.bytes=31457280
+source.kafka.enable.auto.commit=true
+#kafka SASL验证用户名
+source.kafka.user=olap
+#kafka SASL及SSL验证密码
+source.kafka.pin=galaxy2024
+#SSL需要
+source.kafka.tools.library={{ deploy_dir }}/flink/topology/data/
+
+map.filter.expression=FileChunk.offset <= 1073741824
+
+#窗口相关配置
+combiner.window.parallelism={{ combiner_window_parallelism }}
+#窗口大小，单位秒
+combiner.window.size=10
+
+#sink相关参数
+sink.parallelism={{ hos_sink_parallelism }}
+#可选hos、oss、hbase
+sink.type=hos
+sink.async=false
+
+#hos sink相关配置
+#访问nginx或单个hos配置为ip:port；访问多个hos，配置为ip1:port,ip2:port...
+sink.hos.endpoint={{ hos_sink_servers }}
+sink.hos.bucket={{ hos_sink_bucket }}
+sink.hos.token={{ hos_token }}
+sink.hos.batch.size=1048576
+sink.hos.batch.interval.ms=10000
+
+#http相关配置
+sink.http.client.retries.number=3
+sink.http.client.max.total=20
+sink.http.client.max.per.route=10
+sink.http.client.connect.timeout.ms=10000
+sink.http.client.request.timeout.ms=10000
+sink.http.client.socket.timeout.ms=60000
--- a/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/voip_record.yaml.j2
+++ b/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/voip_record.yaml.j2
@@ -148,4 +148,4 @@ application:
      hos.bucket.name.http_file: traffic_http_file_bucket
      hos.bucket.name.eml_file: traffic_eml_file_bucket
      hos.bucket.name.policy_capture_file: traffic_policy_capture_file_bucket
-  {{ topology }}
+  {{ topology }}
--- a/tsg_olap/upgrade/TSG-24.09/hos/hosutil/config.properties
+++ b/tsg_olap/upgrade/TSG-24.09/hos/hosutil/config.properties
@@ -1,7 +1,7 @@
-qgw.server.addr=http://192.168.44.67:9999
-hos.server.addr=http://192.168.44.67:9098
-hos.token=c21f969b5f03d33d43e04f8f136e7682
-kafka.server=192.168.44.11:9092
+qgw.serverAddr=http://{{ vrrp_instance.default.virtual_ipaddress }}:9999
+hos.serverAddr=http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098
+hos.token={{ hos_token }}
+kafka.server={{ groups.kafka[0] }}:9092
 #延迟时间，校验多少秒之前的文件，单位秒
 check.time.delay=180
 hos.traffic.buckets=traffic_policy_capture_file_bucket,traffic_rtp_file_bucket,traffic_http_file_bucket,traffic_eml_file_bucket
--- a/tsg_olap/upgrade/TSG-24.10/clickhouse/tsg_olap_clickhouse_ddl_upgrade_24.10.sql
+++ b/tsg_olap/upgrade/TSG-24.10/clickhouse/tsg_olap_clickhouse_ddl_upgrade_24.10.sql
@@ -1169,7 +1169,7 @@ WHERE empty(monitor_rule_uuid_list) = 0
 -- TSG-22703 clickhouse库表dos_event新增字段rule_uuid，source_country,source_ip 删除字段profile_id,修改*_rate字段类型

 ALTER table tsg_galaxy_v3.dos_event_local on cluster ck_cluster drop column IF EXISTS profile_id;
-ALTER table tsg_galaxy_v3.dos_event on cluster ck_cluster drop column IF EXISTS profile_id
+ALTER table tsg_galaxy_v3.dos_event on cluster ck_cluster drop column IF EXISTS profile_id;

 ALTER table tsg_galaxy_v3.dos_event_local on cluster ck_cluster add column IF NOT EXISTS rule_uuid String after rule_id;
 ALTER table tsg_galaxy_v3.dos_event on cluster ck_cluster add column IF NOT EXISTS rule_uuid String after rule_id;