diff --git a/tsg_olap/installation/configuration/tsg-olap.yml b/tsg_olap/installation/configuration/tsg-olap.yml index 5db88ed..cac1f03 100644 --- a/tsg_olap/installation/configuration/tsg-olap.yml +++ b/tsg_olap/installation/configuration/tsg-olap.yml @@ -2,7 +2,7 @@ config_namespace: "prod" # Name of the data center data_center_name: xxg -data_center_id_num: 2 +data_center_id_num: 1 galaxy_qgw_service: # Running memory of the Galaxy-qgw-service. @@ -87,7 +87,12 @@ default_init: flink: app-protocol-stat-traffic-merge: agg_app_protocol_traffic: - env: "{{ default_init.flink.env }}" + env: + parallelism: 1 + taskmanager.memory.process.size: 2048m + taskmanager.memory.jvm-metaspace.size: 256m + taskmanager.numberOfTaskSlots: 1 + taskmanager.memory.framework.off-heap.size: 512m config: template: agg_app_protocol_traffic @@ -96,19 +101,47 @@ flink: env: "{{ default_init.flink.env }}" config: template: detection_dos_attack - +#---------------------------------------------file chunk file-chunk-combiner: - agg_traffic_file_chunk_combine: + agg_traffic_eml_file_chunk_combiner: env: "{{ default_init.flink.env }}" config: - template: agg_traffic_file_chunk_combine + template: agg_traffic_file_chunk_combiner + kafka_source_topic: TRAFFIC-EML-FILE-STREAM-RECORD + hos_sink_bucket: traffic_eml_file_bucket + combiner_window_parallelism: 1 + hos_sink_parallelism: 1 + agg_traffic_http_file_chunk_combiner: + env: "{{ default_init.flink.env }}" + config: + template: agg_traffic_file_chunk_combiner + kafka_source_topic: TRAFFIC-HTTP-FILE-STREAM-RECORD + hos_sink_bucket: traffic_http_file_bucket + combiner_window_parallelism: 1 + hos_sink_parallelism: 1 + agg_traffic_policy_capture_file_chunk_combiner: + env: "{{ default_init.flink.env }}" + config: + template: agg_traffic_file_chunk_combiner + kafka_source_topic: TRAFFIC-POLICY-CAPTURE-FILE-STREAM-RECORD + hos_sink_bucket: traffic_policy_capture_file_bucket + combiner_window_parallelism: 1 + hos_sink_parallelism: 1 + agg_traffic_rtp_file_chunk_combiner: + env: "{{ default_init.flink.env }}" + config: + template: agg_traffic_file_chunk_combiner + kafka_source_topic: TRAFFIC-RTP-FILE-STREAM-RECORD + hos_sink_bucket: traffic_rtp_file_bucket + combiner_window_parallelism: 1 + hos_sink_parallelism: 1 sip-rtp-correlation: correlation_sip_rtp_session: env: "{{ default_init.flink.env }}" config: template: correlation_sip_rtp_session - +#---------------------------------------------------------------grootstream groot-stream: dos_event_kafka_to_clickhouse: env: "{{ default_init.flink.env }}" diff --git a/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/templates/agg_traffic_file_chunk_combiner b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/templates/agg_traffic_file_chunk_combiner new file mode 100644 index 0000000..0b2e5ef --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/templates/agg_traffic_file_chunk_combiner @@ -0,0 +1,50 @@ +flink.job.name={{ job_name }} + +#kafka source配置 +#9092为无验证 9095为ssl 9094为sasl +source.kafka.broker={{ kafka_source_servers }} +source.kafka.topic={{ kafka_source_topic }} +source.kafka.group.id={{ kafka_source_group_id }} +#earliest从头开始 latest最新 +source.kafka.auto.offset.reset=latest +source.kafka.session.timeout.ms=60000 +#每次拉取操作从分区中获取的最大记录数 +source.kafka.max.poll.records=1000 +#消费者从单个分区中一次性获取的最大字节数 +source.kafka.max.partition.fetch.bytes=31457280 +source.kafka.enable.auto.commit=true +#kafka SASL验证用户名 +source.kafka.user=olap +#kafka SASL及SSL验证密码 +source.kafka.pin=galaxy2024 +#SSL需要 +source.kafka.tools.library={{ deploy_dir }}/flink/topology/data/ + +map.filter.expression=FileChunk.offset <= 1073741824 + +#窗口相关配置 +combiner.window.parallelism={{ combiner_window_parallelism }} +#窗口大小，单位秒 +combiner.window.size=10 + +#sink相关参数 +sink.parallelism={{ hos_sink_parallelism }} +#可选hos、oss、hbase +sink.type=hos +sink.async=false + +#hos sink相关配置 +#访问nginx或单个hos配置为ip:port；访问多个hos，配置为ip1:port,ip2:port... +sink.hos.endpoint={{ hos_sink_servers }} +sink.hos.bucket={{ hos_sink_bucket }} +sink.hos.token={{ hos_token }} +sink.hos.batch.size=1048576 +sink.hos.batch.interval.ms=10000 + +#http相关配置 +sink.http.client.retries.number=3 +sink.http.client.max.total=20 +sink.http.client.max.per.route=10 +sink.http.client.connect.timeout.ms=10000 +sink.http.client.request.timeout.ms=10000 +sink.http.client.socket.timeout.ms=60000 \ No newline at end of file diff --git a/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/voip_record.yaml.j2 b/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/voip_record.yaml.j2 index 17fb5b0..d36029b 100644 --- a/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/voip_record.yaml.j2 +++ b/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/voip_record.yaml.j2 @@ -148,4 +148,4 @@ application: hos.bucket.name.http_file: traffic_http_file_bucket hos.bucket.name.eml_file: traffic_eml_file_bucket hos.bucket.name.policy_capture_file: traffic_policy_capture_file_bucket - {{ topology }} \ No newline at end of file + {{ topology }} diff --git a/tsg_olap/upgrade/TSG-24.09/hos/hosutil/config.properties b/tsg_olap/upgrade/TSG-24.09/hos/hosutil/config.properties index 5cb5164..fc486bf 100644 --- a/tsg_olap/upgrade/TSG-24.09/hos/hosutil/config.properties +++ b/tsg_olap/upgrade/TSG-24.09/hos/hosutil/config.properties @@ -1,7 +1,7 @@ -qgw.server.addr=http://192.168.44.67:9999 -hos.server.addr=http://192.168.44.67:9098 -hos.token=c21f969b5f03d33d43e04f8f136e7682 -kafka.server=192.168.44.11:9092 +qgw.serverAddr=http://{{ vrrp_instance.default.virtual_ipaddress }}:9999 +hos.serverAddr=http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098 +hos.token={{ hos_token }} +kafka.server={{ groups.kafka[0] }}:9092 #延迟时间，校验多少秒之前的文件，单位秒 check.time.delay=180 hos.traffic.buckets=traffic_policy_capture_file_bucket,traffic_rtp_file_bucket,traffic_http_file_bucket,traffic_eml_file_bucket diff --git a/tsg_olap/upgrade/TSG-24.10/clickhouse/tsg_olap_clickhouse_ddl_upgrade_24.10.sql b/tsg_olap/upgrade/TSG-24.10/clickhouse/tsg_olap_clickhouse_ddl_upgrade_24.10.sql index af7d013..72fb125 100644 --- a/tsg_olap/upgrade/TSG-24.10/clickhouse/tsg_olap_clickhouse_ddl_upgrade_24.10.sql +++ b/tsg_olap/upgrade/TSG-24.10/clickhouse/tsg_olap_clickhouse_ddl_upgrade_24.10.sql @@ -1169,7 +1169,7 @@ WHERE empty(monitor_rule_uuid_list) = 0 -- TSG-22703 clickhouse库表dos_event新增字段rule_uuid，source_country,source_ip 删除字段profile_id,修改*_rate字段类型 ALTER table tsg_galaxy_v3.dos_event_local on cluster ck_cluster drop column IF EXISTS profile_id; -ALTER table tsg_galaxy_v3.dos_event on cluster ck_cluster drop column IF EXISTS profile_id +ALTER table tsg_galaxy_v3.dos_event on cluster ck_cluster drop column IF EXISTS profile_id; ALTER table tsg_galaxy_v3.dos_event_local on cluster ck_cluster add column IF NOT EXISTS rule_uuid String after rule_id; ALTER table tsg_galaxy_v3.dos_event on cluster ck_cluster add column IF NOT EXISTS rule_uuid String after rule_id;