From 4c693527cc6b843b24d2a442089d70e4295c89ea Mon Sep 17 00:00:00 2001 From: gujinkai Date: Fri, 8 Nov 2024 19:01:54 +0800 Subject: [PATCH] CN 24.08.1 change groot config --- .../groot-stream/1.3.2/24.08.1/README.md | 4 + .../1.3.2/24.08.1/grootstream.yaml | 94 +++++++++++++++++++ .../groot-stream/1.3.2/24.08.1/udf.plugins | 26 +++++ .../1.7.0/24.08.1/sd_kafka_to_cn_clickhouse | 2 +- 4 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 cyber_narrator/upgrade/2024/CN-24.08/groot-stream/1.3.2/24.08.1/README.md create mode 100644 cyber_narrator/upgrade/2024/CN-24.08/groot-stream/1.3.2/24.08.1/grootstream.yaml create mode 100644 cyber_narrator/upgrade/2024/CN-24.08/groot-stream/1.3.2/24.08.1/udf.plugins diff --git a/cyber_narrator/upgrade/2024/CN-24.08/groot-stream/1.3.2/24.08.1/README.md b/cyber_narrator/upgrade/2024/CN-24.08/groot-stream/1.3.2/24.08.1/README.md new file mode 100644 index 0000000..7e42bea --- /dev/null +++ b/cyber_narrator/upgrade/2024/CN-24.08/groot-stream/1.3.2/24.08.1/README.md @@ -0,0 +1,4 @@ + +groot-stream version base 1.3.2 + +etl_session_record_kafka_to_cn_kafka 需要根据部署环境确定数据源的topic是SESSION-RECORD还是SESSION-RECORD-PROCESSED \ No newline at end of file diff --git a/cyber_narrator/upgrade/2024/CN-24.08/groot-stream/1.3.2/24.08.1/grootstream.yaml b/cyber_narrator/upgrade/2024/CN-24.08/groot-stream/1.3.2/24.08.1/grootstream.yaml new file mode 100644 index 0000000..54acfc5 --- /dev/null +++ b/cyber_narrator/upgrade/2024/CN-24.08/groot-stream/1.3.2/24.08.1/grootstream.yaml @@ -0,0 +1,94 @@ +grootstream: + knowledge_base: + - name: cn_ip_location + fs_type: http + fs_path: http://192.168.44.55:9999/v1/knowledge_base + files: + - 1 + + - name: cn_ip_asn + fs_type: http + fs_path: http://192.168.44.55:9999/v1/knowledge_base + files: + - 2 + + - name: cn_idc_renter + fs_type: http + fs_path: http://192.168.44.55:9999/v1/knowledge_base + files: + - 11 + + - name: cn_link_direction + fs_type: http + fs_path: http://192.168.44.55:9999/v1/knowledge_base + files: + - 13 + + - name: cn_fqdn_category + fs_type: http + fs_path: http://192.168.44.55:9999/v1/knowledge_base + files: + - 5 + + - name: cn_fqdn_icp + fs_type: http + fs_path: http://192.168.44.55:9999/v1/knowledge_base + files: + - 4 + + - name: cn_fqdn_whois + fs_type: http + fs_path: http://192.168.44.55:9999/v1/knowledge_base + files: + - 6 + + - name: cn_dns_server + fs_type: http + fs_path: http://192.168.44.55:9999/v1/knowledge_base + files: + - 3 + + - name: cn_app_category + fs_type: http + fs_path: http://192.168.44.55:9999/v1/knowledge_base + files: + - 9 + + - name: cn_internal_ip + fs_type: http + fs_path: http://192.168.44.55:9999/v1/knowledge_base + files: + - 12 + + - name: cn_ioc_darkweb + fs_type: http + fs_path: http://192.168.44.55:9999/v1/knowledge_base + files: + - 8 + + - name: cn_ioc_malware + fs_type: http + fs_path: http://192.168.44.55:9999/v1/knowledge_base + files: + - 7 + + - name: cn_intelligence_indicator + fs_type: http + fs_path: http://192.168.44.55:9999/v1/knowledge_base + files: + - 16 + + - name: base_station_location + fs_type: local + fs_path: /data/cn/olap/flink/topology/groot-stream/knowledge/ + files: + - base_station_location.csv + + - name: cn_rule + fs_type: http + fs_path: http://192.168.44.54:8090 + properties: + token: 1a653ea0-d39b-4246-94b0-1ba95db4b6a7 + + properties: + scheduler.knowledge_base.update.interval.minutes: 5 \ No newline at end of file diff --git a/cyber_narrator/upgrade/2024/CN-24.08/groot-stream/1.3.2/24.08.1/udf.plugins b/cyber_narrator/upgrade/2024/CN-24.08/groot-stream/1.3.2/24.08.1/udf.plugins new file mode 100644 index 0000000..9508d08 --- /dev/null +++ b/cyber_narrator/upgrade/2024/CN-24.08/groot-stream/1.3.2/24.08.1/udf.plugins @@ -0,0 +1,26 @@ +com.geedgenetworks.core.udf.AsnLookup +com.geedgenetworks.core.udf.Eval +com.geedgenetworks.core.udf.GenerateStringArray +com.geedgenetworks.core.udf.GeoIpLookup +com.geedgenetworks.core.udf.cn.L7ProtocolAndAppExtract +com.geedgenetworks.core.udf.cn.IdcRenterLookup +com.geedgenetworks.core.udf.cn.LinkDirectionLookup +com.geedgenetworks.core.udf.cn.FqdnCategoryLookup +com.geedgenetworks.core.udf.cn.IcpLookup +com.geedgenetworks.core.udf.cn.FqdnWhoisLookup +com.geedgenetworks.core.udf.cn.DnsServerInfoLookup +com.geedgenetworks.core.udf.cn.AppCategoryLookup +com.geedgenetworks.core.udf.cn.IpZoneLookup +com.geedgenetworks.core.udf.cn.VpnLookup +com.geedgenetworks.core.udf.cn.AnonymityLookup +com.geedgenetworks.core.udf.cn.IocLookup +com.geedgenetworks.core.udf.cn.FieldsMerge +com.geedgenetworks.core.udf.cn.ArrayElementsPrepend +com.geedgenetworks.core.udf.cn.IntelligenceIndicatorLookup +com.geedgenetworks.core.udf.SnowflakeId +com.geedgenetworks.core.udf.UnixTimestampConverter +com.geedgenetworks.core.udf.Domain +com.geedgenetworks.core.udf.cn.BaseStationLookup +com.geedgenetworks.core.udf.cn.H3CellLookup +com.geedgenetworks.core.udf.JsonExtract +com.geedgenetworks.core.udf.Rename \ No newline at end of file diff --git a/cyber_narrator/upgrade/2024/CN-24.08/groot-stream/1.7.0/24.08.1/sd_kafka_to_cn_clickhouse b/cyber_narrator/upgrade/2024/CN-24.08/groot-stream/1.7.0/24.08.1/sd_kafka_to_cn_clickhouse index 0f6c4f1..2375832 100644 --- a/cyber_narrator/upgrade/2024/CN-24.08/groot-stream/1.7.0/24.08.1/sd_kafka_to_cn_clickhouse +++ b/cyber_narrator/upgrade/2024/CN-24.08/groot-stream/1.7.0/24.08.1/sd_kafka_to_cn_clickhouse @@ -168,7 +168,7 @@ processing_pipelines: output_fields: [ subscriber_tags ] time_processor: - type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl + type: projection remove_fields: output_fields: functions: