Merge branch 'main' into 'develop'

Main

See merge request galaxy/deployment/tsg-olap-data-initialization!1

groot-stream/templates/datapath_telemetry_record.yaml.j2

@@ -38,6 +38,7 @@ sinks:
     type: kafka
     properties:
       topic: {{ kafka_sink_topic }}
+      kafka.client.id: {{ kafka_sink_topic }}
       kafka.bootstrap.servers: {{ kafka_sink_servers }}
       kafka.retries: 0
       kafka.linger.ms: 10

groot-stream/templates/proxy_event.yaml.j2

@@ -3,15 +3,14 @@ sources:
     type: kafka
     properties:
       topic: PROXY-EVENT
+      kafka.group.id: {{ kafka_source_group_id }}
       kafka.bootstrap.servers: {{ kafka_source_servers }}
-      kafka.client.id: PROXY-EVENT
       kafka.session.timeout.ms: 60000
       kafka.max.poll.records: 3000
       kafka.max.partition.fetch.bytes: 31457280
       kafka.security.protocol: SASL_PLAINTEXT
       kafka.sasl.mechanism: PLAIN
       kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
-      kafka.group.id: {{ kafka_source_group_id }}
       kafka.auto.offset.reset: latest
       format: json
       json.ignore.parse.errors: false
@@ -79,7 +78,6 @@ processing_pipelines:
       parameters:
         path: [props.hos.path, props.hos.bucket.name.policy_capture_file, packet_capture_file]
 
-
     - function: CURRENT_UNIX_TIMESTAMP
       output_fields: [ processing_time ]
       parameters:

groot-stream/templates/traffic_sketch_metric.yaml.j2

@@ -12,15 +12,14 @@ sources:
       kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817
       kafka.group.id: etl_traffic_sketch_metric
       kafka.auto.offset.reset: latest
-      kafka.compression.type: none
       format: json
 
 processing_pipelines:
-  etl_processor: # [object] Processing Pipeline
+  etl_processor:
     type: projection
     remove_fields:
     output_fields:
-    functions: # [array of object] Function List
+    functions:
       - function: UNIX_TIMESTAMP_CONVERTER
         lookup_fields: [ timestamp_ms ]
         output_fields: [ recv_time ]
@@ -38,7 +37,6 @@ processing_pipelines:
       - function: SNOWFLAKE_ID
         lookup_fields: [ '' ]
         output_fields: [ log_id ]
-        filter:
         parameters:
           data_center_id_num: 1
 
@@ -77,11 +75,11 @@ sinks:
 
 application:
 
-  env: # [object] Environment Variables
+  env:
-    name: etl_traffic_sketch_metric # [string] Job Name
+    name: etl_traffic_sketch_metric
     shade.identifier: aes
     pipeline:
-      object-reuse: true # [boolean] Object Reuse, default is false
+      object-reuse: true
   {{ topology }}
 
 

hbase/tsg_olap_hbase_ddl.sql

@@ -0,0 +1,6 @@
+create_namespace 'dos'
+create_namespace 'tsg_galaxy'
+
+create 'tsg_galaxy:saved_query_result', {NAME => 'response', VERSIONS => 1,COMPRESSION => 'GZ',IS_MOB => true, MOB_THRESHOLD => 0}, {NAME => 'detail',COMPRESSION => 'GZ',VERSIONS => 1}
+
+create 'dos:ddos_traffic_baselines', 'TCP SYN Flood','UDP Flood','ICMP Flood','DNS Flood'

hbase/tsg_olap_phoenix_ddl.sql

@@ -0,0 +1,3 @@
+CREATE schema IF NOT EXISTS "tsg_galaxy";
+
+CREATE view IF NOT EXISTS "tsg_galaxy"."saved_query_result"( ROWKEY VARCHAR PRIMARY KEY, "detail"."excute_sql" VARCHAR, "detail"."read_rows" UNSIGNED_LONG, "detail"."result_id" UNSIGNED_INT, "response"."result" VARCHAR);