diff --git a/groot-stream/templates/datapath_telemetry_record.yaml.j2 b/groot-stream/templates/datapath_telemetry_record.yaml.j2 index 2bbd68e..0616617 100644 --- a/groot-stream/templates/datapath_telemetry_record.yaml.j2 +++ b/groot-stream/templates/datapath_telemetry_record.yaml.j2 @@ -38,6 +38,7 @@ sinks: type: kafka properties: topic: {{ kafka_sink_topic }} + kafka.client.id: {{ kafka_sink_topic }} kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.retries: 0 kafka.linger.ms: 10 diff --git a/groot-stream/templates/proxy_event.yaml.j2 b/groot-stream/templates/proxy_event.yaml.j2 index e793d38..939a26f 100644 --- a/groot-stream/templates/proxy_event.yaml.j2 +++ b/groot-stream/templates/proxy_event.yaml.j2 @@ -3,15 +3,14 @@ sources: type: kafka properties: topic: PROXY-EVENT + kafka.group.id: {{ kafka_source_group_id }} kafka.bootstrap.servers: {{ kafka_source_servers }} - kafka.client.id: PROXY-EVENT kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 - kafka.group.id: {{ kafka_source_group_id }} kafka.auto.offset.reset: latest format: json json.ignore.parse.errors: false @@ -79,7 +78,6 @@ processing_pipelines: parameters: path: [props.hos.path, props.hos.bucket.name.policy_capture_file, packet_capture_file] - - function: CURRENT_UNIX_TIMESTAMP output_fields: [ processing_time ] parameters: diff --git a/groot-stream/templates/traffic_sketch_metric.yaml.j2 b/groot-stream/templates/traffic_sketch_metric.yaml.j2 index cd627e2..0b4d2d4 100644 --- a/groot-stream/templates/traffic_sketch_metric.yaml.j2 +++ b/groot-stream/templates/traffic_sketch_metric.yaml.j2 @@ -12,15 +12,14 @@ sources: kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.group.id: etl_traffic_sketch_metric kafka.auto.offset.reset: latest - kafka.compression.type: none format: json processing_pipelines: - etl_processor: # [object] Processing Pipeline + etl_processor: type: projection remove_fields: output_fields: - functions: # [array of object] Function List + functions: - function: UNIX_TIMESTAMP_CONVERTER lookup_fields: [ timestamp_ms ] output_fields: [ recv_time ] @@ -38,7 +37,6 @@ processing_pipelines: - function: SNOWFLAKE_ID lookup_fields: [ '' ] output_fields: [ log_id ] - filter: parameters: data_center_id_num: 1 @@ -77,11 +75,11 @@ sinks: application: - env: # [object] Environment Variables - name: etl_traffic_sketch_metric # [string] Job Name + env: + name: etl_traffic_sketch_metric shade.identifier: aes pipeline: - object-reuse: true # [boolean] Object Reuse, default is false + object-reuse: true {{ topology }} diff --git a/hbase/tsg_olap_hbase_ddl.sql b/hbase/tsg_olap_hbase_ddl.sql new file mode 100644 index 0000000..fed3ece --- /dev/null +++ b/hbase/tsg_olap_hbase_ddl.sql @@ -0,0 +1,6 @@ +create_namespace 'dos' +create_namespace 'tsg_galaxy' + +create 'tsg_galaxy:saved_query_result', {NAME => 'response', VERSIONS => 1,COMPRESSION => 'GZ',IS_MOB => true, MOB_THRESHOLD => 0}, {NAME => 'detail',COMPRESSION => 'GZ',VERSIONS => 1} + +create 'dos:ddos_traffic_baselines', 'TCP SYN Flood','UDP Flood','ICMP Flood','DNS Flood' diff --git a/hbase/tsg_olap_phoenix_ddl.sql b/hbase/tsg_olap_phoenix_ddl.sql new file mode 100644 index 0000000..41006e4 --- /dev/null +++ b/hbase/tsg_olap_phoenix_ddl.sql @@ -0,0 +1,3 @@ +CREATE schema IF NOT EXISTS "tsg_galaxy"; + +CREATE view IF NOT EXISTS "tsg_galaxy"."saved_query_result"( ROWKEY VARCHAR PRIMARY KEY, "detail"."excute_sql" VARCHAR, "detail"."read_rows" UNSIGNED_LONG, "detail"."result_id" UNSIGNED_INT, "response"."result" VARCHAR);