diff --git a/24.05/README.md b/24.05/README.md index 3bee145..52425d8 100644 --- a/24.05/README.md +++ b/24.05/README.md @@ -2,119 +2,78 @@ ## 概述 -TSG OLAP的数据摄入分为三种类型：Logs、Metrics 和 File Chunks。为确保能够正确处理并写入相应存储系统，需要支持端到端业务自检。 +TSG OLAP 支持对Logs，Metrics和Files进行端到端业务自检,自检数据流： Smaple Datasets -> Kafka Topic -> FLINK ETL -> Storage DB -> QGW API . ## 环境依赖 -- 测试数据生成工具 `e2e-mockdata-generator.jar` -- 安装`Newman`，详细参考[帮助文档](https://learning.postman.com/docs/collections/using-newman-cli/newman-options/) +- 样例数据生成工具：`e2e-mockdata-generator` ，需要JDK11环境 +- Report诊断工具：`Newman`，具体参考[帮助文档](https://learning.postman.com/docs/collections/using-newman-cli/newman-options/) ## 使用方法 -### Newman CLI enviroment.json 修改如下配置 -`Newman CLI 使用QGW HTTP Rest API 接口输出诊断报告，一般部署至国家中心` +### 修改 enviroment.json 配置 + +`Newman CLI 使用QGW HTTP REST API 输出诊断报告,需要添加访问IP` ```json +[ { - "key": "qgw_ip", - "value": "192.168.44.30", - "type": "default", - "enabled": true - }, - { - "key": "qgw_port", - "value": "9999", - "type": "default", - "enabled": true - } - + "key": "qgw_ip", + "value": "127.0.0.1", + "type": "default", + "enabled": true + }, + { + "key": "hos_token", + "value": "c21f969b5f03d33d43e04f8f136e7682", + "type": "secret", + "enabled": true + } +] ``` -### 测试数据生成工具修改Kafka地址 +### 修改e2e_test.sh配置 -`Kafka需为分中心地址` - -- 修改`config.properties`, 增加Kafka访问地址 -```props - kafka.server=192.168.41.29:9092 -``` - -### 写入测试集至Kafka -- Logs -```shell - -cd e2e-mockdata-generator/ - -java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --topic SESSION-RECORD -f ./datasets/logs/session_record.dat - -java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --topic VOIP-RECORD -f ./datasets/logs/voip_record.dat - -java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --topic PROXY-EVENT -f ./datasets/logs/proxy_event.dat - -``` -- Metrics +- 增加每个分中心的kafka broker地址 ```shell -cd e2e-mockdata-generator/ - -java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --topic NETWORK-TRAFFIC-METRIC -f ./datasets/metrics/network_traffic_metric.dat - -java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --topic POLICY-RULE-METRIC -f ./datasets/metrics/policy_rule_metric.dat - -java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --topic OBJECT-STATISTICS-METRIC -f ./datasets/metrics/object_statistics_metric.dat - -java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --topic STATISTICS-RULE-METRIC -f ./datasets/metrics/statistics_rule_metric.dat - + # [data_center_name]:kafka_server_list + declare -A KAFKA_SERVERS=( + ["tsg_olap_dc_a"]="192.168.44.11:9092" + ["tsg_olap_dc_b"]="192.168.44.11:9092" + ) +``` +- 增加每个分中心的HOS访问地址 +```shell + # [data_center_name]:hos_endpoint_uri + declare -A HOS_ENDPOINTS=( + ["tsg_olap_dc_a"]="192.168.44.11" + ["tsg_olap_dc_b"]="192.168.44.11" + ) ``` -- Files - - 123e4567-e89b-12d3-a456-426614174001 监测策略 PcapNG - - 123e4567-e89b-12d3-a456-426614174002 HTTP Request Body - - 123e4567-e89b-12d3-a456-426614174003 HTTP Response Body - - 123e4567-e89b-12d3-a456-426614174004 MAIL EML - - 123e4567-e89b-12d3-a456-426614174005 RTP PcapNG - - 123e4567-e89b-12d3-a456-426614174006 Troubleshooting PcapNG - - 123e4567-e89b-12d3-a456-426614174007 Datapath Telemetry PcapNG +### 命令详解 + +`./e2e_test.sh -h` ```shell -cd e2e-mockdata-generator/ +Usage: ./e2e_test.sh [options] -java -cp e2e-mockdata-generator.jar com.geedgenetworks.FileChunkGenerator --topic TRAFFIC-FILE-STREAM-RECORD -n 123e4567-e89b-12d3-a456-426614174001 --file_type traffic_pcapng +Options: + -g Generate data (logs, metrics, files) + -d Run diagnostic report (logs, metrics, files) + -c Clear test data + -a Perform all operations: generate data, run diagnostics, and clear data + -i Set environment variable (data_center, hos_ip) + -v Enable verbose reporting + -e Enable emojitrain reporting + -h Show this help message -java -cp e2e-mockdata-generator.jar com.geedgenetworks.FileChunkGenerator --topic TRAFFIC-FILE-STREAM-RECORD -n 123e4567-e89b-12d3-a456-426614174002 --file_type html - -java -cp e2e-mockdata-generator.jar com.geedgenetworks.FileChunkGenerator --topic TRAFFIC-FILE-STREAM-RECORD -n 123e4567-e89b-12d3-a456-426614174003 --file_type html - -java -cp e2e-mockdata-generator.jar com.geedgenetworks.FileChunkGenerator --topic TRAFFIC-FILE-STREAM-RECORD -n 123e4567-e89b-12d3-a456-426614174004 --file_type eml - -java -cp e2e-mockdata-generator.jar com.geedgenetworks.FileChunkGenerator --topic TRAFFIC-FILE-STREAM-RECORD -n 123e4567-e89b-12d3-a456-426614174005 --file_type traffic_pcapng - -# java -cp e2e-mockdata-generator.jar com.geedgenetworks.FileChunkGenerator --topic TROUBLESHOOTING-FILE-STREAM-RECORD -n 123e4567-e89b-12d3-a456-426614174006 --file_type troubleshooting_pcapng - -# java -cp e2e-mockdata-generator.jar com.geedgenetworks.FileChunkGenerator --topic DATAPATH-TELEMETRY-RECORD -n 123e4567-e89b-12d3-a456-426614174007 --file_type datapath_telemetry_pcapng +Examples: + ./e2e_test.sh -a -e Perform all operations and enable emojitrain reporting + ./e2e_test.sh -g logs -i data_center=my_data_center Generate log data at my_data_center + ./e2e_test.sh -d logs -v Run diagnostics on logs data with verbose reporting + ./e2e_test.sh -d metrics -v Run diagnostics on metrics data with verbose reporting + ./e2e_test.sh -g logs Generate log data use default data center + ./e2e_test.sh -c Clear test data + ``` - -### 输出故障诊断报告（等待3-5分钟） - -```shell - -# -folder logs ：对日志进行故障诊断，输出诊断明细；指定data_center。 -# -folder metrics：对Metrics进行故障诊断，输出诊断明细；指定data_center。 -# -folder files：对文件进行故障诊断，输出诊断明细；指定分中心HOS访问地址。 -newman run ./tsg-olap-e2e-test-collection.json -n 1 -e ./environment.json --delay-request 500 --timeout-script 10000 --timeout-request 300000 --timeout 3600000 --insecure --verbose --ignore-redirects --env-var "data_center=tsg_olap" --folder logs - -newman run ./tsg-olap-e2e-test-collection.json -n 1 -e ./environment.json --delay-request 500 --timeout-script 10000 --timeout-request 300000 --timeout 3600000 --insecure --verbose --ignore-redirects --env-var "data_center=tsg_olap" --folder metrics - -newman run ./tsg-olap-e2e-test-collection.json -n 1 -e ./environment.json --delay-request 500 --timeout-script 10000 --timeout-request 300000 --timeout 3600000 --insecure --verbose --ignore-redirects --env-var "hos_ip=127.0.0.1" --folder files - -# -folder logs：对日志进行故障诊断，通过表情形式输出测试结果 -# -folder files：对文件进行故障诊断，通过表情形式输出测试结果 -newman run ./tsg-olap-e2e-test-collection.json -n 1 --delay-request 500 -e ./environment.json --env-var "data_center=tsg_olap" --ignore-redirects --folder logs -r emojitrain -newman run ./tsg-olap-e2e-test-collection.json -n 1 --delay-request 500 -e ./environment.json --env-var "hos_ip=127.0.0.1" --ignore-redirects --folder files -r emojitrain - -#清除测试数据（暂支持对文件的删除） -newman run ./tsg-olap-e2e-test-collection.json -n 1 --delay-request 500 -e ./environment.json --ignore-redirects --folder clear_test_data -r emojitrain - - -``` - - diff --git a/24.05/bin/e2e_test.sh b/24.05/bin/e2e_test.sh new file mode 100755 index 0000000..fb0e8e1 --- /dev/null +++ b/24.05/bin/e2e_test.sh @@ -0,0 +1,251 @@ +#!/bin/bash +#BASE_DIR=$(cd $(dirname $0) && pwd) +BASE_DIR="$(dirname "$(pwd)")" +# Check if required tools are installed +if ! command -v java &> /dev/null; then + echo "Error: Java is not installed." + exit 1 +fi + +# Define common variables +E2E_MOCKDATA_GENERATOR_PATH="$BASE_DIR/e2e-mockdata-generator" +CONFIG_PATH="$BASE_DIR/config" +COLLECTION="tsg-olap-e2e-test-collection.json" +ENVIRONMENT="$BASE_DIR/environment.json" +DATA_CENTER="tsg_olap_dc_a" # Default Data Center +OTHER_VAR="" + +# data_center_name:kafka_server +declare -A KAFKA_SERVERS=( + ["tsg_olap_dc_a"]="192.168.44.11:9092" + ["tsg_olap_dc_b"]="192.168.44.11:9092" +) + +# data_center_name:hos_endpoint +declare -A HOS_ENDPOINTS=( + ["tsg_olap_dc_a"]="192.168.44.11" + ["tsg_olap_dc_b"]="192.168.44.11" +) + +log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" +} + +generate_logs() { + log_message "Generating logs for $1 $2 ..." + local data_center=$1 + local bootstrap_server=$2 + cd $E2E_MOCKDATA_GENERATOR_PATH || { echo "Error: Cannot change directory to $E2E_MOCKDATA_GENERATOR_PATH"; exit 1; } + declare -A LOGS=( + ["SESSION-RECORD"]="session_record.dat" + ["VOIP-RECORD"]="voip_record.dat" + ["PROXY-EVENT"]="proxy_event.dat" + ) + for topic in "${!LOGS[@]}"; do + # Replace the "data_center" value in the file + sed -i "s/\"data_center\":\"[^\"]*\"/\"data_center\":\"$data_center\"/g" ./datasets/logs/${LOGS[$topic]} + java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --bootstrap_server $bootstrap_server --topic $topic -f ./datasets/logs/${LOGS[$topic]} + done +} + +generate_metrics() { + log_message "Generating metrics for $1 $2..." + local data_center=$1 + local bootstrap_server=$2 + cd $E2E_MOCKDATA_GENERATOR_PATH || { echo "Error: Cannot change directory to $E2E_MOCKDATA_GENERATOR_PATH"; exit 1; } + declare -A METRICS=( + ["NETWORK-TRAFFIC-METRIC"]="network_traffic_metric.dat" + ["POLICY-RULE-METRIC"]="policy_rule_metric.dat" + ["OBJECT-STATISTICS-METRIC"]="object_statistics_metric.dat" + ["STATISTICS-RULE-METRIC"]="statistics_rule_metric.dat" + ) + for topic in "${!METRICS[@]}"; do + # Replace the "data_center" value in the file + sed -i "s/\"data_center\":\"[^\"]*\"/\"data_center\":\"$data_center\"/g" ./datasets/metrics/${METRICS[$topic]} + java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --bootstrap_server $bootstrap_server --topic $topic -f ./datasets/metrics/${METRICS[$topic]} + done +} + + +generate_files() { + log_message "Generating files for $1 $2 ..." + local data_center=$1 + local bootstrap_server=$2 + cd $E2E_MOCKDATA_GENERATOR_PATH || { echo "Error: Cannot change directory to $E2E_MOCKDATA_GENERATOR_PATH"; exit 1; } + # 123e4567-e89b-12d3-a456-426614174006 troubleshooting_pcapng + # 123e4567-e89b-12d3-a456-426614174007 datapath_telemetry_pcapng + declare -A FILES=( + ["123e4567-e89b-12d3-a456-426614174001"]="traffic_pcapng" + ["123e4567-e89b-12d3-a456-426614174002"]="html" + ["123e4567-e89b-12d3-a456-426614174003"]="html" + ["123e4567-e89b-12d3-a456-426614174004"]="eml" + ["123e4567-e89b-12d3-a456-426614174005"]="traffic_pcapng" + ) + for uuid in "${!FILES[@]}"; do + java -cp e2e-mockdata-generator.jar com.geedgenetworks.FileChunkGenerator --bootstrap_server $bootstrap_server --topic TRAFFIC-FILE-STREAM-RECORD -n $uuid --file_type ${FILES[$uuid]} + done +} + + +# Define function to run diagnostic report + # -folder logs ：对日志进行故障诊断，输出诊断明细；指定data_center。 + # -folder metrics：对Metrics进行故障诊断，输出诊断明细；指定data_center。 + # -folder files：对文件进行故障诊断，输出诊断明细；指定分中心HOS访问地址。 + # -folder logs：对日志进行故障诊断，通过表情形式输出测试结果 + # -folder files：对文件进行故障诊断，通过表情形式输出测试结果 + # newman run ./tsg-olap-e2e-test-collection.json -n 1 --delay-request 500 -e $ENVIRONMENT --env-var "data_center=$DATA_CENTER" --ignore-redirects --folder logs -r emojitrain + # newman run ./tsg-olap-e2e-test-collection.json -n 1 --delay-request 500 -e $ENVIRONMENT --env-var "hos_ip=$HOS_IP" --ignore-redirects --folder files -r emojitrain +run_diagnostic() { + log_message "Running diagnostic for $1..." + cd $BASE_DIR || { echo "Error: Cannot change directory to $BASE_DIR"; exit 1; } + local folder=$1 + local env_var=$2 + local verbose="" + local emojitrain="" + if $verbose_flag; then + verbose="--verbose" + fi + + if $emojitrain_flag; then + emojitrain="-r emojitrain" + fi + + newman run $CONFIG_PATH/tsg-olap-e2e-test-collection.json -g $CONFIG_PATH/globals.json -n 1 -e $CONFIG_PATH/environment.json --delay-request 500 --timeout-script 10000 --timeout-request 300000 --timeout 3600000 --insecure $verbose --ignore-redirects --env-var $env_var --folder $folder $emojitrain +} + + +clear_data() { + log_message "Clearing test data..." + newman run $CONFIG_PATH/tsg-olap-e2e-test-collection.json -g $CONFIG_PATH/globals.json -n 1 --delay-request 500 -e $CONFIG_PATH/environment.json --ignore-redirects --folder clear_test_data -r emojitrain +} + +# Help message +show_help() { + echo "" + echo "Usage: $0 [options]" + echo "" + echo "Options:" + echo " -g Generate data (logs, metrics, files)" + echo " -d Run diagnostic report (logs, metrics, files)" + echo " -c Clear test data" + echo " -a Perform all operations: generate data, run diagnostics, and clear data" + echo " -i " + echo " Set environment variable (data_center, hos_ip)" + echo " -v Enable verbose reporting" + echo " -e Enable emojitrain reporting" + echo " -h Show this help message" + echo "" + echo "Examples:" + echo " $0 -a Perform all operations" + echo " $0 -g logs Generate log data" + echo " $0 -d metrics -v Run diagnostics on metrics data with verbose reporting" + echo " $0 -c Clear test data" + echo " $0 -g logs -i data_center=my_data_center Generate log data in my_data_center" + echo "" +} + +# Initialize flags +generate_flag=false +diagnostic_flag=false +verbose_flag=false +emojitrain_flag=false +clear_flag=false +all_flag=false +generate_type="" +diagnostic_type="" + + +# Parse command-line arguments +while getopts ":g:d:i:acveh" opt; do + case $opt in + g) + generate_type=$OPTARG + generate_flag=true + ;; + d) + diagnostic_type=$OPTARG + diagnostic_flag=true + ;; + c) + clear_flag=true + ;; + a) + all_flag=true + ;; + i) + # Split the input based on whitespace, then further split by '=' to assign key-value pairs + IFS=',' read -ra vars <<< "$OPTARG" + for var in "${vars[@]}"; do + # Remove leading and trailing whitespaces + var=$(echo "$var" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + IFS='=' read -r key value <<< "$var" + case $key in + data_center) DATA_CENTER=$value ;; + other_var) OTHER_VAR=$value ;; + *) echo "Invalid environment variable: $key"; show_help; exit 1 ;; + esac + done + ;; + v) + verbose_flag=true + ;; + e) + emojitrain_flag=true + ;; + h) + show_help + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + show_help + exit 1 + ;; + esac +done + + +if $generate_flag; then + case $generate_type in + logs) generate_logs "$DATA_CENTER" "${KAFKA_SERVERS[$DATA_CENTER]}" ;; + metrics) generate_metrics "$DATA_CENTER" "${KAFKA_SERVERS[$DATA_CENTER]}" ;; + files) generate_files "$DATA_CENTER" "${KAFKA_SERVERS[$DATA_CENTER]}" ;; + *) echo "Invalid generate type"; show_help; exit 1 ;; + esac +fi + +if $diagnostic_flag; then + case $diagnostic_type in + logs) run_diagnostic "logs" "data_center=$DATA_CENTER" ;; + metrics) run_diagnostic "metrics" "data_center=$DATA_CENTER" ;; + files) run_diagnostic "files" "data_center=$DATA_CENTER,hos_ip=${HOS_ENDPOINTS[$DATA_CENTER]}" ;; + *) echo "Invalid diagnostic type"; show_help; exit 1 ;; + esac +fi + +if $clear_flag; then + clear_data +fi + +if $all_flag; then + + for data_center_var in "${!KAFKA_SERVERS[@]}"; do + generate_logs "$data_center_var" "${KAFKA_SERVERS[$data_center_var]}" + generate_metrics "$data_center_var" "${KAFKA_SERVERS[$data_center_var]}" + generate_files "$data_center_var" "${KAFKA_SERVERS[$data_center_var]}" + echo "Wait 30 Seconds..." + sleep 30 + run_diagnostic "logs" "data_center=$data_center_var" + run_diagnostic "metrics" "data_center=$data_center_var" + run_diagnostic "files" "data_center=$data_center_var,hos_ip=${HOS_ENDPOINTS[$data_center_var]}" + clear_data + done +fi + +# Display help message if no arguments are passed +if ! $generate_flag && ! $diagnostic_flag && ! $clear_flag && ! $all_flag; then + show_help + exit 1 +fi + +log_message "E2E test execution completed." diff --git a/24.05/bin/function_test.sh b/24.05/bin/function_test.sh new file mode 100755 index 0000000..772e9f3 --- /dev/null +++ b/24.05/bin/function_test.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +show_help() { + echo " +Usage: $(basename "$0") [-f FOLDER_NUMBER] [-a] [-v] [-e] + +Options: + -f FOLDER_NUMBER Specify a single folder to run from a collection: + 1. Query + 2. Dataset + 3. Database + 4. Util + 5. Troubleshooting + 6. HOS + 7. Knowledge Base File + -a Run all options [1-7] + -v Enable verbose reporting + -e Enable emojitrain reporting + -h, --help Show this help message and exit +" +} + +BASE_DIR="$(dirname "$(pwd)")" +CONFIG_PATH="$BASE_DIR/config" +FOLDER="" +ALL=false +VERBOSE_FLAG=false +EMOJITRAIN_FLAG=false + +# Flag to check if any valid option is provided +VALID_OPTION_PROVIDED=false + +# Parse command-line arguments +while [[ $# -gt 0 ]]; do + case $1 in + -f) + VALID_OPTION_PROVIDED=true + if [[ -n $2 && $2 =~ ^[1-7]$ ]]; then + case $2 in + 1) FOLDER="--folder Query" ;; + 2) FOLDER="--folder Dataset" ;; + 3) FOLDER="--folder Database" ;; + 4) FOLDER="--folder Util" ;; + 5) FOLDER="--folder Troubleshooting" ;; + 6) FOLDER="--folder HOS" ;; + 7) FOLDER="--folder Knowledge Base File" ;; + esac + shift 2 + else + echo "Error: Invalid folder number." + show_help + exit 1 + fi + ;; + -a) + VALID_OPTION_PROVIDED=true + ALL=true + shift + ;; + -v) + VERBOSE_FLAG=true + shift + ;; + -e) + EMOJITRAIN_FLAG=true + shift + ;; + -h|--help) + show_help + exit 0 + ;; + *) + echo "Error: Unknown option $1" + show_help + exit 1 + ;; + esac +done + +if ! $VALID_OPTION_PROVIDED; then + show_help + exit 1 +fi + +if $ALL; then + FOLDER="" +fi + +NEW_MAN_OPTS=( + "$CONFIG_PATH/tsg-olap-function-test-collection.json" + -n 1 + -e "$CONFIG_PATH/environment.json" + -g "$CONFIG_PATH/globals.json" + --delay-request 200 + --timeout-script 10000 + --timeout-request 300000 + --timeout 3600000 + --insecure +) + +# Add verbose reporting option if enabled +if $VERBOSE_FLAG; then + NEW_MAN_OPTS+=(--verbose) +fi + +# Add folder option if set +if [[ -n $FOLDER ]]; then + NEW_MAN_OPTS+=($FOLDER) +fi + +# Add emojitrain reporting option if enabled +if $EMOJITRAIN_FLAG; then + NEW_MAN_OPTS+=(-r emojitrain) +fi + +# Run the newman command +newman run "${NEW_MAN_OPTS[@]}" + diff --git a/24.05/environment.json b/24.05/config/environment.json similarity index 100% rename from 24.05/environment.json rename to 24.05/config/environment.json diff --git a/24.05/config/globals.json b/24.05/config/globals.json new file mode 100644 index 0000000..0397c3f --- /dev/null +++ b/24.05/config/globals.json @@ -0,0 +1,39 @@ +{ + "id": "0083244f-f7da-4ec8-8c09-317c1121d3ad", + "values": [ + { + "key": "start_time", + "value": "", + "type": "any", + "enabled": true + }, + { + "key": "end_time", + "value": "", + "type": "any", + "enabled": true + }, + { + "key": "domain", + "value": "", + "type": "any", + "enabled": true + }, + { + "key": "client_ip", + "value": "", + "type": "any", + "enabled": true + }, + { + "key": "server_ip", + "value": "", + "type": "any", + "enabled": true + } + ], + "name": "Globals", + "_postman_variable_scope": "globals", + "_postman_exported_at": "2024-05-21T02:39:11.566Z", + "_postman_exported_using": "Postman/11.1.3" +} \ No newline at end of file diff --git a/24.05/tsg-olap-e2e-test-collection.json b/24.05/config/tsg-olap-e2e-test-collection.json similarity index 100% rename from 24.05/tsg-olap-e2e-test-collection.json rename to 24.05/config/tsg-olap-e2e-test-collection.json diff --git a/24.05/tsg-olap-function-test-collection.json b/24.05/config/tsg-olap-function-test-collection.json similarity index 100% rename from 24.05/tsg-olap-function-test-collection.json rename to 24.05/config/tsg-olap-function-test-collection.json diff --git a/24.05/e2e-mockdata-generator/config.properties b/24.05/e2e-mockdata-generator/config.properties index 6c07568..9080c00 100644 --- a/24.05/e2e-mockdata-generator/config.properties +++ b/24.05/e2e-mockdata-generator/config.properties @@ -1,5 +1,3 @@ -#####9092-Plaintext 9094-SASL -kafka.server=192.168.44.11:9092 #####kafka SASL username/password #kafka.user=admin -#kafka.pin=galaxy2019 +#kafka.pin=galaxy2019 \ No newline at end of file diff --git a/24.05/e2e-mockdata-generator/datasets/logs/proxy_event.dat b/24.05/e2e-mockdata-generator/datasets/logs/proxy_event.dat index c487511..8f8ea89 100644 --- a/24.05/e2e-mockdata-generator/datasets/logs/proxy_event.dat +++ b/24.05/e2e-mockdata-generator/datasets/logs/proxy_event.dat @@ -1 +1 @@ -{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240001,"decoded_as":"HTTP","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.1","server_ip":"192.0.2.1","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.HTTP","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"tsg_olap\"},{\"tag\":\"device_group\",\"value\":\"tsg_olap\"}]}","dup_traffic_flag":0,"sc_rsp_raw":[0],"http_version":"http1","http_request_line":"GET www.google.com/ HTTP/1.1","http_response_line":"HTTP/1.1 200 OK","http_status_code":200,"http_url":"www.google.com/","http_host":"www.google.com","http_cookie":"NID=513","http_user_agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; WOW64; Trident/5.0)","http_response_content_type":"text/html; charset=UTF-8","proxy_rule_list":[4450],"proxy_action":"insert","http_action_file_size":35} +{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240001,"decoded_as":"HTTP","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.1","server_ip":"192.0.2.1","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.HTTP","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","dup_traffic_flag":0,"sc_rsp_raw":[0],"http_version":"http1","http_request_line":"GET www.google.com/ HTTP/1.1","http_response_line":"HTTP/1.1 200 OK","http_status_code":200,"http_url":"www.google.com/","http_host":"www.google.com","http_cookie":"NID=513","http_user_agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; WOW64; Trident/5.0)","http_response_content_type":"text/html; charset=UTF-8","proxy_rule_list":[4450],"proxy_action":"insert","http_action_file_size":35} diff --git a/24.05/e2e-mockdata-generator/datasets/logs/session_record.dat b/24.05/e2e-mockdata-generator/datasets/logs/session_record.dat index 1821406..1db77e7 100644 --- a/24.05/e2e-mockdata-generator/datasets/logs/session_record.dat +++ b/24.05/e2e-mockdata-generator/datasets/logs/session_record.dat @@ -1,12 +1,7 @@ -{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240001,"decoded_as":"BASE","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.1","server_ip":"192.0.2.1","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"tsg_olap\"},{\"tag\":\"device_group\",\"value\":\"tsg_olap\"}]}","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]"} -{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240002,"decoded_as":"HTTP","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.2","server_ip":"192.0.2.2","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.HTTP","server_fqdn":"static-pcs-sdk-server.test.com","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"tsg_olap\"},{\"tag\":\"device_group\",\"value\":\"tsg_olap\"}]}","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","packet_capture_file":"123e4567-e89b-12d3-a456-426614174001","http_request_body":"123e4567-e89b-12d3-a456-426614174002","http_response_body":"123e4567-e89b-12d3-a456-426614174003","http_version":"http1","http_request_line":"POST /a HTTP/1.1","http_user_agent":"WinHttpClient","http_request_content_length":0,"http_host":"static-pcs-sdk-server.test.com","http_url":"static-pcs-sdk-server.test.com/a","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"application/json;charset=UTF-8","http_response_content_length":0,"http_response_latency_ms":0,"http_session_duration_ms":0,"http_sequence":1} -{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240003,"decoded_as":"SSL","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.3","server_ip":"192.0.2.3","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.ssl","server_fqdn":"storeedgefd.dsx.mp.microsoft.com","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"tsg_olap\"},{\"tag\":\"device_group\",\"value\":\"tsg_olap\"}]}","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","ssl_ja3_hash":"","ssl_esni_flag":0,"ssl_ech_flag":0,"ssl_sni":"storeedgefd.dsx.mp.microsoft.com","ssl_ja3s_hash":"","ssl_version":"v3","ssl_cn":"sfdataservice.microsoft.com","ssl_cert_issuer":"","ssl_cert_subject":"","ssl_san":"","ssl_handshake_latency_ms":0} -{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240004,"decoded_as":"DNS","ip_protocol":"udp","address_type":4,"client_ip":"10.0.0.4","server_ip":"192.0.2.4","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.UDP.dns","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"tsg_olap\"},{\"tag\":\"device_group\",\"value\":\"tsg_olap\"}]}","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","dns_response_latency_ms":0,"dns_qr":1,"dns_aa":0,"dns_message_id":47610,"dns_opcode":0,"dns_ra":1,"dns_rcode":0,"dns_rd":1,"dns_tc":0,"dns_qdcount":1,"dns_ancount":1,"dns_nscount":6,"dns_arcount":13,"dns_qname":"","dns_qtype":1,"dns_qclass":1} -{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240005,"decoded_as":"QUIC","ip_protocol":"udp","address_type":4,"client_ip":"10.0.0.5","server_ip":"192.0.2.5","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.UDP.quic","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"tsg_olap\"},{\"tag\":\"device_group\",\"value\":\"tsg_olap\"}]}","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","quic_version":"IETF QUIC RFC9000"} -{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240006,"decoded_as":"MAIL","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.6","server_ip":"192.0.2.6","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.mail.imap","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"tsg_olap\"},{\"tag\":\"device_group\",\"value\":\"tsg_olap\"}]}","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","mail_protocol_type":"IMAP","mail_eml_file":"123e4567-e89b-12d3-a456-426614174004"} -{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240007,"decoded_as":"SSH","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.7","server_ip":"192.0.2.7","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.ssh","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"tsg_olap\"},{\"tag\":\"device_group\",\"value\":\"tsg_olap\"}]}","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","ssh_server_version":"SSH-2.0-OpenSSH_7.4","ssh_client_version":"SSH-2.0-OpenSSH_9.0","ssh_version":2,"ssh_hassh":"0","ssh_kex_alg":"curve25519-sha256","ssh_host_key_alg":"ssh-ed25519","ssh_cipher_alg":"chacha20-poly1305@openssh.com","ssh_mac_alg":"umac-64-etm@openssh.com","ssh_compression_alg":"none","ssh_host_key":"0"} - - - - - +{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240001,"decoded_as":"BASE","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.1","server_ip":"192.0.2.1","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]"} +{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240002,"decoded_as":"HTTP","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.2","server_ip":"192.0.2.2","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.HTTP","server_fqdn":"static-pcs-sdk-server.test.com","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","packet_capture_file":"123e4567-e89b-12d3-a456-426614174001","http_request_body":"123e4567-e89b-12d3-a456-426614174002","http_response_body":"123e4567-e89b-12d3-a456-426614174003","http_version":"http1","http_request_line":"POST /a HTTP/1.1","http_user_agent":"WinHttpClient","http_request_content_length":0,"http_host":"static-pcs-sdk-server.test.com","http_url":"static-pcs-sdk-server.test.com/a","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"application/json;charset=UTF-8","http_response_content_length":0,"http_response_latency_ms":0,"http_session_duration_ms":0,"http_sequence":1} +{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240003,"decoded_as":"SSL","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.3","server_ip":"192.0.2.3","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.ssl","server_fqdn":"storeedgefd.dsx.mp.microsoft.com","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","ssl_ja3_hash":"","ssl_esni_flag":0,"ssl_ech_flag":0,"ssl_sni":"storeedgefd.dsx.mp.microsoft.com","ssl_ja3s_hash":"","ssl_version":"v3","ssl_cn":"sfdataservice.microsoft.com","ssl_cert_issuer":"","ssl_cert_subject":"","ssl_san":"","ssl_handshake_latency_ms":0} +{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240004,"decoded_as":"DNS","ip_protocol":"udp","address_type":4,"client_ip":"10.0.0.4","server_ip":"192.0.2.4","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.UDP.dns","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","dns_response_latency_ms":0,"dns_qr":1,"dns_aa":0,"dns_message_id":47610,"dns_opcode":0,"dns_ra":1,"dns_rcode":0,"dns_rd":1,"dns_tc":0,"dns_qdcount":1,"dns_ancount":1,"dns_nscount":6,"dns_arcount":13,"dns_qname":"","dns_qtype":1,"dns_qclass":1} +{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240005,"decoded_as":"QUIC","ip_protocol":"udp","address_type":4,"client_ip":"10.0.0.5","server_ip":"192.0.2.5","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.UDP.quic","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","quic_version":"IETF QUIC RFC9000"} +{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240006,"decoded_as":"MAIL","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.6","server_ip":"192.0.2.6","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.mail.imap","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","mail_protocol_type":"IMAP","mail_eml_file":"123e4567-e89b-12d3-a456-426614174004"} +{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240007,"decoded_as":"SSH","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.7","server_ip":"192.0.2.7","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.ssh","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","ssh_server_version":"SSH-2.0-OpenSSH_7.4","ssh_client_version":"SSH-2.0-OpenSSH_9.0","ssh_version":2,"ssh_hassh":"0","ssh_kex_alg":"curve25519-sha256","ssh_host_key_alg":"ssh-ed25519","ssh_cipher_alg":"chacha20-poly1305@openssh.com","ssh_mac_alg":"umac-64-etm@openssh.com","ssh_compression_alg":"none","ssh_host_key":"0"} diff --git a/24.05/e2e-mockdata-generator/e2e-mockdata-generator.jar b/24.05/e2e-mockdata-generator/e2e-mockdata-generator.jar index b9c6127..8e6f26c 100644 Binary files a/24.05/e2e-mockdata-generator/e2e-mockdata-generator.jar and b/24.05/e2e-mockdata-generator/e2e-mockdata-generator.jar differ