diff --git a/24.06/bin/e2e_test.sh b/24.06/bin/e2e_test.sh index 0d47f15..ddb3d64 100755 --- a/24.06/bin/e2e_test.sh +++ b/24.06/bin/e2e_test.sh @@ -12,17 +12,22 @@ E2E_MOCKDATA_GENERATOR_PATH="$BASE_DIR/e2e-mockdata-generator" CONFIG_PATH="$BASE_DIR/config" COLLECTION="tsg-olap-e2e-test-collection.json" ENVIRONMENT="$BASE_DIR/environment.json" -DATA_CENTER="tsg_olap_dc_a" # Default Data Center +DATA_CENTER="tsg_olap" # Default Data Center OTHER_VAR="" +current=`date "+%Y-%m-%d %H:%M:%S"` +timeStamp=`date -d "$current" +%s` +currentTimeStamp=$((timeStamp*1000+10#`date "+%N"`/1000000)) # data_center_name:kafka_server -declare -A KAFKA_SERVERS=( +declare -A KAFKA_SERVERS=( + ["tsg_olap"]="192.168.44.11:9092" ["tsg_olap_dc_a"]="192.168.44.11:9092" ["tsg_olap_dc_b"]="192.168.44.11:9092" ) # data_center_name:hos_endpoint -declare -A HOS_ENDPOINTS=( +HOS_ENDPOINTS=( + ["tsg_olap"]="192.168.44.11" ["tsg_olap_dc_a"]="192.168.44.11" ["tsg_olap_dc_b"]="192.168.44.11" ) @@ -58,10 +63,14 @@ generate_metrics() { ["POLICY-RULE-METRIC"]="policy_rule_metric.dat" ["OBJECT-STATISTICS-METRIC"]="object_statistics_metric.dat" ["STATISTICS-RULE-METRIC"]="statistics_rule_metric.dat" + ["TRAFFIC-SKETCH-METRIC"]="traffic_sketch_metric.dat" ) for topic in "${!METRICS[@]}"; do # Replace the "data_center" value in the file sed -i "s/\"data_center\":\"[^\"]*\"/\"data_center\":\"$data_center\"/g" ./datasets/metrics/${METRICS[$topic]} + # Replace the "timestamp_ms" value in the file + sed -i "s/\"timestamp_ms\":[0-9]*/\"timestamp_ms\":$currentTimeStamp/" ./datasets/metrics/${METRICS[$topic]} + java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --bootstrap_server $bootstrap_server --topic $topic -f ./datasets/metrics/${METRICS[$topic]} done } @@ -120,7 +129,6 @@ run_diagnostic() { newman run $CONFIG_PATH/tsg-olap-e2e-test-collection.json -g $CONFIG_PATH/globals.json -n 1 -e $CONFIG_PATH/environment.json --delay-request 500 --timeout-script 10000 --timeout-request 300000 --timeout 3600000 --insecure $verbose --ignore-redirects $env_var_str --folder $folder $emojitrain } - clear_data() { log_message "Clearing test data..." newman run $CONFIG_PATH/tsg-olap-e2e-test-collection.json -g $CONFIG_PATH/globals.json -n 1 --delay-request 500 -e $CONFIG_PATH/environment.json --ignore-redirects --folder clear_test_data -r emojitrain diff --git a/24.06/config/tsg-olap-e2e-test-collection.json b/24.06/config/tsg-olap-e2e-test-collection.json index 4255ec2..89c27f6 100644 --- a/24.06/config/tsg-olap-e2e-test-collection.json +++ b/24.06/config/tsg-olap-e2e-test-collection.json @@ -1,7 +1,7 @@ { "info": { "_postman_id": "8d632c51-8d7b-426a-ae7d-baaf5d41dca3", - "name": "tsg-olap-e2e-test-24.05", + "name": "tsg-olap-e2e-test-24.06", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", "_exporter_id": "8105037" }, @@ -9,6 +9,70 @@ { "name": "logs", "item": [ + { + "name": "session_record_test", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.not.be.error;", + " pm.response.to.be.json;", + " const responseJson = pm.response.json();", + " pm.expect(Number(responseJson.data[0].cnt)).to.at.least(7);", + "});", + "", + "", + "" + ], + "type": "text/javascript", + "packages": {} + } + }, + { + "listen": "prerequest", + "script": { + "exec": [ + "" + ], + "type": "text/javascript", + "packages": {} + } + } + ], + "protocolProfileBehavior": { + "followOriginalHttpMethod": false, + "followRedirects": false + }, + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from session_record where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\",\n \"output_mode\":\"json\",\n \"execution_mode\":\"oneshot\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://{{qgw_ip}}:{{qgw_port}}/v1/query/sql", + "protocol": "http", + "host": [ + "{{qgw_ip}}" + ], + "port": "{{qgw_port}}", + "path": [ + "v1", + "query", + "sql" + ] + } + }, + "response": [] + }, { "name": "proxy_event_test", "event": [ @@ -264,70 +328,6 @@ } }, "response": [] - }, - { - "name": "session_record_test", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test(\"Status code is 200\", function () {", - " pm.response.to.not.be.error;", - " pm.response.to.be.json;", - " const responseJson = pm.response.json();", - " pm.expect(Number(responseJson.data[0].cnt)).to.at.least(7);", - "});", - "", - "", - "" - ], - "type": "text/javascript", - "packages": {} - } - }, - { - "listen": "prerequest", - "script": { - "exec": [ - "" - ], - "type": "text/javascript", - "packages": {} - } - } - ], - "protocolProfileBehavior": { - "followOriginalHttpMethod": false, - "followRedirects": false - }, - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from session_record where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\",\n \"output_mode\":\"json\",\n \"execution_mode\":\"oneshot\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "http://{{qgw_ip}}:{{qgw_port}}/v1/query/sql", - "protocol": "http", - "host": [ - "{{qgw_ip}}" - ], - "port": "{{qgw_port}}", - "path": [ - "v1", - "query", - "sql" - ] - } - }, - "response": [] } ] }, @@ -375,7 +375,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from traffic_general_stat where __time = {{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from traffic_general_stat where __time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -439,7 +439,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from application_protocol_stat where __time = {{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from application_protocol_stat where __time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -503,7 +503,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center,count(*) as cnt from statistics_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center,count(*) as cnt from statistics_rule_hits where __time>={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -567,7 +567,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from security_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from security_rule_hits where __time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -627,7 +627,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from monitor_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from monitor_rule_hits where __time>={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -687,7 +687,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center,count(*) as cnt from proxy_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center,count(*) as cnt from proxy_rule_hits where __time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -747,7 +747,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from service_chaining_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from service_chaining_rule_hits where __time>={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -807,7 +807,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from object_statistics where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from object_statistics where __time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -867,7 +867,67 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from statistics_rule where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from statistics_rule where __time>={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://{{qgw_ip}}:{{qgw_port}}/v1/query/sql", + "protocol": "http", + "host": [ + "{{qgw_ip}}" + ], + "port": "{{qgw_port}}", + "path": [ + "v1", + "query", + "sql" + ] + } + }, + "response": [] + }, + { + "name": "traffic_sketch_metric_test", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.not.be.error;", + " pm.response.to.be.json;", + " const responseJson = pm.response.json();", + " pm.expect(Number(responseJson.data[0].cnt)).to.at.least(1);", + "});", + "", + "", + "" + ], + "type": "text/javascript", + "packages": {} + } + }, + { + "listen": "prerequest", + "script": { + "exec": [ + "" + ], + "type": "text/javascript", + "packages": {} + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from traffic_sketch_metric where recv_time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" diff --git a/24.06/e2e-mockdata-generator/datasets/metrics/traffic_sketch_metric.dat b/24.06/e2e-mockdata-generator/datasets/metrics/traffic_sketch_metric.dat new file mode 100644 index 0000000..4f80dbc --- /dev/null +++ b/24.06/e2e-mockdata-generator/datasets/metrics/traffic_sketch_metric.dat @@ -0,0 +1 @@ +{"name":"traffic_sketch_metric","tags":{"direction":"Outbound","c2s_link_id":0,"s2c_link_id":0,"ip_protocol":"","client_ip":"0.0.0.0","server_ip":"0.0.0.0","server_country":"","client_asn":0,"server_asn":0,"server_fqdn":"","server_domain":"","app":"","app_category":"","c2s_ttl":0,"s2c_ttl":0,"device_id":"0000000000000000","device_group":"tsg_olap","data_center":"tsg_olap","vsys_id":1024},"fields":{"sessions":0,"bytes":0,"sent_bytes":0,"received_bytes":0,"pkts":0,"sent_pkts":0,"received_pkts":0},"timestamp_ms":946681200000} diff --git a/troubleshooting.tar.gz b/troubleshooting.tar.gz new file mode 100644 index 0000000..d01a75d Binary files /dev/null and b/troubleshooting.tar.gz differ