gfwleak/galaxy-platform-galaxy-troubleshooting-api
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Metric datasets use current unix timestamp and add traffic skecth metric dataset

Browse Source
This commit is contained in:
doufenghu
2024-06-28 19:48:02 +08:00
parent 42a0ada497
commit 9afd92cecf
4 changed files with 147 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
24.06/bin/e2e_test.sh
View File

@@ -12,17 +12,22 @@ E2E_MOCKDATA_GENERATOR_PATH="$BASE_DIR/e2e-mockdata-generator"
CONFIG_PATH="$BASE_DIR/config"
COLLECTION="tsg-olap-e2e-test-collection.json"
ENVIRONMENT="$BASE_DIR/environment.json"
DATA_CENTER="tsg_olap_dc_a" # Default Data Center
DATA_CENTER="tsg_olap" # Default Data Center
OTHER_VAR=""
current=`date "+%Y-%m-%d %H:%M:%S"`
timeStamp=`date -d "$current" +%s`
currentTimeStamp=$((timeStamp*1000+10#`date "+%N"`/1000000))
# data_center_name:kafka_server
declare -A KAFKA_SERVERS=(
declare -A KAFKA_SERVERS=(
["tsg_olap"]="192.168.44.11:9092"
["tsg_olap_dc_a"]="192.168.44.11:9092"
["tsg_olap_dc_b"]="192.168.44.11:9092"
)
# data_center_name:hos_endpoint
declare -A HOS_ENDPOINTS=(
HOS_ENDPOINTS=(
["tsg_olap"]="192.168.44.11"
["tsg_olap_dc_a"]="192.168.44.11"
["tsg_olap_dc_b"]="192.168.44.11"
)
@@ -58,10 +63,14 @@ generate_metrics() {
["POLICY-RULE-METRIC"]="policy_rule_metric.dat"
["OBJECT-STATISTICS-METRIC"]="object_statistics_metric.dat"
["STATISTICS-RULE-METRIC"]="statistics_rule_metric.dat"
["TRAFFIC-SKETCH-METRIC"]="traffic_sketch_metric.dat"
)
for topic in "${!METRICS[@]}"; do
# Replace the "data_center" value in the file
sed -i "s/\"data_center\":\"[^\"]*\"/\"data_center\":\"$data_center\"/g" ./datasets/metrics/${METRICS[$topic]}
# Replace the "timestamp_ms" value in the file
sed -i "s/\"timestamp_ms\":[0-9]*/\"timestamp_ms\":$currentTimeStamp/" ./datasets/metrics/${METRICS[$topic]}
java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --bootstrap_server $bootstrap_server --topic $topic -f ./datasets/metrics/${METRICS[$topic]}
done
}
@@ -120,7 +129,6 @@ run_diagnostic() {
newman run $CONFIG_PATH/tsg-olap-e2e-test-collection.json -g $CONFIG_PATH/globals.json -n 1 -e $CONFIG_PATH/environment.json --delay-request 500 --timeout-script 10000 --timeout-request 300000 --timeout 3600000 --insecure $verbose --ignore-redirects $env_var_str --folder $folder $emojitrain
}
clear_data() {
log_message "Clearing test data..."
newman run $CONFIG_PATH/tsg-olap-e2e-test-collection.json -g $CONFIG_PATH/globals.json -n 1 --delay-request 500 -e $CONFIG_PATH/environment.json --ignore-redirects --folder clear_test_data -r emojitrain

208
24.06/config/tsg-olap-e2e-test-collection.json
View File

@@ -1,7 +1,7 @@
{
"info": {
"_postman_id": "8d632c51-8d7b-426a-ae7d-baaf5d41dca3",
"name": "tsg-olap-e2e-test-24.05",
"name": "tsg-olap-e2e-test-24.06",
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
"_exporter_id": "8105037"
},
@@ -9,6 +9,70 @@
{
"name": "logs",
"item": [
{
"name": "session_record_test",
"event": [
{
"listen": "test",
"script": {
"exec": [
"pm.test(\"Status code is 200\", function () {",
" pm.response.to.not.be.error;",
" pm.response.to.be.json;",
" const responseJson = pm.response.json();",
" pm.expect(Number(responseJson.data[0].cnt)).to.at.least(7);",
"});",
"",
"",
""
],
"type": "text/javascript",
"packages": {}
}
},
{
"listen": "prerequest",
"script": {
"exec": [
""
],
"type": "text/javascript",
"packages": {}
}
}
],
"protocolProfileBehavior": {
"followOriginalHttpMethod": false,
"followRedirects": false
},
"request": {
"method": "POST",
"header": [],
"body": {
"mode": "raw",
"raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from session_record where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\",\n \"output_mode\":\"json\",\n \"execution_mode\":\"oneshot\"\n}",
"options": {
"raw": {
"language": "json"
}
}
},
"url": {
"raw": "http://{{qgw_ip}}:{{qgw_port}}/v1/query/sql",
"protocol": "http",
"host": [
"{{qgw_ip}}"
],
"port": "{{qgw_port}}",
"path": [
"v1",
"query",
"sql"
]
}
},
"response": []
},
{
"name": "proxy_event_test",
"event": [
@@ -264,70 +328,6 @@
}
},
"response": []
},
{
"name": "session_record_test",
"event": [
{
"listen": "test",
"script": {
"exec": [
"pm.test(\"Status code is 200\", function () {",
" pm.response.to.not.be.error;",
" pm.response.to.be.json;",
" const responseJson = pm.response.json();",
" pm.expect(Number(responseJson.data[0].cnt)).to.at.least(7);",
"});",
"",
"",
""
],
"type": "text/javascript",
"packages": {}
}
},
{
"listen": "prerequest",
"script": {
"exec": [
""
],
"type": "text/javascript",
"packages": {}
}
}
],
"protocolProfileBehavior": {
"followOriginalHttpMethod": false,
"followRedirects": false
},
"request": {
"method": "POST",
"header": [],
"body": {
"mode": "raw",
"raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from session_record where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\",\n \"output_mode\":\"json\",\n \"execution_mode\":\"oneshot\"\n}",
"options": {
"raw": {
"language": "json"
}
}
},
"url": {
"raw": "http://{{qgw_ip}}:{{qgw_port}}/v1/query/sql",
"protocol": "http",
"host": [
"{{qgw_ip}}"
],
"port": "{{qgw_port}}",
"path": [
"v1",
"query",
"sql"
]
}
},
"response": []
}
]
},
@@ -375,7 +375,7 @@
"header": [],
"body": {
"mode": "raw",
"raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from traffic_general_stat where __time = {{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from traffic_general_stat where __time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"options": {
"raw": {
"language": "json"
@@ -439,7 +439,7 @@
"header": [],
"body": {
"mode": "raw",
"raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from application_protocol_stat where __time = {{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from application_protocol_stat where __time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"options": {
"raw": {
"language": "json"
@@ -503,7 +503,7 @@
"header": [],
"body": {
"mode": "raw",
"raw": "{\n \"statement\" : \"select data_center,count(*) as cnt from statistics_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"raw": "{\n \"statement\" : \"select data_center,count(*) as cnt from statistics_rule_hits where __time>={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"options": {
"raw": {
"language": "json"
@@ -567,7 +567,7 @@
"header": [],
"body": {
"mode": "raw",
"raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from security_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from security_rule_hits where __time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"options": {
"raw": {
"language": "json"
@@ -627,7 +627,7 @@
"header": [],
"body": {
"mode": "raw",
"raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from monitor_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from monitor_rule_hits where __time>={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"options": {
"raw": {
"language": "json"
@@ -687,7 +687,7 @@
"header": [],
"body": {
"mode": "raw",
"raw": "{\n \"statement\" : \"select data_center,count(*) as cnt from proxy_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"raw": "{\n \"statement\" : \"select data_center,count(*) as cnt from proxy_rule_hits where __time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"options": {
"raw": {
"language": "json"
@@ -747,7 +747,7 @@
"header": [],
"body": {
"mode": "raw",
"raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from service_chaining_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from service_chaining_rule_hits where __time>={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"options": {
"raw": {
"language": "json"
@@ -807,7 +807,7 @@
"header": [],
"body": {
"mode": "raw",
"raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from object_statistics where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from object_statistics where __time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"options": {
"raw": {
"language": "json"
@@ -867,7 +867,67 @@
"header": [],
"body": {
"mode": "raw",
"raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from statistics_rule where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from statistics_rule where __time>={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"options": {
"raw": {
"language": "json"
}
}
},
"url": {
"raw": "http://{{qgw_ip}}:{{qgw_port}}/v1/query/sql",
"protocol": "http",
"host": [
"{{qgw_ip}}"
],
"port": "{{qgw_port}}",
"path": [
"v1",
"query",
"sql"
]
}
},
"response": []
},
{
"name": "traffic_sketch_metric_test",
"event": [
{
"listen": "test",
"script": {
"exec": [
"pm.test(\"Status code is 200\", function () {",
" pm.response.to.not.be.error;",
" pm.response.to.be.json;",
" const responseJson = pm.response.json();",
" pm.expect(Number(responseJson.data[0].cnt)).to.at.least(1);",
"});",
"",
"",
""
],
"type": "text/javascript",
"packages": {}
}
},
{
"listen": "prerequest",
"script": {
"exec": [
""
],
"type": "text/javascript",
"packages": {}
}
}
],
"request": {
"method": "POST",
"header": [],
"body": {
"mode": "raw",
"raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from traffic_sketch_metric where recv_time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}",
"options": {
"raw": {
"language": "json"

1
24.06/e2e-mockdata-generator/datasets/metrics/traffic_sketch_metric.dat Normal file
View File

@@ -0,0 +1 @@
{"name":"traffic_sketch_metric","tags":{"direction":"Outbound","c2s_link_id":0,"s2c_link_id":0,"ip_protocol":"","client_ip":"0.0.0.0","server_ip":"0.0.0.0","server_country":"","client_asn":0,"server_asn":0,"server_fqdn":"","server_domain":"","app":"","app_category":"","c2s_ttl":0,"s2c_ttl":0,"device_id":"0000000000000000","device_group":"tsg_olap","data_center":"tsg_olap","vsys_id":1024},"fields":{"sessions":0,"bytes":0,"sent_bytes":0,"received_bytes":0,"pkts":0,"sent_pkts":0,"received_pkts":0},"timestamp_ms":946681200000}

BIN
troubleshooting.tar.gz Normal file
View File

Binary file not shown.