gfwleak/dongxiaoyan-tsg-autotest
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

统计日志自动化代码

Browse Source
This commit is contained in:
yyq
2020-05-26 19:54:19 +08:00
parent 527d32c491
commit 854a9add36
8 changed files with 128 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
01-TestCase/tsg_bfapi/api_log/LogStatistics-Country.robot
View File

@@ -9,9 +9,9 @@ Statistics-Country-00001
#请求地址 例如：/report/domain/recommend
${url} set variable /report/country/sourceipnum
#开始时间格式Y-M-D h:i:s
${startTime} set variable 2020-4-11 13:36:35.000
${startTime} set variable 2020-05-10 15:56:05
#结束时间格式Y-M-D h:i:s
${endTime} set variable 2020-4-30 13:36:35.000
${endTime} set variable 2020-05-25 15:56:35
#limit int型
${limit} set variable 50
GetApi ${apiName} ${url} startTime=${startTime} endTime=${endTime} limit=${limit}

2
01-TestCase/tsg_bfapi/api_log/LogStatistics-domain.robot
View File

@@ -8,5 +8,5 @@ Statistics-domain-00001
#请求地址 例如：/report/domain/recommend
${url} set variable /report/domain/recommend
# \ 统计时间， \ 查询最近24小时数据；例如当前时间为2020-03-26 15:14:00, 查询时间应设为推前1小时的整点，实际传入参数为 "2020-03-26 14:00:00"
${statisticTime} set variable 2020-04-26 23:00:00
${statisticTime} set variable 2020-05-10 15:56:05
GetApi ${apiName} ${url} statisticTime=${statisticTime}

19
01-TestCase/tsg_bfapi/api_log/LogStatistics-domain_resource.robot Normal file
View File

@@ -0,0 +1,19 @@
*** Settings ***
Library DateTime
Resource ../../../02-Keyword/tsg_bfapi/LogStatistics.robot
*** Test Cases ***
LogStatistics-domain_resource-00001
#apiName，请求大数据时，用于区分每个接口的sql
${apiName} set variable domain_resource
#请求地址 例如：/report/domain/recommend
${url} set variable /report/domain/resource/analyze
#结束时间格式Y-M-D h:i:s
${Time} get current date
${endTime}= add time to date ${Time} -00:05:00
${endTime} Get Substring ${endTime} \ -4
#开始时间格式Y-M-D h:i:s
${startTime}= add time to date ${endTime} -30 days
${startTime} Get Substring ${startTime} \ -4
${policyId} set variable 0
GetApi ${apiName} ${url} startTime=${startTime} endTime=${endTime} policy_id=${policyId}

4
01-TestCase/tsg_bfapi/api_log/LogStatistics-hijack_srcip_location.robot
View File

@@ -9,9 +9,9 @@ Statistics-hijack_srcip_location-00001
#请求地址 例如：/report/domain/recommend
${url} set variable /report/hijack_srcip_location
#开始时间格式Y-M-D h:i:s
${startTime} set variable 2020-04-01 16:43:48
${startTime} set variable 2020-04-26 16:40:01
#结束时间格式Y-M-D h:i:s
${endTime} set variable 2020-04-26 16:40:01
${endTime} set variable 2020-05-26 10:57:47
#策略ID
${policy_id} set variable 0
GetApi ${apiName} ${url} startTime=${startTime} endTime=${endTime} policy_id=${policy_id}

30
01-TestCase/tsg_bfapi/api_log/LogStatistics-hijack_srcip_num.robot Normal file
View File

@@ -0,0 +1,30 @@
*** Settings ***
Resource ../../../02-Keyword/tsg_bfapi/LogStatistics.robot
*** Test Cases ***
LogStatistics-hijack_scrip_num-00001
#120未更新
#apiName，请求大数据时，用于区分每个接口的sql
${apiName} set variable hijack_srcip_num
#请求地址 例如：/report/domain/recommend
${url} set variable /report/hijack_srcip_num
#开始时间格式Y-M-D h:i:s
${startTime} set variable 2020-04-26 16:40:01
#结束时间格式Y-M-D h:i:s
${endTime} set variable 2020-05-26 10:57:47
#策略ID列表，多个逗号分隔
${policy_ids} set variable 0
GetApi ${apiName} ${url} startTime=${startTime} endTime=${endTime} policy_ids=${policy_ids}
LogStatistics-hijack_scrip_num-00002
#apiName，请求大数据时，用于区分每个接口的sql
${apiName} set variable hijack_srcip_num
#请求地址 例如：/report/domain/recommend
${url} set variable /report/hijack_srcip_num
#开始时间格式Y-M-D h:i:s
${startTime} set variable 2020-04-26 16:40:01
#结束时间格式Y-M-D h:i:s
${endTime} set variable 2020-05-26 10:57:47
#策略ID列表，多个逗号分隔
${policy_ids} set variable 0,1
GetApi ${apiName} ${url} startTime=${startTime} endTime=${endTime} policy_ids=${policy_ids}

21
01-TestCase/tsg_bfapi/api_log/LogStatistics-injectionstatistics.robot Normal file
View File

@@ -0,0 +1,21 @@
*** Settings ***
Resource ../../../02-Keyword/tsg_bfapi/LogStatistics.robot
*** Test Cases ***
LogStatistics-injectionstatistics-00001
#120未更新
#apiName，请求大数据时，用于区分每个接口的sql
${apiName} set variable injectionstatistics
#请求地址 例如：/report/domain/recommend
${url} set variable /report/traffic/injectionstatistics
#开始时间格式Y-M-D h:i:s
${startTime} set variable 2020-04-26 16:40:01
#结束时间格式Y-M-D h:i:s
${endTime} set variable 2020-05-26 10:57:47
#返回结果记录数
${limit} set variable 50
#设备id，多个id逗号分隔 \ 支持设备列表查询 ，后台查询条件如：device_id in ($device_ids)
${device_ids} set variable CBT2201925000001
#统计时间粒度，单位秒，最小统计粒度为30秒
${timeGranularity} set variable 30
GetApi ${apiName} ${url} startTime=${startTime} endTime=${endTime} timeGranularity=${timeGranularity} limit=${limit} #device_ids=${device_ids}

19
01-TestCase/tsg_bfapi/api_log/LogStatistics-topn.robot Normal file
View File

@@ -0,0 +1,19 @@
*** Settings ***
Resource ../../../02-Keyword/tsg_bfapi/LogStatistics.robot
*** Test Cases ***
LogStatistics-topn-00001
#131已通过
#apiName，请求大数据时，用于区分每个接口的sql
${apiName} set variable topn
#请求地址 例如：/report/domain/recommend
${url} set variable /report/traffic/topn/domain
#开始时间格式Y-M-D h:i:s
${startTime} set variable 2020-04-26 18:38:07
#结束时间格式Y-M-D h:i:s
${endTime} set variable 2020-05-26 18:38:20
#排序依据： \ sessions、bytes、packets
${orderBy} set variable sessions
#limit int型
${limit} set variable 50
GetApi ${apiName} ${url} startTime=${startTime} endTime=${endTime} orderBy=${orderBy} limit=${limit}

42
02-Keyword/tsg_bfapi/LogStatistics.robot
View File

@@ -6,7 +6,8 @@ Library Collections
*** Keywords ***
GetApi
[Arguments] ${apiName} ${url} ${startTime}=${None} ${endTime}=${None} ${limit}=50000 ${statisticTime}=${None} ${statisticsUnit}=${None} ${timeGranularity}=${None} ${policy_id}=${None} ${ip}=${None}
[Arguments] ${apiName} ${url} ${startTime}=${None} ${endTime}=${None} ${limit}=50000 ${statisticTime}=${None} ${statisticsUnit}=${None} ${timeGranularity}=${None} ${policy_id}=${None} ${ip}=${None} ${device_ids}=${None} ${policy_ids}=${None} ${orderBy}=${None}
log policy_ids:${policy_ids}
#拼接查询条件
${strstrartTime}= run keyword if '${startTime}'!='${None}' Set Variable startTime=${startTime}&
... ELSE Set Variable ${EMPTY}
@@ -20,11 +21,21 @@ GetApi
... ELSE Set Variable ${EMPTY}
${strtimeGranularity}= run keyword if '${timeGranularity}'!='${None}' Set Variable timeGranularity=${timeGranularity}&
... ELSE Set Variable ${EMPTY}
${strpolicy_id} run keyword if '${policy_id}'!='${None}' Set Variable policy_id=${policy_id}&
${strpolicy_id}= run keyword if '${policy_id}'!='${None}' Set Variable policy_id=${policy_id}&
... ELSE Set Variable ${EMPTY}
${strip} run keyword if '${ip}'!='${None}' Set Variable ip=${ip}&
${strip}= run keyword if '${ip}'!='${None}' Set Variable ip=${ip}&
... ELSE Set Variable ${EMPTY}
${condition} catenate SEPARATOR= ${strstrartTime} ${strendTime} ${strstatisticTime} ${strlimit} ${strstatisticsUnit} ${strtimeGranularity} ${strpolicy_id} ${strip}
${strdevice_ids}= run keyword if '${device_ids}'!='${None}' Set Variable device_ids=${device_ids}&
... ELSE Set Variable ${EMPTY}
${strpolicy_ids}= run keyword if '${policy_ids}'!='${None}' Set Variable policy_ids=${policy_ids}&
... ELSE Set Variable ${EMPTY}
${strorderBy}= run keyword if '${orderBy}'!='${None}' Set Variable orderBy=${orderBy}&
... ELSE Set Variable ${EMPTY}
#替换policy_id为policyId
${strpolicy_id}= run keyword if '${apiName}'=='domain_resource' replace string ${strpolicy_id} policy_id policyId
... ELSE Set Variable ${strpolicy_id}
#bifang请求信息
${condition} catenate SEPARATOR= ${strstrartTime} ${strendTime} ${strstatisticTime} ${strlimit} ${strstatisticsUnit} ${strtimeGranularity} ${strpolicy_id} ${strip} ${strdevice_ids} ${strpolicy_ids} ${strorderBy}
#创建请求信息
${header} Create Dictionary Content-Type=application/x-www-form-urlencoded Authorization=${token}
Create Session api http://${host}:${port}/${version}
@@ -45,15 +56,25 @@ GetApi
... ELSE Set Variable ${EMPTY}
${strtimeGranularity}= run keyword if '${timeGranularity}'!='${None}' Set Variable timeGranularity=${timeGranularity} and
... ELSE Set Variable ${EMPTY}
${strpolicy_id} run keyword if '${policy_id}'!='${None}' Set Variable policy_id=${policy_id} and
${strpolicy_id}= run keyword if '${policy_id}'!='${None}' Set Variable policy_id=${policy_id} and
... ELSE Set Variable ${EMPTY}
${strip} run keyword if '${ip}'!='${None}' Set Variable ip=${ip} and
${strip}= run keyword if '${ip}'!='${None}' Set Variable ip=${ip} and
... ELSE Set Variable ${EMPTY}
${strdevice_ids}= run keyword if '${device_ids}'!='${None}' Set Variable device_ids=${device_ids} and
... ELSE Set Variable ${EMPTY}
#sql拼接 ${where}
${sqlwhere} catenate ${strstrartTime} ${strendTime} ${strstatisticTime} ${strstatisticsUnit} ${strtimeGranularity} ${strpolicy_id} ${strip}
${sqlwhere} catenate ${strstrartTime} ${strendTime} ${strstatisticTime} ${strstatisticsUnit} ${strtimeGranularity} ${strpolicy_id} ${strip} ${strdevice_ids}
${strippedwhere}= Strip String ${sqlwhere} mode=right
#截取最后一个and
${where}= Get Substring ${strippedwhere} \ -4
#injectionstatics 时间粒度默认为30
${timeGranularity}= run keyword if '${timeGranularity}'=="${None}" and '${apiName}'=="injectionstatics" set variable 30
... ELSE set variable ${timeGranularity}
log ${apiName}
#topn排序依据： \ sessions、totalbytes、totalpackets
${TotalOrderBy}= run keyword if '${orderBy}'=="bytes" and '${apiName}'=="topn" set variable totalBytes
... ELSE IF '${orderBy}'=="packets" and '${apiName}'=="topn" set variable totalPackets
... ELSE IF '${orderBy}'=="sessions" and '${apiName}'=="topn" set variable sessions
#BigData
${sql}= run keyword if '${apiName}'=="country" Set Variable SELECT country, APPROX_COUNT_DISTINCT_DS_HLL(ip_object) as num FROM source_country_ip_num_log where ${where} group by country order by num desc limit ${limit}
... ELSE IF '${apiName}'=="domain" Set Variable SELECT domain, protocol_type, ip_num, session_num \ FROM top_domain_recommend_daily_log where __time = '${statisticTime}'
@@ -64,7 +85,12 @@ GetApi
... ELSE IF '${apiName}'=="traffic" and '${statisticsUnit}'=='${None}' Set Variable select sum(total_in_bytes) as trafficInBytes, sum(total_out_bytes) as trafficOutBytes, \ sum(total_in_bytes+total_out_bytes) as trafficTotalBytes, sum(total_in_packets) as trafficInPackets,sum(total_out_packets) as trafficOutPackets,sum(total_in_packets+total_out_packets) as trafficTotalPackets, sum(new_conn_num) as sessions from traffic_metrics_log where __time >= '${startTime}' and __time < '${endTime}'
... ELSE IF '${apiName}'=="ip_correlation_domain" and '${ip}'!='${None}' Set Variable SELECT \ ip, FROM_UNIXTIME(max(stat_time)) as last_time, FROM_UNIXTIME(min(stat_time)) as first_time, groupUniqArrayMerge(domain_list) as domain_list, groupUniqArrayMerge(port_list) as port_list \ FROM security_ip_info where \ stat_time >= '${startTime}' and stat_time < '${endTime}' and ip='${ip}' group by ip
... ELSE IF '${apiName}'=="ip_correlation_domain" and ${ip}==${None} Set Variable SELECT ip,FROM_UNIXTIME(max(stat_time)) as last_time, FROM_UNIXTIME(min(stat_time)) as first_time, groupUniqArrayMerge(domain_list) as domain_list, groupUniqArrayMerge(port_list) as port_list \ FROM security_ip_info where \ stat_time >= '${startTime}' and stat_time < '${endTime}' \ group by ip
... ELSE IF '${apiName}'=="hijack_srcip_num" Set Variable SELECT policy_id,country,location, APPROX_COUNT_DISTINCT_DS_HLL(ip_object) as num \ FROM proxy_event_hits_log where \ __time >='${startTime}' and __time <'${endTime}' and policy_id in \ ( ${policy_ids} ) group by policy_id , country,location order by num desc
... ELSE IF '${apiName}'=="hijack_srcip_location" Set Variable select ip, IP_TO_CITY(ip) as location, IP_TO_GEO(ip) as geo from (SELECT policy_id, arrayJoin(groupUniqArrayMerge(ip_list)) as ip FROM proxy_ip_info where \ stat_time >= '${startTime}' and stat_time < '${endTime}' and policy_id = ${policy_id} group by policy_id )
... ELSE IF '${apiName}'=="domain_resource" Set Variable SELECT \ domain, groupUniqArrayMerge(ip_list) as ip_list, groupUniqArrayMerge(cdn_list) as cdn_list, groupUniqArrayMerge(protocol_type_list) as protocol_type_list, groupUniqArrayMerge(port_list) as port_list \ FROM security_website_domain_info where \ stat_time >= '${startTime}' and stat_time < '${endTime}' and policy_id=${policy_id} group by domain
... ELSE IF '${apiName}'=="injectionstatistics" Set Variable select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/${timeGranularity})*${timeGranularity}),'%Y-%m-%d %H:%i:%s') as stat_time, \ sum(bytes) as bytes from injection_metrics_log \ where __time >= '${startTime}' and __time < '${endTime}' group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/${timeGranularity})*${timeGranularity}),'%Y-%m-%d %H:%i:%s') limit ${limit}
... ELSE IF '${apiName}'=="topn" Set Variable select domain,sum(session_num) as sessions,sum(c2s_byte_num) as sentBytes,sum(s2c_byte_num) as receivedBytes,sum(c2s_byte_num+s2c_byte_num) as totalBytes,sum(c2s_pkt_num) as sentPackets,sum(s2c_pkt_num) as receivedPackets,sum(c2s_pkt_num+s2c_pkt_num) as totalPackets from top_website_domain_log where __time >= '${startTime}' \ and __time < '${endTime}' \ and order_by='${orderBy}' \ group by domain order by ${TotalOrderBy} desc limit ${limit}
log ${sql}
log sql------------${sql}
#traffic二次请求sql
${sql2}= run keyword if '${apiName}'=='traffic' SplicingTrafficSql ${apiName} ${startTime} ${endTime} statisticsUnit=${statisticsUnit} timeGranularity=${timeGranularity} limit=${limit}
@@ -82,7 +108,7 @@ GetApi
... AND Should Be Equal As Strings ${bigdata2}[status] 200
#traffic二次请求json拼接与毕方一致
${bigdataTraffic}= run keyword if '${apiName}'=="traffic" SplicingTrafficJson ${bigdata} ${bigdata2}
${bifanglist} run keyword if '${apiName}'=="ip_correlation_domain" set variable ${bifang['data']['list']['list']}
${bifanglist} run keyword if '${apiName}'=="ip_correlation_domain" \ or '${apiName}'=='domain_resource' set variable ${bifang['data']['list']['list']}
... ELSE IF '${apiName}'=="traffic" set variable ${bifang['data']}
... ELSE set variable ${bifang['data']['list']}
${bigdatalist} run keyword if '${apiName}'=="traffic" set variable ${bigdataTraffic}