统计日志自动化代码

01-TestCase/tsg_bfapi/api_log/LogStatistics-Country.robot

@@ -9,9 +9,9 @@ Statistics-Country-00001
     #请求地址 例如：/report/domain/recommend
     ${url}    set variable    /report/country/sourceipnum
     #开始时间格式Y-M-D h:i:s
-    ${startTime}    set variable    2020-4-11 13:36:35.000
+    ${startTime}    set variable    2020-05-10 15:56:05
     #结束时间格式Y-M-D h:i:s
-    ${endTime}    set variable    2020-4-30 13:36:35.000
+    ${endTime}    set variable    2020-05-25 15:56:35
     #limit int型
     ${limit}    set variable    50
     GetApi    ${apiName}    ${url}    startTime=${startTime}    endTime=${endTime}    limit=${limit}

01-TestCase/tsg_bfapi/api_log/LogStatistics-domain.robot

@@ -8,5 +8,5 @@ Statistics-domain-00001
     #请求地址 例如：/report/domain/recommend
     ${url}    set variable    /report/domain/recommend
     #    \ 统计时间， \ 查询最近24小时数据；例如当前时间为2020-03-26 15:14:00, 查询时间应设为推前1小时的整点，实际传入参数为 "2020-03-26 14:00:00"
-    ${statisticTime}    set variable    2020-04-26 23:00:00
+    ${statisticTime}    set variable    2020-05-10 15:56:05
     GetApi    ${apiName}    ${url}    statisticTime=${statisticTime}

01-TestCase/tsg_bfapi/api_log/LogStatistics-domain_resource.robot

@@ -0,0 +1,19 @@
+*** Settings ***
+Library           DateTime
+Resource          ../../../02-Keyword/tsg_bfapi/LogStatistics.robot
+
+*** Test Cases ***
+LogStatistics-domain_resource-00001
+    #apiName，请求大数据时，用于区分每个接口的sql
+    ${apiName}    set variable    domain_resource
+    #请求地址 例如：/report/domain/recommend
+    ${url}    set variable    /report/domain/resource/analyze
+    #结束时间格式Y-M-D h:i:s
+    ${Time}    get current date
+    ${endTime}=    add time to date    ${Time}    -00:05:00
+    ${endTime}    Get Substring    ${endTime}    \    -4
+    #开始时间格式Y-M-D h:i:s
+    ${startTime}=    add time to date    ${endTime}    -30 days
+    ${startTime}    Get Substring    ${startTime}    \    -4
+    ${policyId}    set variable    0
+    GetApi    ${apiName}    ${url}    startTime=${startTime}    endTime=${endTime}    policy_id=${policyId}

01-TestCase/tsg_bfapi/api_log/LogStatistics-hijack_srcip_location.robot

@@ -9,9 +9,9 @@ Statistics-hijack_srcip_location-00001
     #请求地址 例如：/report/domain/recommend
     ${url}    set variable    /report/hijack_srcip_location
     #开始时间格式Y-M-D h:i:s
-    ${startTime}    set variable    2020-04-01 16:43:48
+    ${startTime}    set variable    2020-04-26 16:40:01
     #结束时间格式Y-M-D h:i:s
-    ${endTime}    set variable    2020-04-26 16:40:01
+    ${endTime}    set variable    2020-05-26 10:57:47
     #策略ID
     ${policy_id}    set variable    0
     GetApi    ${apiName}    ${url}    startTime=${startTime}    endTime=${endTime}    policy_id=${policy_id}

01-TestCase/tsg_bfapi/api_log/LogStatistics-hijack_srcip_num.robot

@@ -0,0 +1,30 @@
+*** Settings ***
+Resource          ../../../02-Keyword/tsg_bfapi/LogStatistics.robot
+
+*** Test Cases ***
+LogStatistics-hijack_scrip_num-00001
+    #120未更新
+    #apiName，请求大数据时，用于区分每个接口的sql
+    ${apiName}    set variable    hijack_srcip_num
+    #请求地址 例如：/report/domain/recommend
+    ${url}    set variable    /report/hijack_srcip_num
+    #开始时间格式Y-M-D h:i:s
+    ${startTime}    set variable    2020-04-26 16:40:01
+    #结束时间格式Y-M-D h:i:s
+    ${endTime}    set variable    2020-05-26 10:57:47
+    #策略ID列表，多个逗号分隔
+    ${policy_ids}    set variable    0
+    GetApi    ${apiName}    ${url}    startTime=${startTime}    endTime=${endTime}    policy_ids=${policy_ids}
+
+LogStatistics-hijack_scrip_num-00002
+    #apiName，请求大数据时，用于区分每个接口的sql
+    ${apiName}    set variable    hijack_srcip_num
+    #请求地址 例如：/report/domain/recommend
+    ${url}    set variable    /report/hijack_srcip_num
+    #开始时间格式Y-M-D h:i:s
+    ${startTime}    set variable    2020-04-26 16:40:01
+    #结束时间格式Y-M-D h:i:s
+    ${endTime}    set variable    2020-05-26 10:57:47
+    #策略ID列表，多个逗号分隔
+    ${policy_ids}    set variable    0,1
+    GetApi    ${apiName}    ${url}    startTime=${startTime}    endTime=${endTime}    policy_ids=${policy_ids}

01-TestCase/tsg_bfapi/api_log/LogStatistics-injectionstatistics.robot

@@ -0,0 +1,21 @@
+*** Settings ***
+Resource          ../../../02-Keyword/tsg_bfapi/LogStatistics.robot
+
+*** Test Cases ***
+LogStatistics-injectionstatistics-00001
+    #120未更新
+    #apiName，请求大数据时，用于区分每个接口的sql
+    ${apiName}    set variable    injectionstatistics
+    #请求地址 例如：/report/domain/recommend
+    ${url}    set variable    /report/traffic/injectionstatistics
+    #开始时间格式Y-M-D h:i:s
+    ${startTime}    set variable    2020-04-26 16:40:01
+    #结束时间格式Y-M-D h:i:s
+    ${endTime}    set variable    2020-05-26 10:57:47
+    #返回结果记录数
+    ${limit}    set variable    50
+    #设备id，多个id逗号分隔 \ 支持设备列表查询 ，后台查询条件如：device_id in ($device_ids)
+    ${device_ids}    set variable    CBT2201925000001
+    #统计时间粒度，单位秒，最小统计粒度为30秒
+    ${timeGranularity}    set variable    30
+    GetApi    ${apiName}    ${url}    startTime=${startTime}    endTime=${endTime}    timeGranularity=${timeGranularity}    limit=${limit}    #device_ids=${device_ids}

01-TestCase/tsg_bfapi/api_log/LogStatistics-topn.robot

@@ -0,0 +1,19 @@
+*** Settings ***
+Resource          ../../../02-Keyword/tsg_bfapi/LogStatistics.robot
+
+*** Test Cases ***
+LogStatistics-topn-00001
+    #131已通过
+    #apiName，请求大数据时，用于区分每个接口的sql
+    ${apiName}    set variable    topn
+    #请求地址 例如：/report/domain/recommend
+    ${url}    set variable    /report/traffic/topn/domain
+    #开始时间格式Y-M-D h:i:s
+    ${startTime}    set variable    2020-04-26 18:38:07
+    #结束时间格式Y-M-D h:i:s
+    ${endTime}    set variable    2020-05-26 18:38:20
+    #排序依据： \ sessions、bytes、packets
+    ${orderBy}    set variable    sessions
+    #limit int型
+    ${limit}    set variable    50
+    GetApi    ${apiName}    ${url}    startTime=${startTime}    endTime=${endTime}    orderBy=${orderBy}    limit=${limit}

02-Keyword/tsg_bfapi/LogStatistics.robot

@@ -6,7 +6,8 @@ Library           Collections
 
 *** Keywords ***
 GetApi
-    [Arguments]    ${apiName}    ${url}    ${startTime}=${None}    ${endTime}=${None}    ${limit}=50000    ${statisticTime}=${None}    ${statisticsUnit}=${None}    ${timeGranularity}=${None}    ${policy_id}=${None}    ${ip}=${None}
+    [Arguments]    ${apiName}    ${url}    ${startTime}=${None}    ${endTime}=${None}    ${limit}=50000    ${statisticTime}=${None}    ${statisticsUnit}=${None}    ${timeGranularity}=${None}    ${policy_id}=${None}    ${ip}=${None}    ${device_ids}=${None}    ${policy_ids}=${None}    ${orderBy}=${None}
+    log    policy_ids:${policy_ids}
     #拼接查询条件
     ${strstrartTime}=    run keyword if    '${startTime}'!='${None}'    Set Variable    startTime=${startTime}&
     ...    ELSE    Set Variable    ${EMPTY}
@@ -20,11 +21,21 @@ GetApi
     ...    ELSE    Set Variable    ${EMPTY}
     ${strtimeGranularity}=    run keyword if    '${timeGranularity}'!='${None}'    Set Variable    timeGranularity=${timeGranularity}&
     ...    ELSE    Set Variable    ${EMPTY}
-    ${strpolicy_id}    run keyword if    '${policy_id}'!='${None}'    Set Variable    policy_id=${policy_id}&
+    ${strpolicy_id}=    run keyword if    '${policy_id}'!='${None}'    Set Variable    policy_id=${policy_id}&
     ...    ELSE    Set Variable    ${EMPTY}
-    ${strip}    run keyword if    '${ip}'!='${None}'    Set Variable    ip=${ip}&
+    ${strip}=    run keyword if    '${ip}'!='${None}'    Set Variable    ip=${ip}&
     ...    ELSE    Set Variable    ${EMPTY}
-    ${condition}    catenate    SEPARATOR=    ${strstrartTime}    ${strendTime}    ${strstatisticTime}    ${strlimit}    ${strstatisticsUnit}    ${strtimeGranularity}    ${strpolicy_id}    ${strip}
+    ${strdevice_ids}=    run keyword if    '${device_ids}'!='${None}'    Set Variable    device_ids=${device_ids}&
+    ...    ELSE    Set Variable    ${EMPTY}
+    ${strpolicy_ids}=    run keyword if    '${policy_ids}'!='${None}'    Set Variable    policy_ids=${policy_ids}&
+    ...    ELSE    Set Variable    ${EMPTY}
+    ${strorderBy}=    run keyword if    '${orderBy}'!='${None}'    Set Variable    orderBy=${orderBy}&
+    ...    ELSE    Set Variable    ${EMPTY}
+    #替换policy_id为policyId
+    ${strpolicy_id}=    run keyword if    '${apiName}'=='domain_resource'    replace string    ${strpolicy_id}    policy_id    policyId
+    ...    ELSE    Set Variable    ${strpolicy_id}
+    #bifang请求信息
+    ${condition}    catenate    SEPARATOR=    ${strstrartTime}    ${strendTime}    ${strstatisticTime}    ${strlimit}    ${strstatisticsUnit}    ${strtimeGranularity}    ${strpolicy_id}    ${strip}    ${strdevice_ids}    ${strpolicy_ids}    ${strorderBy}
     #创建请求信息
     ${header}    Create Dictionary    Content-Type=application/x-www-form-urlencoded    Authorization=${token}
     Create Session    api    http://${host}:${port}/${version}
@@ -45,15 +56,25 @@ GetApi
     ...    ELSE    Set Variable    ${EMPTY}
     ${strtimeGranularity}=    run keyword if    '${timeGranularity}'!='${None}'    Set Variable    timeGranularity=${timeGranularity} and
     ...    ELSE    Set Variable    ${EMPTY}
-    ${strpolicy_id}    run keyword if    '${policy_id}'!='${None}'    Set Variable    policy_id=${policy_id} and
+    ${strpolicy_id}=    run keyword if    '${policy_id}'!='${None}'    Set Variable    policy_id=${policy_id} and
     ...    ELSE    Set Variable    ${EMPTY}
-    ${strip}    run keyword if    '${ip}'!='${None}'    Set Variable    ip=${ip} and
+    ${strip}=    run keyword if    '${ip}'!='${None}'    Set Variable    ip=${ip} and
+    ...    ELSE    Set Variable    ${EMPTY}
+    ${strdevice_ids}=    run keyword if    '${device_ids}'!='${None}'    Set Variable    device_ids=${device_ids} and
     ...    ELSE    Set Variable    ${EMPTY}
     #sql拼接    ${where}
-    ${sqlwhere}    catenate    ${strstrartTime}    ${strendTime}    ${strstatisticTime}    ${strstatisticsUnit}    ${strtimeGranularity}    ${strpolicy_id}    ${strip}
+    ${sqlwhere}    catenate    ${strstrartTime}    ${strendTime}    ${strstatisticTime}    ${strstatisticsUnit}    ${strtimeGranularity}    ${strpolicy_id}    ${strip}    ${strdevice_ids}
     ${strippedwhere}=    Strip String    ${sqlwhere}    mode=right
     #截取最后一个and
     ${where}=    Get Substring    ${strippedwhere}    \    -4
+    #injectionstatics 时间粒度默认为30
+    ${timeGranularity}=    run keyword if    '${timeGranularity}'=="${None}" and '${apiName}'=="injectionstatics"     set variable    30
+    ...    ELSE    set variable    ${timeGranularity}
+    log    ${apiName}
+    #topn排序依据： \ sessions、totalbytes、totalpackets
+    ${TotalOrderBy}=    run keyword if    '${orderBy}'=="bytes" and '${apiName}'=="topn"     set variable    totalBytes
+    ...    ELSE IF    '${orderBy}'=="packets" and '${apiName}'=="topn"     set variable    totalPackets
+    ...    ELSE IF    '${orderBy}'=="sessions" and '${apiName}'=="topn"     set variable    sessions
     #BigData
     ${sql}=    run keyword if    '${apiName}'=="country"    Set Variable    SELECT country, APPROX_COUNT_DISTINCT_DS_HLL(ip_object) as num FROM source_country_ip_num_log where ${where} group by country order by num desc limit ${limit}
     ...    ELSE IF    '${apiName}'=="domain"    Set Variable    SELECT domain, protocol_type, ip_num, session_num \ FROM top_domain_recommend_daily_log where __time = '${statisticTime}'
@@ -64,7 +85,12 @@ GetApi
     ...    ELSE IF    '${apiName}'=="traffic" and '${statisticsUnit}'=='${None}'    Set Variable    select sum(total_in_bytes) as trafficInBytes, sum(total_out_bytes) as trafficOutBytes, \ sum(total_in_bytes+total_out_bytes) as trafficTotalBytes, sum(total_in_packets) as trafficInPackets,sum(total_out_packets) as trafficOutPackets,sum(total_in_packets+total_out_packets) as trafficTotalPackets, sum(new_conn_num) as sessions from traffic_metrics_log where __time >= '${startTime}' and __time < '${endTime}'
     ...    ELSE IF    '${apiName}'=="ip_correlation_domain" and '${ip}'!='${None}'    Set Variable    SELECT \ ip, FROM_UNIXTIME(max(stat_time)) as last_time, FROM_UNIXTIME(min(stat_time)) as first_time, groupUniqArrayMerge(domain_list) as domain_list, groupUniqArrayMerge(port_list) as port_list \ FROM security_ip_info where \ stat_time >= '${startTime}' and stat_time < '${endTime}' and ip='${ip}' group by ip
     ...    ELSE IF    '${apiName}'=="ip_correlation_domain" and ${ip}==${None}    Set Variable    SELECT ip,FROM_UNIXTIME(max(stat_time)) as last_time, FROM_UNIXTIME(min(stat_time)) as first_time, groupUniqArrayMerge(domain_list) as domain_list, groupUniqArrayMerge(port_list) as port_list \ FROM security_ip_info where \ stat_time >= '${startTime}' and stat_time < '${endTime}' \ group by ip
+    ...    ELSE IF    '${apiName}'=="hijack_srcip_num"    Set Variable    SELECT policy_id,country,location, APPROX_COUNT_DISTINCT_DS_HLL(ip_object) as num \ FROM proxy_event_hits_log where \ __time >='${startTime}' and __time <'${endTime}' and policy_id in \ ( ${policy_ids} ) group by policy_id , country,location order by num desc
     ...    ELSE IF    '${apiName}'=="hijack_srcip_location"    Set Variable    select ip, IP_TO_CITY(ip) as location, IP_TO_GEO(ip) as geo from (SELECT policy_id, arrayJoin(groupUniqArrayMerge(ip_list)) as ip FROM proxy_ip_info where \ stat_time >= '${startTime}' and stat_time < '${endTime}' and policy_id = ${policy_id} group by policy_id )
+    ...    ELSE IF    '${apiName}'=="domain_resource"    Set Variable    SELECT \ domain, groupUniqArrayMerge(ip_list) as ip_list, groupUniqArrayMerge(cdn_list) as cdn_list, groupUniqArrayMerge(protocol_type_list) as protocol_type_list, groupUniqArrayMerge(port_list) as port_list \ FROM security_website_domain_info where \ stat_time >= '${startTime}' and stat_time < '${endTime}' and policy_id=${policy_id} group by domain
+    ...    ELSE IF    '${apiName}'=="injectionstatistics"    Set Variable    select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/${timeGranularity})*${timeGranularity}),'%Y-%m-%d %H:%i:%s') as stat_time, \ sum(bytes) as bytes from injection_metrics_log \ where __time >= '${startTime}' and __time < '${endTime}' group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/${timeGranularity})*${timeGranularity}),'%Y-%m-%d %H:%i:%s') limit ${limit}
+    ...    ELSE IF    '${apiName}'=="topn"    Set Variable    select domain,sum(session_num) as sessions,sum(c2s_byte_num) as sentBytes,sum(s2c_byte_num) as receivedBytes,sum(c2s_byte_num+s2c_byte_num) as totalBytes,sum(c2s_pkt_num) as sentPackets,sum(s2c_pkt_num) as receivedPackets,sum(c2s_pkt_num+s2c_pkt_num) as totalPackets from top_website_domain_log where __time >= '${startTime}' \ and __time < '${endTime}' \ and order_by='${orderBy}' \ group by domain order by ${TotalOrderBy} desc limit ${limit}
+    log    ${sql}
     log    sql------------${sql}
     #traffic二次请求sql
     ${sql2}=    run keyword if    '${apiName}'=='traffic'    SplicingTrafficSql    ${apiName}    ${startTime}    ${endTime}    statisticsUnit=${statisticsUnit}    timeGranularity=${timeGranularity}    limit=${limit}
@@ -82,7 +108,7 @@ GetApi
     ...    AND    Should Be Equal As Strings    ${bigdata2}[status]    200
     #traffic二次请求json拼接与毕方一致
     ${bigdataTraffic}=    run keyword if    '${apiName}'=="traffic"    SplicingTrafficJson    ${bigdata}    ${bigdata2}
-    ${bifanglist}    run keyword if    '${apiName}'=="ip_correlation_domain"    set variable    ${bifang['data']['list']['list']}
+    ${bifanglist}    run keyword if    '${apiName}'=="ip_correlation_domain" \ or '${apiName}'=='domain_resource'    set variable    ${bifang['data']['list']['list']}
     ...    ELSE IF    '${apiName}'=="traffic"    set variable    ${bifang['data']}
     ...    ELSE    set variable    ${bifang['data']['list']}
     ${bigdatalist}    run keyword if    '${apiName}'=="traffic"    set variable    ${bigdataTraffic}