[CN-1587] Subscriber实体检索相关库表创建
This commit is contained in:
houjinchuan
6
CN发布版本更新记录/2024/CN-24.03/clickhouse/check-24.03-ck.sql
Normal file
6
CN发布版本更新记录/2024/CN-24.03/clickhouse/check-24.03-ck.sql
Normal file
@@ -0,0 +1,6 @@
|
||||
set distributed_ddl_task_timeout = 180;
|
||||
|
||||
SELECT subscriber_id, app, imei, imsi, phone_number, apn, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
|
||||
FROM cyber_narrator_galaxy.metric_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location, stat_time
|
||||
FROM cyber_narrator_galaxy.location_subscriber where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
18
CN发布版本更新记录/2024/CN-24.04/clickhouse/check-24.04-ck.sql
Normal file
18
CN发布版本更新记录/2024/CN-24.04/clickhouse/check-24.04-ck.sql
Normal file
@@ -0,0 +1,18 @@
|
||||
SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source, stat_time
|
||||
FROM cyber_narrator_galaxy.location_subscriber where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT app_name, `domain`, ip, ip_country_region, ip_super_admin_area, ip_admin_area, ip_asn, ip_isp, domain_category_name, domain_category_group, app_category, app_subcategory, entity_tags, create_time, update_time
|
||||
FROM cyber_narrator_galaxy.session_relation_domain_ip_app where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT app_name ,domain ,ip ,ip_country_region ,ip_super_admin_area ,ip_admin_area ,ip_asn ,ip_isp ,domain_category_name ,domain_category_group ,app_category ,app_subcategory ,entity_tags ,stat_time
|
||||
FROM cyber_narrator_galaxy.raw_session_relation_domain_ip_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT ip, l7_protocol, port, stat_time
|
||||
FROM cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT app_name, subscriber_id, imei, imsi, phone_number, apn, app_category, app_subcategory, create_time, update_time
|
||||
FROM cyber_narrator_galaxy.session_relation_subscriber_app where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT app_name ,subscriber_id ,imei ,imsi ,phone_number ,apn ,app_category ,app_subcategory ,stat_time
|
||||
FROM cyber_narrator_galaxy.raw_session_relation_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT app_name, `domain`, ip, ip_country_region, ip_super_admin_area, ip_admin_area, ip_asn, ip_isp, domain_category_name, domain_category_group, app_category, app_subcategory, entity_tags, create_time, update_time
|
||||
FROM cyber_narrator_galaxy.session_relation_domain_ip_app_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT app_name,subscriber_id,imei,imsi,phone_number,apn,app_category,app_subcategory,create_time,update_time
|
||||
FROM cyber_narrator_galaxy.session_relation_subscriber_app_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT ip, l7_protocol, port, create_time, update_time
|
||||
FROM cyber_narrator_galaxy.cn_ip_dynamic_attribute_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
299
CN发布版本更新记录/2024/CN-24.04/clickhouse/update-24.04-ck.sql
Normal file
299
CN发布版本更新记录/2024/CN-24.04/clickhouse/update-24.04-ck.sql
Normal file
@@ -0,0 +1,299 @@
|
||||
set distributed_ddl_task_timeout = 180;
|
||||
|
||||
ALTER table cyber_narrator_galaxy.location_subscriber_local on cluster ck_cluster add column IF NOT EXISTS data_source String after third_location;
|
||||
ALTER table cyber_narrator_galaxy.location_subscriber on cluster ck_cluster add column IF NOT EXISTS data_source String after third_location;
|
||||
ALTER table cyber_narrator_galaxy.location_subscriber on cluster ck_query add column IF NOT EXISTS data_source String after third_location;
|
||||
|
||||
drop view IF EXISTS cyber_narrator_galaxy.cn_entity_relation_view ON CLUSTER ck_cluster;
|
||||
drop view IF EXISTS cyber_narrator_galaxy.cn_ip_dynamic_attribute_view ON CLUSTER ck_cluster;
|
||||
drop view IF EXISTS cyber_narrator_galaxy.cn_entity_relation_view_metric ON CLUSTER ck_cluster;
|
||||
drop view IF EXISTS cyber_narrator_galaxy.cn_ip_dynamic_attribute_view_metric ON CLUSTER ck_cluster;
|
||||
|
||||
drop table IF EXISTS cyber_narrator_galaxy.cn_entity_relation on cluster ck_cluster;
|
||||
drop table IF EXISTS cyber_narrator_galaxy.cn_entity_relation on cluster ck_query;
|
||||
drop table IF EXISTS cyber_narrator_galaxy.metric_relation on cluster ck_cluster;
|
||||
drop table IF EXISTS cyber_narrator_galaxy.metric_relation on cluster ck_query;
|
||||
drop table IF EXISTS cyber_narrator_galaxy.metric_ip_dynamic_attribute on cluster ck_cluster;
|
||||
drop table IF EXISTS cyber_narrator_galaxy.metric_ip_dynamic_attribute on cluster ck_query;
|
||||
|
||||
|
||||
RENAME TABLE cyber_narrator_galaxy.cn_entity_relation_local TO cyber_narrator_galaxy.session_relation_domain_ip_app_local ON CLUSTER ck_cluster;
|
||||
RENAME TABLE cyber_narrator_galaxy.metric_relation_local TO cyber_narrator_galaxy.raw_session_relation_domain_ip_app_local ON CLUSTER ck_cluster;
|
||||
RENAME TABLE cyber_narrator_galaxy.metric_ip_dynamic_attribute_local TO cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute_local ON CLUSTER ck_cluster;
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app_local ON CLUSTER ck_cluster (
|
||||
app_name String,
|
||||
subscriber_id String,
|
||||
imei String,
|
||||
imsi String,
|
||||
phone_number String,
|
||||
apn String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
create_time Int64,
|
||||
update_time Int64
|
||||
) ENGINE = MergeTree
|
||||
ORDER BY (subscriber_id, app_name)
|
||||
TTL toDateTime(update_time) + toIntervalSecond(2592000),
|
||||
toDateTime(update_time) + toIntervalSecond(1)
|
||||
GROUP BY subscriber_id, app_name
|
||||
SET create_time = min(create_time),
|
||||
update_time = max(update_time),
|
||||
imei = anyLast(imei),
|
||||
imsi = anyLast(imsi),
|
||||
phone_number = anyLast(phone_number),
|
||||
apn = anyLast(apn),
|
||||
app_category = anyLast(app_category),
|
||||
app_subcategory = anyLast(app_subcategory);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app ON CLUSTER ck_cluster (
|
||||
app_name String,
|
||||
subscriber_id String,
|
||||
imei String,
|
||||
imsi String,
|
||||
phone_number String,
|
||||
apn String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
create_time Int64,
|
||||
update_time Int64
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_relation_subscriber_app_local', rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app ON CLUSTER ck_query (
|
||||
app_name String,
|
||||
subscriber_id String,
|
||||
imei String,
|
||||
imsi String,
|
||||
phone_number String,
|
||||
apn String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
create_time Int64,
|
||||
update_time Int64
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_relation_subscriber_app_local', rand());
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber_app_local ON CLUSTER ck_cluster (
|
||||
app_name String,
|
||||
subscriber_id String,
|
||||
imei String,
|
||||
imsi String,
|
||||
phone_number String,
|
||||
apn String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
stat_time Int64
|
||||
) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time) SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber_app ON CLUSTER ck_cluster (
|
||||
app_name String,
|
||||
subscriber_id String,
|
||||
imei String,
|
||||
imsi String,
|
||||
phone_number String,
|
||||
apn String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
stat_time Int64
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_subscriber_app_local', rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber_app ON CLUSTER ck_query (
|
||||
app_name String,
|
||||
subscriber_id String,
|
||||
imei String,
|
||||
imsi String,
|
||||
phone_number String,
|
||||
apn String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
stat_time Int64
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_subscriber_app_local', rand());
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_domain_ip_app on cluster ck_query
|
||||
(
|
||||
app_name String,
|
||||
domain String,
|
||||
ip String,
|
||||
ip_country_region String,
|
||||
ip_super_admin_area String,
|
||||
ip_admin_area String,
|
||||
ip_asn String,
|
||||
ip_isp String,
|
||||
domain_category_name String,
|
||||
domain_category_group String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
entity_tags Array(String),
|
||||
create_time Int64,
|
||||
update_time Int64
|
||||
)
|
||||
ENGINE = Distributed('ck_cluster',
|
||||
'cyber_narrator_galaxy',
|
||||
'session_relation_domain_ip_app_local',
|
||||
rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_domain_ip_app on cluster ck_cluster
|
||||
(
|
||||
app_name String,
|
||||
domain String,
|
||||
ip String,
|
||||
ip_country_region String,
|
||||
ip_super_admin_area String,
|
||||
ip_admin_area String,
|
||||
ip_asn String,
|
||||
ip_isp String,
|
||||
domain_category_name String,
|
||||
domain_category_group String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
entity_tags Array(String),
|
||||
create_time Int64,
|
||||
update_time Int64
|
||||
)
|
||||
ENGINE = Distributed('ck_cluster',
|
||||
'cyber_narrator_galaxy',
|
||||
'session_relation_domain_ip_app_local',
|
||||
rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_domain_ip_app ON CLUSTER ck_cluster (
|
||||
app_name String,
|
||||
domain String,
|
||||
ip String,
|
||||
ip_country_region String,
|
||||
ip_super_admin_area String,
|
||||
ip_admin_area String,
|
||||
ip_asn String,
|
||||
ip_isp String,
|
||||
domain_category_name String,
|
||||
domain_category_group String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
entity_tags Array(String),
|
||||
stat_time Int64
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_domain_ip_app_local', rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_domain_ip_app ON CLUSTER ck_query (
|
||||
app_name String,
|
||||
domain String,
|
||||
ip String,
|
||||
ip_country_region String,
|
||||
ip_super_admin_area String,
|
||||
ip_admin_area String,
|
||||
ip_asn String,
|
||||
ip_isp String,
|
||||
domain_category_name String,
|
||||
domain_category_group String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
entity_tags Array(String),
|
||||
stat_time Int64
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_domain_ip_app_local', rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute ON CLUSTER ck_cluster (
|
||||
ip String,
|
||||
l7_protocol String,
|
||||
port Int64,
|
||||
stat_time Int64
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_cn_ip_dynamic_attribute_local', rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute ON CLUSTER ck_query (
|
||||
ip String,
|
||||
l7_protocol String,
|
||||
port Int64,
|
||||
stat_time Int64
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_cn_ip_dynamic_attribute_local', rand());
|
||||
|
||||
|
||||
CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.session_relation_domain_ip_app_view_metric on cluster ck_cluster
|
||||
TO cyber_narrator_galaxy.session_relation_domain_ip_app_local
|
||||
(
|
||||
app_name String,
|
||||
domain String,
|
||||
ip String,
|
||||
ip_country_region String,
|
||||
ip_super_admin_area String,
|
||||
ip_admin_area String,
|
||||
ip_asn String,
|
||||
ip_isp String,
|
||||
domain_category_name String,
|
||||
domain_category_group String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
entity_tags Array(String),
|
||||
create_time Int64,
|
||||
update_time Int64
|
||||
) AS
|
||||
SELECT
|
||||
app_name AS app_name,
|
||||
domain AS domain,
|
||||
ip AS ip,
|
||||
anyLast(ip_country_region) AS ip_country_region,
|
||||
anyLast(ip_super_admin_area) AS ip_super_admin_area,
|
||||
anyLast(ip_admin_area) AS ip_admin_area,
|
||||
anyLast(ip_asn) AS ip_asn,
|
||||
anyLast(ip_isp) AS ip_isp,
|
||||
anyLast(domain_category_name) AS domain_category_name,
|
||||
anyLast(domain_category_group) AS domain_category_group,
|
||||
anyLast(app_category) AS app_category,
|
||||
anyLast(app_subcategory) AS app_subcategory,
|
||||
groupUniqArrayArray(entity_tags) AS entity_tags,
|
||||
min(c1.stat_time) AS create_time,
|
||||
max(c1.stat_time) AS update_time
|
||||
FROM cyber_narrator_galaxy.raw_session_relation_domain_ip_app_local AS c1
|
||||
GROUP BY
|
||||
ip,
|
||||
app_name,
|
||||
domain;
|
||||
|
||||
CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.session_relation_subscriber_app_view_metric on cluster ck_cluster
|
||||
TO cyber_narrator_galaxy.session_relation_subscriber_app_local
|
||||
(
|
||||
app_name String,
|
||||
subscriber_id String,
|
||||
imei String,
|
||||
imsi String,
|
||||
phone_number String,
|
||||
apn String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
create_time Int64,
|
||||
update_time Int64
|
||||
) AS
|
||||
SELECT
|
||||
app_name AS app_name,
|
||||
subscriber_id AS subscriber_id,
|
||||
anyLast(imei) AS imei,
|
||||
anyLast(imsi) AS imsi,
|
||||
anyLast(phone_number) AS phone_number,
|
||||
anyLast(apn) AS apn,
|
||||
anyLast(app_category) AS app_category,
|
||||
anyLast(app_subcategory) AS app_subcategory,
|
||||
min(c1.stat_time) AS create_time,
|
||||
max(c1.stat_time) AS update_time
|
||||
FROM cyber_narrator_galaxy.raw_session_relation_subscriber_app_local AS c1
|
||||
GROUP BY
|
||||
subscriber_id,
|
||||
app_name;
|
||||
|
||||
CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.cn_ip_dynamic_attribute_view_metric on cluster ck_cluster
|
||||
TO cyber_narrator_galaxy.cn_ip_dynamic_attribute_local
|
||||
(
|
||||
ip String,
|
||||
l7_protocol String,
|
||||
port Int64,
|
||||
create_time Int64,
|
||||
update_time Int64
|
||||
) AS
|
||||
SELECT
|
||||
ip AS ip,
|
||||
l7_protocol AS l7_protocol,
|
||||
port AS port,
|
||||
min(c1.stat_time) AS create_time,
|
||||
max(c1.stat_time) AS update_time
|
||||
FROM cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute_local AS c1
|
||||
GROUP BY
|
||||
ip,
|
||||
l7_protocol,
|
||||
port;
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/bin/sh
|
||||
|
||||
#gohangout启动文件
|
||||
|
||||
#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下
|
||||
YML_NAME=k2ck_raw_cn_ip_dynamic_attribute_cn
|
||||
#gohangout的二进制启动文件路径
|
||||
BASE_DIR=$(cd $(dirname $0); cd ../../; pwd)
|
||||
#日志级别,1,5,10,数字越大日志越详细
|
||||
LOG_LV=5
|
||||
#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程
|
||||
THREAD_SUM=3
|
||||
#进程总数
|
||||
PROCESS_SUM=$1
|
||||
|
||||
if [ ! -d "$BASE_DIR/logs" ]; then
|
||||
mkdir -p $BASE_DIR/logs
|
||||
fi
|
||||
|
||||
echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
|
||||
id=0
|
||||
logid=0
|
||||
while true ; do
|
||||
NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l`
|
||||
pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}')
|
||||
time_stamp=$(date +%Y%m%d%H%M%S)
|
||||
if [ "${NUM}" -lt ${PROCESS_SUM} ];then
|
||||
$BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 &
|
||||
echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
|
||||
((logid++))
|
||||
((id++))
|
||||
if [ ${logid} -gt ${PROCESS_SUM} ];then
|
||||
logid=0
|
||||
pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}')
|
||||
for pid in $pids
|
||||
do
|
||||
kill -9 $pid
|
||||
done
|
||||
echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
|
||||
sleep 30
|
||||
fi
|
||||
#大于设置进程数,杀掉所有进程,重启
|
||||
elif [ "${NUM}" -gt ${PROCESS_SUM} ];then
|
||||
for pid in $pids
|
||||
do
|
||||
kill -9 $pid
|
||||
done
|
||||
id=0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/bin/sh
|
||||
|
||||
#gohangout启动文件
|
||||
|
||||
#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下
|
||||
YML_NAME=k2ck_raw_session_relation_domain_ip_app_cn
|
||||
#gohangout的二进制启动文件路径
|
||||
BASE_DIR=$(cd $(dirname $0); cd ../../; pwd)
|
||||
#日志级别,1,5,10,数字越大日志越详细
|
||||
LOG_LV=5
|
||||
#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程
|
||||
THREAD_SUM=3
|
||||
#进程总数
|
||||
PROCESS_SUM=$1
|
||||
|
||||
if [ ! -d "$BASE_DIR/logs" ]; then
|
||||
mkdir -p $BASE_DIR/logs
|
||||
fi
|
||||
|
||||
echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
|
||||
id=0
|
||||
logid=0
|
||||
while true ; do
|
||||
NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l`
|
||||
pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}')
|
||||
time_stamp=$(date +%Y%m%d%H%M%S)
|
||||
if [ "${NUM}" -lt ${PROCESS_SUM} ];then
|
||||
$BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 &
|
||||
echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
|
||||
((logid++))
|
||||
((id++))
|
||||
if [ ${logid} -gt ${PROCESS_SUM} ];then
|
||||
logid=0
|
||||
pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}')
|
||||
for pid in $pids
|
||||
do
|
||||
kill -9 $pid
|
||||
done
|
||||
echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
|
||||
sleep 30
|
||||
fi
|
||||
#大于设置进程数,杀掉所有进程,重启
|
||||
elif [ "${NUM}" -gt ${PROCESS_SUM} ];then
|
||||
for pid in $pids
|
||||
do
|
||||
kill -9 $pid
|
||||
done
|
||||
id=0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/bin/sh
|
||||
|
||||
#gohangout启动文件
|
||||
|
||||
#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下
|
||||
YML_NAME=k2ck_raw_session_relation_subscriber_app_cn
|
||||
#gohangout的二进制启动文件路径
|
||||
BASE_DIR=$(cd $(dirname $0); cd ../../; pwd)
|
||||
#日志级别,1,5,10,数字越大日志越详细
|
||||
LOG_LV=5
|
||||
#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程
|
||||
THREAD_SUM=3
|
||||
#进程总数
|
||||
PROCESS_SUM=$1
|
||||
|
||||
if [ ! -d "$BASE_DIR/logs" ]; then
|
||||
mkdir -p $BASE_DIR/logs
|
||||
fi
|
||||
|
||||
echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
|
||||
id=0
|
||||
logid=0
|
||||
while true ; do
|
||||
NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l`
|
||||
pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}')
|
||||
time_stamp=$(date +%Y%m%d%H%M%S)
|
||||
if [ "${NUM}" -lt ${PROCESS_SUM} ];then
|
||||
$BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 &
|
||||
echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
|
||||
((logid++))
|
||||
((id++))
|
||||
if [ ${logid} -gt ${PROCESS_SUM} ];then
|
||||
logid=0
|
||||
pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}')
|
||||
for pid in $pids
|
||||
do
|
||||
kill -9 $pid
|
||||
done
|
||||
echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
|
||||
sleep 30
|
||||
fi
|
||||
#大于设置进程数,杀掉所有进程,重启
|
||||
elif [ "${NUM}" -gt ${PROCESS_SUM} ];then
|
||||
for pid in $pids
|
||||
do
|
||||
kill -9 $pid
|
||||
done
|
||||
id=0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
@@ -0,0 +1,28 @@
|
||||
inputs:
|
||||
- Kafka:
|
||||
topic:
|
||||
RAW-CN-IP-DYNAMIC-ATTRIBUTE: 1
|
||||
#assign:
|
||||
# weblog: [0,9]
|
||||
codec: json
|
||||
consumer_settings:
|
||||
bootstrap.servers: "192.168.44.55:9092"
|
||||
group.id: tsg-raw_cn_ip_dynamic_attribute_cn
|
||||
max.partition.fetch.bytes: '10485760'
|
||||
auto.commit.interval.ms: '5000'
|
||||
# from.beginning: 'true'
|
||||
# sasl.mechanism: PLAIN
|
||||
# sasl.user: admin
|
||||
# sasl.password: admin-secret
|
||||
|
||||
outputs:
|
||||
- Clickhouse:
|
||||
table: 'cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute_local'
|
||||
username: 'default'
|
||||
password: 'ceiec2019'
|
||||
hosts:
|
||||
- 'tcp://192.168.44.55:9001'
|
||||
bulk_actions: 100000
|
||||
flush_interval: 30
|
||||
concurrent: 2
|
||||
conn_max_life_time: 60
|
||||
@@ -0,0 +1,28 @@
|
||||
inputs:
|
||||
- Kafka:
|
||||
topic:
|
||||
RAW-SESSION-RELATION-DOMAIN-IP-APP: 1
|
||||
#assign:
|
||||
# weblog: [0,9]
|
||||
codec: json
|
||||
consumer_settings:
|
||||
bootstrap.servers: "192.168.44.55:9092"
|
||||
group.id: tsg-raw_session_relation_domain_ip_app_cn
|
||||
max.partition.fetch.bytes: '10485760'
|
||||
auto.commit.interval.ms: '5000'
|
||||
# from.beginning: 'true'
|
||||
# sasl.mechanism: PLAIN
|
||||
# sasl.user: admin
|
||||
# sasl.password: admin-secret
|
||||
|
||||
outputs:
|
||||
- Clickhouse:
|
||||
table: 'cyber_narrator_galaxy.raw_session_relation_domain_ip_app_local'
|
||||
username: 'default'
|
||||
password: 'ceiec2019'
|
||||
hosts:
|
||||
- 'tcp://192.168.44.55:9001'
|
||||
bulk_actions: 100000
|
||||
flush_interval: 30
|
||||
concurrent: 2
|
||||
conn_max_life_time: 60
|
||||
@@ -0,0 +1,28 @@
|
||||
inputs:
|
||||
- Kafka:
|
||||
topic:
|
||||
RAW-SESSION-RELATION-SUBSCRIBER-APP: 1
|
||||
#assign:
|
||||
# weblog: [0,9]
|
||||
codec: json
|
||||
consumer_settings:
|
||||
bootstrap.servers: "192.168.44.55:9092"
|
||||
group.id: tsg-raw_session_relation_subscriber_app_cn
|
||||
max.partition.fetch.bytes: '10485760'
|
||||
auto.commit.interval.ms: '5000'
|
||||
# from.beginning: 'true'
|
||||
# sasl.mechanism: PLAIN
|
||||
# sasl.user: admin
|
||||
# sasl.password: admin-secret
|
||||
|
||||
outputs:
|
||||
- Clickhouse:
|
||||
table: 'cyber_narrator_galaxy.raw_session_relation_subscriber_app_local'
|
||||
username: 'default'
|
||||
password: 'ceiec2019'
|
||||
hosts:
|
||||
- 'tcp://192.168.44.55:9001'
|
||||
bulk_actions: 100000
|
||||
flush_interval: 30
|
||||
concurrent: 2
|
||||
conn_max_life_time: 60
|
||||
24
CN发布版本更新记录/2024/CN-24.04/start_all.sh
Normal file
24
CN发布版本更新记录/2024/CN-24.04/start_all.sh
Normal file
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
|
||||
STARTDIR=$(cd $(dirname $0); pwd)
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_location_subscriber_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_subscriber_app_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_security_event_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_ip_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_region_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_asn_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_application_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_domain_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_protocol_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_link_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_qname_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_server_ip_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_qtype_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rcode_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rr_a_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rr_aaaa_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rr_cname_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_raw_session_relation_domain_ip_app_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_raw_session_relation_subscriber_app_cn.sh $1 > /dev/null 2>&1 &
|
||||
nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_raw_cn_ip_dynamic_attribute_cn.sh $1 > /dev/null 2>&1 &
|
||||
$STARTDIR/bin/ghoStart/start_gohangout_k2ck_cn_record.sh $1
|
||||
@@ -1280,6 +1280,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber_local ON CL
|
||||
first_location String,
|
||||
second_location String,
|
||||
third_location String,
|
||||
data_source String,
|
||||
stat_time Int64
|
||||
) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,subscriber_id) SETTINGS index_granularity = 8192;
|
||||
|
||||
@@ -1294,6 +1295,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER
|
||||
first_location String,
|
||||
second_location String,
|
||||
third_location String,
|
||||
data_source String,
|
||||
stat_time Int64
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'location_subscriber_local', rand());
|
||||
|
||||
@@ -1308,6 +1310,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER
|
||||
first_location String,
|
||||
second_location String,
|
||||
third_location String,
|
||||
data_source String,
|
||||
stat_time Int64
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'location_subscriber_local', rand());
|
||||
|
||||
@@ -1571,7 +1574,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_cname ON CLUSTER
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_rr_cname_local', rand());
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_relation_local ON CLUSTER ck_cluster (
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_domain_ip_app_local ON CLUSTER ck_cluster (
|
||||
app_name String,
|
||||
domain String,
|
||||
ip String,
|
||||
@@ -1588,7 +1591,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_relation_local ON CLUSTE
|
||||
stat_time Int64
|
||||
) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time) SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_relation ON CLUSTER ck_cluster (
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_domain_ip_app ON CLUSTER ck_cluster (
|
||||
app_name String,
|
||||
domain String,
|
||||
ip String,
|
||||
@@ -1603,9 +1606,9 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_relation ON CLUSTER ck_c
|
||||
app_subcategory String,
|
||||
entity_tags Array(String),
|
||||
stat_time Int64
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_relation_local', rand());
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_domain_ip_app_local', rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_relation ON CLUSTER ck_query (
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_domain_ip_app ON CLUSTER ck_query (
|
||||
app_name String,
|
||||
domain String,
|
||||
ip String,
|
||||
@@ -1620,34 +1623,33 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_relation ON CLUSTER ck_q
|
||||
app_subcategory String,
|
||||
entity_tags Array(String),
|
||||
stat_time Int64
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_relation_local', rand());
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_domain_ip_app_local', rand());
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_ip_dynamic_attribute_local ON CLUSTER ck_cluster (
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute_local ON CLUSTER ck_cluster (
|
||||
ip String,
|
||||
l7_protocol String,
|
||||
port Int64,
|
||||
stat_time Int64
|
||||
) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time) SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_ip_dynamic_attribute ON CLUSTER ck_cluster (
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute ON CLUSTER ck_cluster (
|
||||
ip String,
|
||||
l7_protocol String,
|
||||
port Int64,
|
||||
stat_time Int64
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_ip_dynamic_attribute_local', rand());
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_cn_ip_dynamic_attribute_local', rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_ip_dynamic_attribute ON CLUSTER ck_query (
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute ON CLUSTER ck_query (
|
||||
ip String,
|
||||
l7_protocol String,
|
||||
port Int64,
|
||||
stat_time Int64
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_ip_dynamic_attribute_local', rand());
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_cn_ip_dynamic_attribute_local', rand());
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_entity_relation_local on cluster ck_cluster
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_domain_ip_app_local on cluster ck_cluster
|
||||
(
|
||||
|
||||
app_name String,
|
||||
domain String,
|
||||
ip String,
|
||||
@@ -1684,9 +1686,8 @@ TTL toDateTime(update_time) + toIntervalSecond(2592000),
|
||||
app_subcategory = anyLast(app_subcategory),
|
||||
entity_tags = groupUniqArrayArray(entity_tags);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_entity_relation on cluster ck_query
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_domain_ip_app on cluster ck_query
|
||||
(
|
||||
|
||||
app_name String,
|
||||
domain String,
|
||||
ip String,
|
||||
@@ -1705,12 +1706,11 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_entity_relation on cluster c
|
||||
)
|
||||
ENGINE = Distributed('ck_cluster',
|
||||
'cyber_narrator_galaxy',
|
||||
'cn_entity_relation_local',
|
||||
'session_relation_domain_ip_app_local',
|
||||
rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_entity_relation on cluster ck_cluster
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_domain_ip_app on cluster ck_cluster
|
||||
(
|
||||
|
||||
app_name String,
|
||||
domain String,
|
||||
ip String,
|
||||
@@ -1729,93 +1729,98 @@ ENGINE = Distributed('ck_cluster',
|
||||
)
|
||||
ENGINE = Distributed('ck_cluster',
|
||||
'cyber_narrator_galaxy',
|
||||
'cn_entity_relation_local',
|
||||
'session_relation_domain_ip_app_local',
|
||||
rand());
|
||||
|
||||
|
||||
CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.cn_entity_relation_view TO cyber_narrator_galaxy.cn_entity_relation_local
|
||||
(
|
||||
app_name String,
|
||||
domain String,
|
||||
ip String,
|
||||
ip_country_region String,
|
||||
ip_super_admin_area String,
|
||||
ip_admin_area String,
|
||||
ip_asn String,
|
||||
ip_isp String,
|
||||
domain_category_name String,
|
||||
domain_category_group String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
entity_tags Array(String),
|
||||
create_time Int64,
|
||||
update_time Int64
|
||||
) AS
|
||||
SELECT
|
||||
app AS app_name,
|
||||
domain AS domain,
|
||||
server_ip AS ip,
|
||||
anyLast(server_country_region) AS ip_country_region,
|
||||
anyLast(server_super_admin_area) AS ip_super_admin_area,
|
||||
anyLast(server_admin_area) AS ip_admin_area,
|
||||
anyLast(server_asn) AS ip_asn,
|
||||
anyLast(server_isp) AS ip_isp,
|
||||
anyLast(domain_category_name) AS domain_category_name,
|
||||
anyLast(domain_category_group) AS domain_category_group,
|
||||
anyLast(app_category) AS app_category,
|
||||
anyLast(app_subcategory) AS app_subcategory,
|
||||
groupUniqArrayArray(arrayConcat(server_ip_tags, domain_tags, app_tags)) AS entity_tags,
|
||||
min(c1.recv_time) AS create_time,
|
||||
max(c1.recv_time) AS update_time
|
||||
FROM cyber_narrator_galaxy.session_record_cn_local AS c1
|
||||
WHERE (decoded_path LIKE '%IPv4_TCP%') OR (server_port IN (53, 443))
|
||||
GROUP BY
|
||||
ip,
|
||||
app_name,
|
||||
domain
|
||||
;
|
||||
|
||||
CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.cn_entity_relation_view_metric on cluster ck_cluster TO cyber_narrator_galaxy.cn_entity_relation_local
|
||||
(
|
||||
|
||||
app_name String,
|
||||
domain String,
|
||||
ip String,
|
||||
ip_country_region String,
|
||||
ip_super_admin_area String,
|
||||
ip_admin_area String,
|
||||
ip_asn String,
|
||||
ip_isp String,
|
||||
domain_category_name String,
|
||||
domain_category_group String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
entity_tags Array(String),
|
||||
create_time Int64,
|
||||
update_time Int64
|
||||
) AS
|
||||
SELECT
|
||||
app_name AS app_name,
|
||||
domain AS domain,
|
||||
ip AS ip,
|
||||
anyLast(ip_country_region) AS ip_country_region,
|
||||
anyLast(ip_super_admin_area) AS ip_super_admin_area,
|
||||
anyLast(ip_admin_area) AS ip_admin_area,
|
||||
anyLast(ip_asn) AS ip_asn,
|
||||
anyLast(ip_isp) AS ip_isp,
|
||||
anyLast(domain_category_name) AS domain_category_name,
|
||||
anyLast(domain_category_group) AS domain_category_group,
|
||||
anyLast(app_category) AS app_category,
|
||||
anyLast(app_subcategory) AS app_subcategory,
|
||||
groupUniqArrayArray(entity_tags) AS entity_tags,
|
||||
min(c1.stat_time) AS create_time,
|
||||
max(c1.stat_time) AS update_time
|
||||
FROM cyber_narrator_galaxy.metric_relation_local AS c1
|
||||
GROUP BY
|
||||
ip,
|
||||
app_name,
|
||||
domain;
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app_local ON CLUSTER ck_cluster (
|
||||
app_name String,
|
||||
subscriber_id String,
|
||||
imei String,
|
||||
imsi String,
|
||||
phone_number String,
|
||||
apn String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
create_time Int64,
|
||||
update_time Int64
|
||||
) ENGINE = MergeTree
|
||||
ORDER BY (subscriber_id, app_name)
|
||||
TTL toDateTime(update_time) + toIntervalSecond(2592000),
|
||||
toDateTime(update_time) + toIntervalSecond(1)
|
||||
GROUP BY subscriber_id, app_name
|
||||
SET create_time = min(create_time),
|
||||
update_time = max(update_time),
|
||||
imei = anyLast(imei),
|
||||
imsi = anyLast(imsi),
|
||||
phone_number = anyLast(phone_number),
|
||||
apn = anyLast(apn),
|
||||
app_category = anyLast(app_category),
|
||||
app_subcategory = anyLast(app_subcategory);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app ON CLUSTER ck_cluster (
|
||||
app_name String,
|
||||
subscriber_id String,
|
||||
imei String,
|
||||
imsi String,
|
||||
phone_number String,
|
||||
apn String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
create_time Int64,
|
||||
update_time Int64
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_relation_subscriber_app_local', rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app ON CLUSTER ck_query (
|
||||
app_name String,
|
||||
subscriber_id String,
|
||||
imei String,
|
||||
imsi String,
|
||||
phone_number String,
|
||||
apn String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
create_time Int64,
|
||||
update_time Int64
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_relation_subscriber_app_local', rand());
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber_app_local ON CLUSTER ck_cluster (
|
||||
app_name String,
|
||||
subscriber_id String,
|
||||
imei String,
|
||||
imsi String,
|
||||
phone_number String,
|
||||
apn String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
stat_time Int64
|
||||
) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time) SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber_app ON CLUSTER ck_cluster (
|
||||
app_name String,
|
||||
subscriber_id String,
|
||||
imei String,
|
||||
imsi String,
|
||||
phone_number String,
|
||||
apn String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
stat_time Int64
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_subscriber_app_local', rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber_app ON CLUSTER ck_query (
|
||||
app_name String,
|
||||
subscriber_id String,
|
||||
imei String,
|
||||
imsi String,
|
||||
phone_number String,
|
||||
apn String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
stat_time Int64
|
||||
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_subscriber_app_local', rand());
|
||||
|
||||
|
||||
CREATE TABLE if not exists cyber_narrator_galaxy.cn_ip_dynamic_attribute_local on cluster ck_cluster
|
||||
(
|
||||
@@ -1860,57 +1865,15 @@ ENGINE = Distributed('ck_cluster',
|
||||
'cyber_narrator_galaxy',
|
||||
'cn_ip_dynamic_attribute_local',
|
||||
rand());
|
||||
|
||||
|
||||
CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.cn_ip_dynamic_attribute_view TO cyber_narrator_galaxy.cn_ip_dynamic_attribute_local
|
||||
(
|
||||
ip String,
|
||||
l7_protocol String,
|
||||
port Int64,
|
||||
create_time Int64,
|
||||
update_time Int64
|
||||
) AS
|
||||
SELECT
|
||||
server_ip AS ip,
|
||||
l7_protocol AS l7_protocol,
|
||||
server_port AS port,
|
||||
min(c1.recv_time) AS create_time,
|
||||
max(c1.recv_time) AS update_time
|
||||
FROM cyber_narrator_galaxy.session_record_cn_local AS c1
|
||||
WHERE (decoded_path LIKE '%IPv4_TCP%') OR (server_port IN (53, 443))
|
||||
GROUP BY
|
||||
ip,
|
||||
l7_protocol,
|
||||
port
|
||||
;
|
||||
|
||||
CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.cn_ip_dynamic_attribute_view_metric on cluster ck_cluster TO cyber_narrator_galaxy.cn_ip_dynamic_attribute_local
|
||||
(
|
||||
ip String,
|
||||
l7_protocol String,
|
||||
port Int64,
|
||||
create_time Int64,
|
||||
update_time Int64
|
||||
) AS
|
||||
SELECT
|
||||
ip AS ip,
|
||||
l7_protocol AS l7_protocol,
|
||||
port AS port,
|
||||
min(c1.stat_time) AS create_time,
|
||||
max(c1.stat_time) AS update_time
|
||||
FROM cyber_narrator_galaxy.metric_ip_dynamic_attribute_local AS c1
|
||||
GROUP BY
|
||||
ip,
|
||||
l7_protocol,
|
||||
port;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_security_event_local ON CLUSTER ck_cluster
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_security_event_local ON CLUSTER ck_cluster
|
||||
(
|
||||
status Int64,
|
||||
is_builtin Int64,
|
||||
rule_type String,
|
||||
victim_ip String,
|
||||
victim_country_region String,
|
||||
victim_country_region String,
|
||||
victim_super_admin_area String,
|
||||
victim_admin_area String,
|
||||
victim_longitude Nullable(Float64),
|
||||
@@ -1968,7 +1931,6 @@ TTL toDateTime(end_time) + toIntervalSecond(2592000),
|
||||
end_time = max(end_time),
|
||||
match_times = anyLast(match_times);
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_security_event ON CLUSTER ck_cluster
|
||||
(
|
||||
status Int64,
|
||||
@@ -2040,3 +2002,96 @@ ENGINE = Distributed('ck_cluster',
|
||||
'cyber_narrator_galaxy',
|
||||
'cn_security_event_local',
|
||||
rand());
|
||||
|
||||
|
||||
CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.session_relation_domain_ip_app_view_metric on cluster ck_cluster
|
||||
TO cyber_narrator_galaxy.session_relation_domain_ip_app_local
|
||||
(
|
||||
app_name String,
|
||||
domain String,
|
||||
ip String,
|
||||
ip_country_region String,
|
||||
ip_super_admin_area String,
|
||||
ip_admin_area String,
|
||||
ip_asn String,
|
||||
ip_isp String,
|
||||
domain_category_name String,
|
||||
domain_category_group String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
entity_tags Array(String),
|
||||
create_time Int64,
|
||||
update_time Int64
|
||||
) AS
|
||||
SELECT
|
||||
app_name AS app_name,
|
||||
domain AS domain,
|
||||
ip AS ip,
|
||||
anyLast(ip_country_region) AS ip_country_region,
|
||||
anyLast(ip_super_admin_area) AS ip_super_admin_area,
|
||||
anyLast(ip_admin_area) AS ip_admin_area,
|
||||
anyLast(ip_asn) AS ip_asn,
|
||||
anyLast(ip_isp) AS ip_isp,
|
||||
anyLast(domain_category_name) AS domain_category_name,
|
||||
anyLast(domain_category_group) AS domain_category_group,
|
||||
anyLast(app_category) AS app_category,
|
||||
anyLast(app_subcategory) AS app_subcategory,
|
||||
groupUniqArrayArray(entity_tags) AS entity_tags,
|
||||
min(c1.stat_time) AS create_time,
|
||||
max(c1.stat_time) AS update_time
|
||||
FROM cyber_narrator_galaxy.raw_session_relation_domain_ip_app_local AS c1
|
||||
GROUP BY
|
||||
ip,
|
||||
app_name,
|
||||
domain;
|
||||
|
||||
CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.session_relation_subscriber_app_view_metric on cluster ck_cluster
|
||||
TO cyber_narrator_galaxy.session_relation_subscriber_app_local
|
||||
(
|
||||
app_name String,
|
||||
subscriber_id String,
|
||||
imei String,
|
||||
imsi String,
|
||||
phone_number String,
|
||||
apn String,
|
||||
app_category String,
|
||||
app_subcategory String,
|
||||
create_time Int64,
|
||||
update_time Int64
|
||||
) AS
|
||||
SELECT
|
||||
app_name AS app_name,
|
||||
subscriber_id AS subscriber_id,
|
||||
anyLast(imei) AS imei,
|
||||
anyLast(imsi) AS imsi,
|
||||
anyLast(phone_number) AS phone_number,
|
||||
anyLast(apn) AS apn,
|
||||
anyLast(app_category) AS app_category,
|
||||
anyLast(app_subcategory) AS app_subcategory,
|
||||
min(c1.stat_time) AS create_time,
|
||||
max(c1.stat_time) AS update_time
|
||||
FROM cyber_narrator_galaxy.raw_session_relation_subscriber_app_local AS c1
|
||||
GROUP BY
|
||||
subscriber_id,
|
||||
app_name;
|
||||
|
||||
CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.cn_ip_dynamic_attribute_view_metric on cluster ck_cluster
|
||||
TO cyber_narrator_galaxy.cn_ip_dynamic_attribute_local
|
||||
(
|
||||
ip String,
|
||||
l7_protocol String,
|
||||
port Int64,
|
||||
create_time Int64,
|
||||
update_time Int64
|
||||
) AS
|
||||
SELECT
|
||||
ip AS ip,
|
||||
l7_protocol AS l7_protocol,
|
||||
port AS port,
|
||||
min(c1.stat_time) AS create_time,
|
||||
max(c1.stat_time) AS update_time
|
||||
FROM cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute_local AS c1
|
||||
GROUP BY
|
||||
ip,
|
||||
l7_protocol,
|
||||
port;
|
||||
|
||||
@@ -1,17 +1,21 @@
|
||||
SELECT app_name, `domain`, ip, ip_country_region, ip_super_admin_area, ip_admin_area, ip_asn, ip_isp, domain_category_name, domain_category_group, app_category, app_subcategory, entity_tags, create_time, update_time
|
||||
FROM cyber_narrator_galaxy.cn_entity_relation where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT app_name, `domain`, ip, ip_country_region, ip_super_admin_area, ip_admin_area, ip_asn, ip_isp, domain_category_name, domain_category_group, app_category, app_subcategory, entity_tags, create_time, update_time
|
||||
FROM cyber_narrator_galaxy.cn_entity_relation_view where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT app_name, `domain`, ip, ip_country_region, ip_super_admin_area, ip_admin_area, ip_asn, ip_isp, domain_category_name, domain_category_group, app_category, app_subcategory, entity_tags, create_time, update_time
|
||||
FROM cyber_narrator_galaxy.cn_entity_relation_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
FROM cyber_narrator_galaxy.session_relation_domain_ip_app where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT ip, l7_protocol, port, create_time, update_time
|
||||
FROM cyber_narrator_galaxy.cn_ip_dynamic_attribute where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT ip, l7_protocol, port, create_time, update_time
|
||||
FROM cyber_narrator_galaxy.cn_ip_dynamic_attribute_view where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT ip, l7_protocol, port, create_time, update_time
|
||||
FROM cyber_narrator_galaxy.cn_ip_dynamic_attribute_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT status, is_builtin, rule_type, victim_ip, offender_ip, event_info, event_key, severity, event_type, duration_s, event_name, app, `domain`, event_id, rule_id, start_time, end_time, match_times
|
||||
FROM cyber_narrator_galaxy.cn_security_event where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT app_name ,domain ,ip ,ip_country_region ,ip_super_admin_area ,ip_admin_area ,ip_asn ,ip_isp ,domain_category_name ,domain_category_group ,app_category ,app_subcategory ,entity_tags ,stat_time
|
||||
FROM cyber_narrator_galaxy.raw_session_relation_domain_ip_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT app_name, subscriber_id, imei, imsi, phone_number, apn, app_category, app_subcategory, create_time, update_time
|
||||
FROM cyber_narrator_galaxy.session_relation_subscriber_app where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT app_name , subscriber_id , imei , imsi , phone_number , apn , app_category , app_subcategory , stat_time
|
||||
FROM cyber_narrator_galaxy.raw_session_relation_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT app_name, `domain`, ip, ip_country_region, ip_super_admin_area, ip_admin_area, ip_asn, ip_isp, domain_category_name, domain_category_group, app_category, app_subcategory, entity_tags, create_time, update_time
|
||||
FROM cyber_narrator_galaxy.session_relation_domain_ip_app_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT app_name,subscriber_id,imei,imsi,phone_number,apn,app_category,app_subcategory,create_time,update_time
|
||||
FROM cyber_narrator_galaxy.session_relation_subscriber_app_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT ip, l7_protocol, port, create_time, update_time
|
||||
FROM cyber_narrator_galaxy.cn_ip_dynamic_attribute_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
|
||||
SELECT app, app_category, app_subcategory, app_company, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
|
||||
FROM cyber_narrator_galaxy.metric_application where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
@@ -36,7 +40,7 @@ FROM cyber_narrator_galaxy.metric_domain where stat_time >= toUnixTimestamp('203
|
||||
SELECT ip, side, `zone`, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms, asymmetric_sessions, bulky_sessions, cbr_streaming_sessions, download_sessions, interactive_sessions, pseudo_unidirectional_sessions, streaming_sessions, unidirectional_sessions, random_looking_sessions, bidirectional_sessions
|
||||
FROM cyber_narrator_galaxy.metric_ip where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT ip, l7_protocol, port, stat_time
|
||||
FROM cyber_narrator_galaxy.metric_ip_dynamic_attribute where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
FROM cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT client_country_region, client_super_admin_area, client_admin_area, client_zone, server_country_region, server_super_admin_area, server_admin_area, server_zone, out_link_id, in_link_id, out_link_direction, in_link_direction, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
|
||||
FROM cyber_narrator_galaxy.metric_link where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT l7_protocol, server_port, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
|
||||
@@ -45,8 +49,7 @@ SELECT country_region, super_admin_area, admin_area, side, stat_time, sent_pkts,
|
||||
FROM cyber_narrator_galaxy.metric_region where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT subscriber_id, app, imei, imsi, phone_number, apn, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
|
||||
FROM cyber_narrator_galaxy.metric_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location, stat_time
|
||||
SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source, stat_time
|
||||
FROM cyber_narrator_galaxy.location_subscriber where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, flags, start_timestamp_ms, end_timestamp_ms, duration_ms, decoded_as, client_ip, server_ip, client_port, server_port, app, app_transition, decoded_path, ip_protocol, l7_protocol, out_link_id, in_link_id, subscriber_id, imei, imsi, phone_number, apn, http_host, http_url, http_cookie, http_referer, http_user_agent, http_request_line, http_response_line, http_status_code, ssl_version, ssl_sni, ssl_san, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, dns_qr, dns_opcode, dns_aa, dns_rcode, dns_qname, dns_qtype, dns_qclass, dns_sub, dns_rr, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, out_link_direction, in_link_direction, `domain`, domain_sld, domain_category_name, domain_category_group, domain_reputation_level, domain_icp_company_name, domain_whois_org, domain_tags, client_zone, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, client_isp, client_asn, client_ip_tags, server_zone, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, server_isp, server_asn, server_ip_tags, app_category, app_subcategory, app_company, app_company_category, app_tags, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_rtt_ms, http_response_latency_ms, ssl_handshake_latency_ms, dns_response_latency_ms
|
||||
FROM cyber_narrator_galaxy.session_record_cn where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
|
||||
|
||||
Reference in New Issue
Block a user