gfwleak/galaxy-tsg-olap-storm-log-stream-completion
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

radius日志补全代码更新

Browse Source
This commit is contained in:
李玺康
2019-11-12 11:29:19 +08:00
parent 7122a40f33
commit 11a389c14f
18 changed files with 2340 additions and 1015 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
pom.xml
View File

@@ -67,7 +67,7 @@
<directory>properties</directory>
<includes>
<include>**/*.properties</include>
<!--<include>**/*.xml</include>-->
<include>**/*.xml</include>
</includes>
<filtering>false</filtering>
</resource>
@@ -132,12 +132,6 @@
<version>${storm.version}</version>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.8.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>

36
properties/hbase-site.xml
View File

@@ -31,13 +31,13 @@
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>192.168.40.202,192.168.40.203,192.168.40.206</value>
<value>master,slave1,slave2</value>
</property>
<property>
<name>hbase.master.info.port</name>
<value>60010</value>
</property>
<!-- 开启启schema支持 对应hbase的namespace -->
#开启schema支持 对应hbase的namespace
<property>
<name>phoenix.schema.isNamespaceMappingEnabled</name>
<value>true</value>
@@ -46,36 +46,4 @@
<name>phoenix.schema.mapSystemTablesToNamespace</name>
<value>true</value>
</property>
<property>
<name>hbase.client.keyvalue.maxsize</name>
<value>99428800</value>
</property>
<property>
<name>hbase.server.keyvalue.maxsize</name>
<value>99428800</value>
</property>
<property>
<name>hbase.regionserver.wal.codec</name>
<value>org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec</value>
</property>
<property>
<name>phoenix.query.timeoutMs</name>
<value>1800000</value>
</property>
<property>
<name>hbase.regionserver.lease.period</name>
<value>1200000</value>
</property>
<property>
<name>hbase.rpc.timeout</name>
<value>1200000</value>
</property>
<property>
<name>hbase.client.scanner.caching</name>
<value>1000</value>
</property>
<property>
<name>hbase.client.scanner.timeout.period</name>
<value>1200000</value>
</property>
</configuration>

12
properties/hdfs-site.xml
View File

@@ -31,7 +31,7 @@
</property>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>192.168.40.202:9001</value>
<value>master:9001</value>
</property>
<property>
<name>dfs.webhdfs.enabled</name>
@@ -60,27 +60,27 @@
<!-- nn1的RPC通信地址，nn1所在地址 -->
<property>
<name>dfs.namenode.rpc-address.ns1.nn1</name>
<value>192.168.40.202:9000</value>
<value>master:9000</value>
</property>
<!-- nn1的http通信地址，外部访问地址 -->
<property>
<name>dfs.namenode.http-address.ns1.nn1</name>
<value>192.168.40.202:50070</value>
<value>master:50070</value>
</property>
<!-- nn2的RPC通信地址，nn2所在地址 -->
<property>
<name>dfs.namenode.rpc-address.ns1.nn2</name>
<value>192.168.40.203:9000</value>
<value>slave1:9000</value>
</property>
<!-- nn2的http通信地址，外部访问地址 -->
<property>
<name>dfs.namenode.http-address.ns1.nn2</name>
<value>192.168.40.203:50070</value>
<value>slave1:50070</value>
</property>
<!-- 指定NameNode的元数据在JournalNode日志上的存放位置(一般和zookeeper部署在一起) -->
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://192.168.40.203:8485;192.168.40.206:8485;192.168.40.202:8485/ns1</value>
<value>qjournal://slave1:8485;slave2:8485;master:8485/ns1</value>
</property>
<!-- 指定JournalNode在本地磁盘存放数据的位置 -->
<property>

24
properties/service_flow_config.properties
View File

@@ -1,11 +1,11 @@
#管理kafka地址
bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092
bootstrap.servers=192.168.40.152:9092
#zookeeper 地址
zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181
zookeeper.servers=192.168.40.152:2181
#hbase zookeeper地址
hbase.zookeeper.servers=192.168.40.203:2186
hbase.zookeeper.servers=192.168.40.203:2181,192.168.40.206:2181
#hbase tablename
hbase.table.name=subcriber_info
@@ -14,29 +14,29 @@ hbase.table.name=subcriber_info
auto.offset.reset=latest
#kafka broker下的topic名称
kafka.topic=SESSION-RECORD-LOG
kafka.topic=SECURITY-POLICY-LOG
#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据；
group.id=session-completion-program
#输出topic
results.output.topic=SESSION-RECORD-COMPLETED-LOG
results.output.topic=SECURITY-POLICY-COMPLETED-LOG
#storm topology workers
topology.workers=2
topology.workers=1
#spout并行度 建议与kafka分区数相同
spout.parallelism=3
spout.parallelism=1
#处理补全操作的bolt并行度-worker的倍数
datacenter.bolt.parallelism=3
datacenter.bolt.parallelism=1
#写入kafkad的并行度
kafka.bolt.parallelism=3
#写入kafka的并行度
kafka.bolt.parallelism=1
#定位库地址
ip.library=/home/ceiec/topology/dat/
#ip.library=/home/ceiec/topology/dat/
ip.library=D:/dat/
#kafka批量条数
batch.insert.num=5000

677
src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java Normal file
View File

@@ -0,0 +1,677 @@
package cn.ac.iie.bean;
public class CollectProtocolRecordLog extends PublicSessionRecordLog{
//TODO HTTP协议属性 21
private String http_url;
private String http_host;
private String http_domain;
private String http_request_line;
private String http_response_line;
private String http_request_header;
private String http_response_header;
private String http_request_body;
private String http_response_body;
private String http_request_body_key;
private String http_response_body_key;
private int http_proxy_flag;
private int http_sequence;
private String http_snapshot;
private String http_cookie;
private String http_referer;
private String http_user_agent;
private String http_content_length;
private String http_content_type;
private String http_set_cookie;
private String http_version;
//TODO MAIL协议属性 9
private String mail_protocol_type;
private String mail_sender;
private String mail_receiver;
private String mail_subject;
private String mail_content;
private String mail_attachment_name;
private String mail_attachment_content;
private String mail_eml_file;
private String mail_snapshot;
//TODO 3DNS协议属性 18
private int dns_message_id;
private int dns_qr;
private int dns_opcode;
private int dns_aa;
private int dns_tc;
private int dns_rd;
private int dns_ra;
private int dns_rcode;
private int dns_qdcount;
private int dns_ancount;
private int dns_nscount;
private int dns_arcount;
private String dns_qname;
private int dns_qtype;
private int dns_qclass;
private String dns_cname;
private int dns_sub;
private String dns_rr;
//TODO SSL协议属性 13
private int ssl_pinningst;
private int ssl_intercept_state;
private int ssl_server_side_latency;
private int ssl_client_side_latency;
private int ssl_cert_verify;
private int ssl_con_latency_ms;
private String ssl_version;
private String ssl_sni;
private String ssl_san;
private String ssl_cn;
private String ssl_server_side_version;
private String ssl_client_side_version;
private String ssl_error;
//TODO FTP协议属性 2
private String ftp_url;
private String ftp_content;
//TODO BGP协议属性 3
private int bgp_type;
private String bgp_as_num;
private String bgp_route;
//TODO VOIP协议属性 4
private String voip_calling_account;
private String voip_called_account;
private String voip_calling_number;
private String voip_called_number;
//TODO STREAMING_MEDIA协议属性 2
private String streaming_media_url;
private String streaming_media_protocol;
public String getHttp_url() {
return http_url;
}
public void setHttp_url(String http_url) {
this.http_url = http_url;
}
public String getHttp_host() {
return http_host;
}
public void setHttp_host(String http_host) {
this.http_host = http_host;
}
public String getHttp_domain() {
return http_domain;
}
public void setHttp_domain(String http_domain) {
this.http_domain = http_domain;
}
public String getHttp_request_line() {
return http_request_line;
}
public void setHttp_request_line(String http_request_line) {
this.http_request_line = http_request_line;
}
public String getHttp_response_line() {
return http_response_line;
}
public void setHttp_response_line(String http_response_line) {
this.http_response_line = http_response_line;
}
public String getHttp_request_header() {
return http_request_header;
}
public void setHttp_request_header(String http_request_header) {
this.http_request_header = http_request_header;
}
public String getHttp_response_header() {
return http_response_header;
}
public void setHttp_response_header(String http_response_header) {
this.http_response_header = http_response_header;
}
public String getHttp_request_body() {
return http_request_body;
}
public void setHttp_request_body(String http_request_body) {
this.http_request_body = http_request_body;
}
public String getHttp_response_body() {
return http_response_body;
}
public void setHttp_response_body(String http_response_body) {
this.http_response_body = http_response_body;
}
public String getHttp_request_body_key() {
return http_request_body_key;
}
public void setHttp_request_body_key(String http_request_body_key) {
this.http_request_body_key = http_request_body_key;
}
public String getHttp_response_body_key() {
return http_response_body_key;
}
public void setHttp_response_body_key(String http_response_body_key) {
this.http_response_body_key = http_response_body_key;
}
public int getHttp_proxy_flag() {
return http_proxy_flag;
}
public void setHttp_proxy_flag(int http_proxy_flag) {
this.http_proxy_flag = http_proxy_flag;
}
public int getHttp_sequence() {
return http_sequence;
}
public void setHttp_sequence(int http_sequence) {
this.http_sequence = http_sequence;
}
public String getHttp_snapshot() {
return http_snapshot;
}
public void setHttp_snapshot(String http_snapshot) {
this.http_snapshot = http_snapshot;
}
public String getHttp_cookie() {
return http_cookie;
}
public void setHttp_cookie(String http_cookie) {
this.http_cookie = http_cookie;
}
public String getHttp_referer() {
return http_referer;
}
public void setHttp_referer(String http_referer) {
this.http_referer = http_referer;
}
public String getHttp_user_agent() {
return http_user_agent;
}
public void setHttp_user_agent(String http_user_agent) {
this.http_user_agent = http_user_agent;
}
public String getHttp_content_length() {
return http_content_length;
}
public void setHttp_content_length(String http_content_length) {
this.http_content_length = http_content_length;
}
public String getHttp_content_type() {
return http_content_type;
}
public void setHttp_content_type(String http_content_type) {
this.http_content_type = http_content_type;
}
public String getHttp_set_cookie() {
return http_set_cookie;
}
public void setHttp_set_cookie(String http_set_cookie) {
this.http_set_cookie = http_set_cookie;
}
public String getHttp_version() {
return http_version;
}
public void setHttp_version(String http_version) {
this.http_version = http_version;
}
public String getMail_protocol_type() {
return mail_protocol_type;
}
public void setMail_protocol_type(String mail_protocol_type) {
this.mail_protocol_type = mail_protocol_type;
}
public String getMail_sender() {
return mail_sender;
}
public void setMail_sender(String mail_sender) {
this.mail_sender = mail_sender;
}
public String getMail_receiver() {
return mail_receiver;
}
public void setMail_receiver(String mail_receiver) {
this.mail_receiver = mail_receiver;
}
public String getMail_subject() {
return mail_subject;
}
public void setMail_subject(String mail_subject) {
this.mail_subject = mail_subject;
}
public String getMail_content() {
return mail_content;
}
public void setMail_content(String mail_content) {
this.mail_content = mail_content;
}
public String getMail_attachment_name() {
return mail_attachment_name;
}
public void setMail_attachment_name(String mail_attachment_name) {
this.mail_attachment_name = mail_attachment_name;
}
public String getMail_attachment_content() {
return mail_attachment_content;
}
public void setMail_attachment_content(String mail_attachment_content) {
this.mail_attachment_content = mail_attachment_content;
}
public String getMail_eml_file() {
return mail_eml_file;
}
public void setMail_eml_file(String mail_eml_file) {
this.mail_eml_file = mail_eml_file;
}
public String getMail_snapshot() {
return mail_snapshot;
}
public void setMail_snapshot(String mail_snapshot) {
this.mail_snapshot = mail_snapshot;
}
public int getDns_message_id() {
return dns_message_id;
}
public void setDns_message_id(int dns_message_id) {
this.dns_message_id = dns_message_id;
}
public int getDns_qr() {
return dns_qr;
}
public void setDns_qr(int dns_qr) {
this.dns_qr = dns_qr;
}
public int getDns_opcode() {
return dns_opcode;
}
public void setDns_opcode(int dns_opcode) {
this.dns_opcode = dns_opcode;
}
public int getDns_aa() {
return dns_aa;
}
public void setDns_aa(int dns_aa) {
this.dns_aa = dns_aa;
}
public int getDns_tc() {
return dns_tc;
}
public void setDns_tc(int dns_tc) {
this.dns_tc = dns_tc;
}
public int getDns_rd() {
return dns_rd;
}
public void setDns_rd(int dns_rd) {
this.dns_rd = dns_rd;
}
public int getDns_ra() {
return dns_ra;
}
public void setDns_ra(int dns_ra) {
this.dns_ra = dns_ra;
}
public int getDns_rcode() {
return dns_rcode;
}
public void setDns_rcode(int dns_rcode) {
this.dns_rcode = dns_rcode;
}
public int getDns_qdcount() {
return dns_qdcount;
}
public void setDns_qdcount(int dns_qdcount) {
this.dns_qdcount = dns_qdcount;
}
public int getDns_ancount() {
return dns_ancount;
}
public void setDns_ancount(int dns_ancount) {
this.dns_ancount = dns_ancount;
}
public int getDns_nscount() {
return dns_nscount;
}
public void setDns_nscount(int dns_nscount) {
this.dns_nscount = dns_nscount;
}
public int getDns_arcount() {
return dns_arcount;
}
public void setDns_arcount(int dns_arcount) {
this.dns_arcount = dns_arcount;
}
public String getDns_qname() {
return dns_qname;
}
public void setDns_qname(String dns_qname) {
this.dns_qname = dns_qname;
}
public int getDns_qtype() {
return dns_qtype;
}
public void setDns_qtype(int dns_qtype) {
this.dns_qtype = dns_qtype;
}
public int getDns_qclass() {
return dns_qclass;
}
public void setDns_qclass(int dns_qclass) {
this.dns_qclass = dns_qclass;
}
public String getDns_cname() {
return dns_cname;
}
public void setDns_cname(String dns_cname) {
this.dns_cname = dns_cname;
}
public int getDns_sub() {
return dns_sub;
}
public void setDns_sub(int dns_sub) {
this.dns_sub = dns_sub;
}
public String getDns_rr() {
return dns_rr;
}
public void setDns_rr(String dns_rr) {
this.dns_rr = dns_rr;
}
public int getSsl_pinningst() {
return ssl_pinningst;
}
public void setSsl_pinningst(int ssl_pinningst) {
this.ssl_pinningst = ssl_pinningst;
}
public int getSsl_intercept_state() {
return ssl_intercept_state;
}
public void setSsl_intercept_state(int ssl_intercept_state) {
this.ssl_intercept_state = ssl_intercept_state;
}
public int getSsl_server_side_latency() {
return ssl_server_side_latency;
}
public void setSsl_server_side_latency(int ssl_server_side_latency) {
this.ssl_server_side_latency = ssl_server_side_latency;
}
public int getSsl_client_side_latency() {
return ssl_client_side_latency;
}
public void setSsl_client_side_latency(int ssl_client_side_latency) {
this.ssl_client_side_latency = ssl_client_side_latency;
}
public int getSsl_cert_verify() {
return ssl_cert_verify;
}
public void setSsl_cert_verify(int ssl_cert_verify) {
this.ssl_cert_verify = ssl_cert_verify;
}
public int getSsl_con_latency_ms() {
return ssl_con_latency_ms;
}
public void setSsl_con_latency_ms(int ssl_con_latency_ms) {
this.ssl_con_latency_ms = ssl_con_latency_ms;
}
public String getSsl_version() {
return ssl_version;
}
public void setSsl_version(String ssl_version) {
this.ssl_version = ssl_version;
}
public String getSsl_sni() {
return ssl_sni;
}
public void setSsl_sni(String ssl_sni) {
this.ssl_sni = ssl_sni;
}
public String getSsl_san() {
return ssl_san;
}
public void setSsl_san(String ssl_san) {
this.ssl_san = ssl_san;
}
public String getSsl_cn() {
return ssl_cn;
}
public void setSsl_cn(String ssl_cn) {
this.ssl_cn = ssl_cn;
}
public String getSsl_server_side_version() {
return ssl_server_side_version;
}
public void setSsl_server_side_version(String ssl_server_side_version) {
this.ssl_server_side_version = ssl_server_side_version;
}
public String getSsl_client_side_version() {
return ssl_client_side_version;
}
public void setSsl_client_side_version(String ssl_client_side_version) {
this.ssl_client_side_version = ssl_client_side_version;
}
public String getSsl_error() {
return ssl_error;
}
public void setSsl_error(String ssl_error) {
this.ssl_error = ssl_error;
}
public String getFtp_url() {
return ftp_url;
}
public void setFtp_url(String ftp_url) {
this.ftp_url = ftp_url;
}
public String getFtp_content() {
return ftp_content;
}
public void setFtp_content(String ftp_content) {
this.ftp_content = ftp_content;
}
public int getBgp_type() {
return bgp_type;
}
public void setBgp_type(int bgp_type) {
this.bgp_type = bgp_type;
}
public String getBgp_as_num() {
return bgp_as_num;
}
public void setBgp_as_num(String bgp_as_num) {
this.bgp_as_num = bgp_as_num;
}
public String getBgp_route() {
return bgp_route;
}
public void setBgp_route(String bgp_route) {
this.bgp_route = bgp_route;
}
public String getVoip_calling_account() {
return voip_calling_account;
}
public void setVoip_calling_account(String voip_calling_account) {
this.voip_calling_account = voip_calling_account;
}
public String getVoip_called_account() {
return voip_called_account;
}
public void setVoip_called_account(String voip_called_account) {
this.voip_called_account = voip_called_account;
}
public String getVoip_calling_number() {
return voip_calling_number;
}
public void setVoip_calling_number(String voip_calling_number) {
this.voip_calling_number = voip_calling_number;
}
public String getVoip_called_number() {
return voip_called_number;
}
public void setVoip_called_number(String voip_called_number) {
this.voip_called_number = voip_called_number;
}
public String getStreaming_media_url() {
return streaming_media_url;
}
public void setStreaming_media_url(String streaming_media_url) {
this.streaming_media_url = streaming_media_url;
}
public String getStreaming_media_protocol() {
return streaming_media_protocol;
}
public void setStreaming_media_protocol(String streaming_media_protocol) {
this.streaming_media_protocol = streaming_media_protocol;
}
}

198
src/main/java/cn/ac/iie/bean/ProxySessionRecordLog.java Normal file
View File

@@ -0,0 +1,198 @@
package cn.ac.iie.bean;
import com.alibaba.fastjson.annotation.JSONField;
import com.alibaba.fastjson.support.spring.annotation.FastJsonFilter;
public class ProxySessionRecordLog extends PublicSessionRecordLog{
//TODO HTTP协议属性 21
private String http_url;
private String http_host;
private String http_domain;
private String http_request_line;
private String http_response_line;
private String http_request_header;
private String http_response_header;
private String http_request_body;
private String http_response_body;
private String http_request_body_key;
private String http_response_body_key;
private int http_proxy_flag;
private int http_sequence;
private String http_snapshot;
private String http_cookie;
private String http_referer;
private String http_user_agent;
private String http_content_length;
private String http_content_type;
private String http_set_cookie;
private String http_version;
public String getHttp_url() {
return http_url;
}
public void setHttp_url(String http_url) {
this.http_url = http_url;
}
public String getHttp_host() {
return http_host;
}
public void setHttp_host(String http_host) {
this.http_host = http_host;
}
public String getHttp_domain() {
return http_domain;
}
public void setHttp_domain(String http_domain) {
this.http_domain = http_domain;
}
public String getHttp_request_line() {
return http_request_line;
}
public void setHttp_request_line(String http_request_line) {
this.http_request_line = http_request_line;
}
public String getHttp_response_line() {
return http_response_line;
}
public void setHttp_response_line(String http_response_line) {
this.http_response_line = http_response_line;
}
public String getHttp_request_header() {
return http_request_header;
}
public void setHttp_request_header(String http_request_header) {
this.http_request_header = http_request_header;
}
public String getHttp_response_header() {
return http_response_header;
}
public void setHttp_response_header(String http_response_header) {
this.http_response_header = http_response_header;
}
public String getHttp_request_body() {
return http_request_body;
}
public void setHttp_request_body(String http_request_body) {
this.http_request_body = http_request_body;
}
public String getHttp_response_body() {
return http_response_body;
}
public void setHttp_response_body(String http_response_body) {
this.http_response_body = http_response_body;
}
public String getHttp_request_body_key() {
return http_request_body_key;
}
public void setHttp_request_body_key(String http_request_body_key) {
this.http_request_body_key = http_request_body_key;
}
public String getHttp_response_body_key() {
return http_response_body_key;
}
public void setHttp_response_body_key(String http_response_body_key) {
this.http_response_body_key = http_response_body_key;
}
public int getHttp_proxy_flag() {
return http_proxy_flag;
}
public void setHttp_proxy_flag(int http_proxy_flag) {
this.http_proxy_flag = http_proxy_flag;
}
public int getHttp_sequence() {
return http_sequence;
}
public void setHttp_sequence(int http_sequence) {
this.http_sequence = http_sequence;
}
public String getHttp_snapshot() {
return http_snapshot;
}
public void setHttp_snapshot(String http_snapshot) {
this.http_snapshot = http_snapshot;
}
public String getHttp_cookie() {
return http_cookie;
}
public void setHttp_cookie(String http_cookie) {
this.http_cookie = http_cookie;
}
public String getHttp_referer() {
return http_referer;
}
public void setHttp_referer(String http_referer) {
this.http_referer = http_referer;
}
public String getHttp_user_agent() {
return http_user_agent;
}
public void setHttp_user_agent(String http_user_agent) {
this.http_user_agent = http_user_agent;
}
public String getHttp_content_length() {
return http_content_length;
}
public void setHttp_content_length(String http_content_length) {
this.http_content_length = http_content_length;
}
public String getHttp_content_type() {
return http_content_type;
}
public void setHttp_content_type(String http_content_type) {
this.http_content_type = http_content_type;
}
public String getHttp_set_cookie() {
return http_set_cookie;
}
public void setHttp_set_cookie(String http_set_cookie) {
this.http_set_cookie = http_set_cookie;
}
public String getHttp_version() {
return http_version;
}
public void setHttp_version(String http_version) {
this.http_version = http_version;
}
}

380
src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java Normal file
View File

@@ -0,0 +1,380 @@
package cn.ac.iie.bean;
import com.alibaba.fastjson.annotation.JSONField;
import com.alibaba.fastjson.support.spring.annotation.FastJsonFilter;
public class PublicSessionRecordLog {
//TODO 基础属性 40 int22
private int common_service;
private int common_direction;
private long common_recv_time;
private int common_address_type;
private int common_policy_id;
private int common_action;
private int common_client_port;
private int common_entrance_id;
private int common_link_id;
private int common_encapsulation;
private int common_server_port;
private int common_app_id;
private int common_protocol_id;
private int common_c2s_pkt_num;
private int common_s2c_pkt_num;
private long common_c2s_byte_num;
private long common_s2c_byte_num;
private int common_start_time;
private int common_end_time;
private int common_con_duration_ms;
private int common_stream_dir;
private int common_has_dup_traffic;
private long common_log_id;
private String common_schema_type;
private String common_l4_protocol;
private String common_user_tags;
private String common_user_region;
private String common_client_ip;
private String common_device_id;
private String common_isp;
private String common_sled_ip;
private String common_client_location;
private String common_client_asn;
private String common_subscriber_id;
private String common_server_ip;
private String common_server_location;
private String common_server_asn;
private String common_app_label;
private String common_address_list;
private String common_stream_error;
private String common_stream_trace_id;
public int getCommon_service() {
return common_service;
}
public void setCommon_service(int common_service) {
this.common_service = common_service;
}
public int getCommon_direction() {
return common_direction;
}
public void setCommon_direction(int common_direction) {
this.common_direction = common_direction;
}
public long getCommon_recv_time() {
return common_recv_time;
}
public void setCommon_recv_time(long common_recv_time) {
this.common_recv_time = common_recv_time;
}
public int getCommon_address_type() {
return common_address_type;
}
public void setCommon_address_type(int common_address_type) {
this.common_address_type = common_address_type;
}
public int getCommon_policy_id() {
return common_policy_id;
}
public void setCommon_policy_id(int common_policy_id) {
this.common_policy_id = common_policy_id;
}
public int getCommon_action() {
return common_action;
}
public void setCommon_action(int common_action) {
this.common_action = common_action;
}
public int getCommon_client_port() {
return common_client_port;
}
public void setCommon_client_port(int common_client_port) {
this.common_client_port = common_client_port;
}
public int getCommon_entrance_id() {
return common_entrance_id;
}
public void setCommon_entrance_id(int common_entrance_id) {
this.common_entrance_id = common_entrance_id;
}
public int getCommon_link_id() {
return common_link_id;
}
public void setCommon_link_id(int common_link_id) {
this.common_link_id = common_link_id;
}
public int getCommon_encapsulation() {
return common_encapsulation;
}
public void setCommon_encapsulation(int common_encapsulation) {
this.common_encapsulation = common_encapsulation;
}
public int getCommon_server_port() {
return common_server_port;
}
public void setCommon_server_port(int common_server_port) {
this.common_server_port = common_server_port;
}
public int getCommon_app_id() {
return common_app_id;
}
public void setCommon_app_id(int common_app_id) {
this.common_app_id = common_app_id;
}
public int getCommon_protocol_id() {
return common_protocol_id;
}
public void setCommon_protocol_id(int common_protocol_id) {
this.common_protocol_id = common_protocol_id;
}
public int getCommon_c2s_pkt_num() {
return common_c2s_pkt_num;
}
public void setCommon_c2s_pkt_num(int common_c2s_pkt_num) {
this.common_c2s_pkt_num = common_c2s_pkt_num;
}
public int getCommon_s2c_pkt_num() {
return common_s2c_pkt_num;
}
public void setCommon_s2c_pkt_num(int common_s2c_pkt_num) {
this.common_s2c_pkt_num = common_s2c_pkt_num;
}
public long getCommon_c2s_byte_num() {
return common_c2s_byte_num;
}
public void setCommon_c2s_byte_num(long common_c2s_byte_num) {
this.common_c2s_byte_num = common_c2s_byte_num;
}
public long getCommon_s2c_byte_num() {
return common_s2c_byte_num;
}
public void setCommon_s2c_byte_num(long common_s2c_byte_num) {
this.common_s2c_byte_num = common_s2c_byte_num;
}
public int getCommon_start_time() {
return common_start_time;
}
public void setCommon_start_time(int common_start_time) {
this.common_start_time = common_start_time;
}
public int getCommon_end_time() {
return common_end_time;
}
public void setCommon_end_time(int common_end_time) {
this.common_end_time = common_end_time;
}
public int getCommon_con_duration_ms() {
return common_con_duration_ms;
}
public void setCommon_con_duration_ms(int common_con_duration_ms) {
this.common_con_duration_ms = common_con_duration_ms;
}
public int getCommon_stream_dir() {
return common_stream_dir;
}
public void setCommon_stream_dir(int common_stream_dir) {
this.common_stream_dir = common_stream_dir;
}
public int getCommon_has_dup_traffic() {
return common_has_dup_traffic;
}
public void setCommon_has_dup_traffic(int common_has_dup_traffic) {
this.common_has_dup_traffic = common_has_dup_traffic;
}
public String getCommon_schema_type() {
return common_schema_type;
}
public void setCommon_schema_type(String common_schema_type) {
this.common_schema_type = common_schema_type;
}
public long getCommon_log_id() {
return common_log_id;
}
public void setCommon_log_id(long common_log_id) {
this.common_log_id = common_log_id;
}
public String getCommon_l4_protocol() {
return common_l4_protocol;
}
public void setCommon_l4_protocol(String common_l4_protocol) {
this.common_l4_protocol = common_l4_protocol;
}
public String getCommon_user_tags() {
return common_user_tags;
}
public void setCommon_user_tags(String common_user_tags) {
this.common_user_tags = common_user_tags;
}
public String getCommon_user_region() {
return common_user_region;
}
public void setCommon_user_region(String common_user_region) {
this.common_user_region = common_user_region;
}
public String getCommon_client_ip() {
return common_client_ip;
}
public void setCommon_client_ip(String common_client_ip) {
this.common_client_ip = common_client_ip;
}
public String getCommon_device_id() {
return common_device_id;
}
public void setCommon_device_id(String common_device_id) {
this.common_device_id = common_device_id;
}
public String getCommon_isp() {
return common_isp;
}
public void setCommon_isp(String common_isp) {
this.common_isp = common_isp;
}
public String getCommon_sled_ip() {
return common_sled_ip;
}
public void setCommon_sled_ip(String common_sled_ip) {
this.common_sled_ip = common_sled_ip;
}
public String getCommon_client_location() {
return common_client_location;
}
public void setCommon_client_location(String common_client_location) {
this.common_client_location = common_client_location;
}
public String getCommon_client_asn() {
return common_client_asn;
}
public void setCommon_client_asn(String common_client_asn) {
this.common_client_asn = common_client_asn;
}
public String getCommon_subscriber_id() {
return common_subscriber_id;
}
public void setCommon_subscriber_id(String common_subscriber_id) {
this.common_subscriber_id = common_subscriber_id;
}
public String getCommon_server_ip() {
return common_server_ip;
}
public void setCommon_server_ip(String common_server_ip) {
this.common_server_ip = common_server_ip;
}
public String getCommon_server_location() {
return common_server_location;
}
public void setCommon_server_location(String common_server_location) {
this.common_server_location = common_server_location;
}
public String getCommon_server_asn() {
return common_server_asn;
}
public void setCommon_server_asn(String common_server_asn) {
this.common_server_asn = common_server_asn;
}
public String getCommon_app_label() {
return common_app_label;
}
public void setCommon_app_label(String common_app_label) {
this.common_app_label = common_app_label;
}
public String getCommon_address_list() {
return common_address_list;
}
public void setCommon_address_list(String common_address_list) {
this.common_address_list = common_address_list;
}
public String getCommon_stream_error() {
return common_stream_error;
}
public void setCommon_stream_error(String common_stream_error) {
this.common_stream_error = common_stream_error;
}
public String getCommon_stream_trace_id() {
return common_stream_trace_id;
}
public void setCommon_stream_trace_id(String common_stream_trace_id) {
this.common_stream_trace_id = common_stream_trace_id;
}
}

84
src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java Normal file
View File

@@ -0,0 +1,84 @@
package cn.ac.iie.bean;
import com.alibaba.fastjson.annotation.JSONField;
import com.alibaba.fastjson.support.spring.annotation.FastJsonFilter;
public class RadiusSessionRecordLog extends PublicSessionRecordLog{
//TODO RADIUS协议属性 4
private int radius_session_timeout;
private int radius_idle_timeout;
private int radius_acct_status_type;
private int radius_acct_terminate_cause;
public int getRadius_session_timeout() {
return radius_session_timeout;
}
public void setRadius_session_timeout(int radius_session_timeout) {
this.radius_session_timeout = radius_session_timeout;
}
public int getRadius_idle_timeout() {
return radius_idle_timeout;
}
public void setRadius_idle_timeout(int radius_idle_timeout) {
this.radius_idle_timeout = radius_idle_timeout;
}
public int getRadius_acct_status_type() {
return radius_acct_status_type;
}
public void setRadius_acct_status_type(int radius_acct_status_type) {
this.radius_acct_status_type = radius_acct_status_type;
}
public int getRadius_acct_terminate_cause() {
return radius_acct_terminate_cause;
}
public void setRadius_acct_terminate_cause(int radius_acct_terminate_cause) {
this.radius_acct_terminate_cause = radius_acct_terminate_cause;
}
private int radius_packet_type;
private String radius_nas_ip;
private String radius_framed_ip;
private String radius_account;
public int getRadius_packet_type() {
return radius_packet_type;
}
public void setRadius_packet_type(int radius_packet_type) {
this.radius_packet_type = radius_packet_type;
}
public String getRadius_nas_ip() {
return radius_nas_ip;
}
public void setRadius_nas_ip(String radius_nas_ip) {
this.radius_nas_ip = radius_nas_ip;
}
public String getRadius_framed_ip() {
return radius_framed_ip;
}
public void setRadius_framed_ip(String radius_framed_ip) {
this.radius_framed_ip = radius_framed_ip;
}
public String getRadius_account() {
return radius_account;
}
public void setRadius_account(String radius_account) {
this.radius_account = radius_account;
}
}

677
src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java Normal file
View File

@@ -0,0 +1,677 @@
package cn.ac.iie.bean;
public class SecurityPolicyLog extends PublicSessionRecordLog{
//TODO HTTP协议属性 21
private String http_url;
private String http_host;
private String http_domain;
private String http_request_line;
private String http_response_line;
private String http_request_header;
private String http_response_header;
private String http_request_body;
private String http_response_body;
private String http_request_body_key;
private String http_response_body_key;
private int http_proxy_flag;
private int http_sequence;
private String http_snapshot;
private String http_cookie;
private String http_referer;
private String http_user_agent;
private String http_content_length;
private String http_content_type;
private String http_set_cookie;
private String http_version;
//TODO MAIL协议属性 9
private String mail_protocol_type;
private String mail_sender;
private String mail_receiver;
private String mail_subject;
private String mail_content;
private String mail_attachment_name;
private String mail_attachment_content;
private String mail_eml_file;
private String mail_snapshot;
//TODO 3DNS协议属性 18
private int dns_message_id;
private int dns_qr;
private int dns_opcode;
private int dns_aa;
private int dns_tc;
private int dns_rd;
private int dns_ra;
private int dns_rcode;
private int dns_qdcount;
private int dns_ancount;
private int dns_nscount;
private int dns_arcount;
private String dns_qname;
private int dns_qtype;
private int dns_qclass;
private String dns_cname;
private int dns_sub;
private String dns_rr;
//TODO SSL协议属性 13
private int ssl_pinningst;
private int ssl_intercept_state;
private int ssl_server_side_latency;
private int ssl_client_side_latency;
private int ssl_cert_verify;
private int ssl_con_latency_ms;
private String ssl_version;
private String ssl_sni;
private String ssl_san;
private String ssl_cn;
private String ssl_server_side_version;
private String ssl_client_side_version;
private String ssl_error;
//TODO FTP协议属性 2
private String ftp_url;
private String ftp_content;
//TODO BGP协议属性 3
private int bgp_type;
private String bgp_as_num;
private String bgp_route;
//TODO VOIP协议属性 4
private String voip_calling_account;
private String voip_called_account;
private String voip_calling_number;
private String voip_called_number;
//TODO STREAMING_MEDIA协议属性 2
private String streaming_media_url;
private String streaming_media_protocol;
public String getHttp_url() {
return http_url;
}
public void setHttp_url(String http_url) {
this.http_url = http_url;
}
public String getHttp_host() {
return http_host;
}
public void setHttp_host(String http_host) {
this.http_host = http_host;
}
public String getHttp_domain() {
return http_domain;
}
public void setHttp_domain(String http_domain) {
this.http_domain = http_domain;
}
public String getHttp_request_line() {
return http_request_line;
}
public void setHttp_request_line(String http_request_line) {
this.http_request_line = http_request_line;
}
public String getHttp_response_line() {
return http_response_line;
}
public void setHttp_response_line(String http_response_line) {
this.http_response_line = http_response_line;
}
public String getHttp_request_header() {
return http_request_header;
}
public void setHttp_request_header(String http_request_header) {
this.http_request_header = http_request_header;
}
public String getHttp_response_header() {
return http_response_header;
}
public void setHttp_response_header(String http_response_header) {
this.http_response_header = http_response_header;
}
public String getHttp_request_body() {
return http_request_body;
}
public void setHttp_request_body(String http_request_body) {
this.http_request_body = http_request_body;
}
public String getHttp_response_body() {
return http_response_body;
}
public void setHttp_response_body(String http_response_body) {
this.http_response_body = http_response_body;
}
public String getHttp_request_body_key() {
return http_request_body_key;
}
public void setHttp_request_body_key(String http_request_body_key) {
this.http_request_body_key = http_request_body_key;
}
public String getHttp_response_body_key() {
return http_response_body_key;
}
public void setHttp_response_body_key(String http_response_body_key) {
this.http_response_body_key = http_response_body_key;
}
public int getHttp_proxy_flag() {
return http_proxy_flag;
}
public void setHttp_proxy_flag(int http_proxy_flag) {
this.http_proxy_flag = http_proxy_flag;
}
public int getHttp_sequence() {
return http_sequence;
}
public void setHttp_sequence(int http_sequence) {
this.http_sequence = http_sequence;
}
public String getHttp_snapshot() {
return http_snapshot;
}
public void setHttp_snapshot(String http_snapshot) {
this.http_snapshot = http_snapshot;
}
public String getHttp_cookie() {
return http_cookie;
}
public void setHttp_cookie(String http_cookie) {
this.http_cookie = http_cookie;
}
public String getHttp_referer() {
return http_referer;
}
public void setHttp_referer(String http_referer) {
this.http_referer = http_referer;
}
public String getHttp_user_agent() {
return http_user_agent;
}
public void setHttp_user_agent(String http_user_agent) {
this.http_user_agent = http_user_agent;
}
public String getHttp_content_length() {
return http_content_length;
}
public void setHttp_content_length(String http_content_length) {
this.http_content_length = http_content_length;
}
public String getHttp_content_type() {
return http_content_type;
}
public void setHttp_content_type(String http_content_type) {
this.http_content_type = http_content_type;
}
public String getHttp_set_cookie() {
return http_set_cookie;
}
public void setHttp_set_cookie(String http_set_cookie) {
this.http_set_cookie = http_set_cookie;
}
public String getHttp_version() {
return http_version;
}
public void setHttp_version(String http_version) {
this.http_version = http_version;
}
public String getMail_protocol_type() {
return mail_protocol_type;
}
public void setMail_protocol_type(String mail_protocol_type) {
this.mail_protocol_type = mail_protocol_type;
}
public String getMail_sender() {
return mail_sender;
}
public void setMail_sender(String mail_sender) {
this.mail_sender = mail_sender;
}
public String getMail_receiver() {
return mail_receiver;
}
public void setMail_receiver(String mail_receiver) {
this.mail_receiver = mail_receiver;
}
public String getMail_subject() {
return mail_subject;
}
public void setMail_subject(String mail_subject) {
this.mail_subject = mail_subject;
}
public String getMail_content() {
return mail_content;
}
public void setMail_content(String mail_content) {
this.mail_content = mail_content;
}
public String getMail_attachment_name() {
return mail_attachment_name;
}
public void setMail_attachment_name(String mail_attachment_name) {
this.mail_attachment_name = mail_attachment_name;
}
public String getMail_attachment_content() {
return mail_attachment_content;
}
public void setMail_attachment_content(String mail_attachment_content) {
this.mail_attachment_content = mail_attachment_content;
}
public String getMail_eml_file() {
return mail_eml_file;
}
public void setMail_eml_file(String mail_eml_file) {
this.mail_eml_file = mail_eml_file;
}
public String getMail_snapshot() {
return mail_snapshot;
}
public void setMail_snapshot(String mail_snapshot) {
this.mail_snapshot = mail_snapshot;
}
public int getDns_message_id() {
return dns_message_id;
}
public void setDns_message_id(int dns_message_id) {
this.dns_message_id = dns_message_id;
}
public int getDns_qr() {
return dns_qr;
}
public void setDns_qr(int dns_qr) {
this.dns_qr = dns_qr;
}
public int getDns_opcode() {
return dns_opcode;
}
public void setDns_opcode(int dns_opcode) {
this.dns_opcode = dns_opcode;
}
public int getDns_aa() {
return dns_aa;
}
public void setDns_aa(int dns_aa) {
this.dns_aa = dns_aa;
}
public int getDns_tc() {
return dns_tc;
}
public void setDns_tc(int dns_tc) {
this.dns_tc = dns_tc;
}
public int getDns_rd() {
return dns_rd;
}
public void setDns_rd(int dns_rd) {
this.dns_rd = dns_rd;
}
public int getDns_ra() {
return dns_ra;
}
public void setDns_ra(int dns_ra) {
this.dns_ra = dns_ra;
}
public int getDns_rcode() {
return dns_rcode;
}
public void setDns_rcode(int dns_rcode) {
this.dns_rcode = dns_rcode;
}
public int getDns_qdcount() {
return dns_qdcount;
}
public void setDns_qdcount(int dns_qdcount) {
this.dns_qdcount = dns_qdcount;
}
public int getDns_ancount() {
return dns_ancount;
}
public void setDns_ancount(int dns_ancount) {
this.dns_ancount = dns_ancount;
}
public int getDns_nscount() {
return dns_nscount;
}
public void setDns_nscount(int dns_nscount) {
this.dns_nscount = dns_nscount;
}
public int getDns_arcount() {
return dns_arcount;
}
public void setDns_arcount(int dns_arcount) {
this.dns_arcount = dns_arcount;
}
public String getDns_qname() {
return dns_qname;
}
public void setDns_qname(String dns_qname) {
this.dns_qname = dns_qname;
}
public int getDns_qtype() {
return dns_qtype;
}
public void setDns_qtype(int dns_qtype) {
this.dns_qtype = dns_qtype;
}
public int getDns_qclass() {
return dns_qclass;
}
public void setDns_qclass(int dns_qclass) {
this.dns_qclass = dns_qclass;
}
public String getDns_cname() {
return dns_cname;
}
public void setDns_cname(String dns_cname) {
this.dns_cname = dns_cname;
}
public int getDns_sub() {
return dns_sub;
}
public void setDns_sub(int dns_sub) {
this.dns_sub = dns_sub;
}
public String getDns_rr() {
return dns_rr;
}
public void setDns_rr(String dns_rr) {
this.dns_rr = dns_rr;
}
public int getSsl_pinningst() {
return ssl_pinningst;
}
public void setSsl_pinningst(int ssl_pinningst) {
this.ssl_pinningst = ssl_pinningst;
}
public int getSsl_intercept_state() {
return ssl_intercept_state;
}
public void setSsl_intercept_state(int ssl_intercept_state) {
this.ssl_intercept_state = ssl_intercept_state;
}
public int getSsl_server_side_latency() {
return ssl_server_side_latency;
}
public void setSsl_server_side_latency(int ssl_server_side_latency) {
this.ssl_server_side_latency = ssl_server_side_latency;
}
public int getSsl_client_side_latency() {
return ssl_client_side_latency;
}
public void setSsl_client_side_latency(int ssl_client_side_latency) {
this.ssl_client_side_latency = ssl_client_side_latency;
}
public int getSsl_cert_verify() {
return ssl_cert_verify;
}
public void setSsl_cert_verify(int ssl_cert_verify) {
this.ssl_cert_verify = ssl_cert_verify;
}
public int getSsl_con_latency_ms() {
return ssl_con_latency_ms;
}
public void setSsl_con_latency_ms(int ssl_con_latency_ms) {
this.ssl_con_latency_ms = ssl_con_latency_ms;
}
public String getSsl_version() {
return ssl_version;
}
public void setSsl_version(String ssl_version) {
this.ssl_version = ssl_version;
}
public String getSsl_sni() {
return ssl_sni;
}
public void setSsl_sni(String ssl_sni) {
this.ssl_sni = ssl_sni;
}
public String getSsl_san() {
return ssl_san;
}
public void setSsl_san(String ssl_san) {
this.ssl_san = ssl_san;
}
public String getSsl_cn() {
return ssl_cn;
}
public void setSsl_cn(String ssl_cn) {
this.ssl_cn = ssl_cn;
}
public String getSsl_server_side_version() {
return ssl_server_side_version;
}
public void setSsl_server_side_version(String ssl_server_side_version) {
this.ssl_server_side_version = ssl_server_side_version;
}
public String getSsl_client_side_version() {
return ssl_client_side_version;
}
public void setSsl_client_side_version(String ssl_client_side_version) {
this.ssl_client_side_version = ssl_client_side_version;
}
public String getSsl_error() {
return ssl_error;
}
public void setSsl_error(String ssl_error) {
this.ssl_error = ssl_error;
}
public String getFtp_url() {
return ftp_url;
}
public void setFtp_url(String ftp_url) {
this.ftp_url = ftp_url;
}
public String getFtp_content() {
return ftp_content;
}
public void setFtp_content(String ftp_content) {
this.ftp_content = ftp_content;
}
public int getBgp_type() {
return bgp_type;
}
public void setBgp_type(int bgp_type) {
this.bgp_type = bgp_type;
}
public String getBgp_as_num() {
return bgp_as_num;
}
public void setBgp_as_num(String bgp_as_num) {
this.bgp_as_num = bgp_as_num;
}
public String getBgp_route() {
return bgp_route;
}
public void setBgp_route(String bgp_route) {
this.bgp_route = bgp_route;
}
public String getVoip_calling_account() {
return voip_calling_account;
}
public void setVoip_calling_account(String voip_calling_account) {
this.voip_calling_account = voip_calling_account;
}
public String getVoip_called_account() {
return voip_called_account;
}
public void setVoip_called_account(String voip_called_account) {
this.voip_called_account = voip_called_account;
}
public String getVoip_calling_number() {
return voip_calling_number;
}
public void setVoip_calling_number(String voip_calling_number) {
this.voip_calling_number = voip_calling_number;
}
public String getVoip_called_number() {
return voip_called_number;
}
public void setVoip_called_number(String voip_called_number) {
this.voip_called_number = voip_called_number;
}
public String getStreaming_media_url() {
return streaming_media_url;
}
public void setStreaming_media_url(String streaming_media_url) {
this.streaming_media_url = streaming_media_url;
}
public String getStreaming_media_protocol() {
return streaming_media_protocol;
}
public void setStreaming_media_protocol(String streaming_media_protocol) {
this.streaming_media_protocol = streaming_media_protocol;
}
}

672
src/main/java/cn/ac/iie/bean/SessionRecordLog.java
View File

@@ -1,672 +0,0 @@
package cn.ac.iie.bean;
import com.alibaba.fastjson.annotation.JSONField;
import com.alibaba.fastjson.support.spring.annotation.FastJsonFilter;
/**
* @author qidaijie
*/
public class SessionRecordLog {
private long uid;
private int policy_id;
private long action;
private int start_time;
private int end_time;
private long recv_time;
private String trans_proto;
private String app_proto;
private int addr_type;
private String server_ip;
private String client_ip;
private int server_port;
private int client_port;
private int service;
private int entrance_id;
private int device_id;
private int Link_id;
private String isp;
private int encap_type;
private int direction;
private int stream_dir;
private String cap_ip;
private String addr_list;
private String server_location;
private String client_location;
private String client_asn;
private String server_asn;
private String subscribe_id;
private long con_duration_ms;
private String url;
private String host;
private String domain;
private String category;
private String req_line;
private String res_line;
private String cookie;
private String referer;
private String user_agent;
private String content_len;
private String content_type;
private String set_cookie;
private String req_header;
private String resp_header;
private String req_body_key;
private String req_body;
private String res_body_key;
private String resp_body;
private String version;
private String sni;
private String san;
private String cn;
private int app_id;
private int protocol_id;
private long con_latency_ms;
private int pinningst;
private int intercept_state;
private long ssl_server_side_latency;
private long ssl_client_side_latency;
private String ssl_server_side_version;
private String ssl_client_side_version;
private int ssl_cert_verify;
private String stream_trace_id;
private String ssl_error;
private long c2s_pkt_num;
private long S2c_pkt_num;
private long c2s_byte_num;
private long s2c_byte_num;
private String nas_ip;
private String framed_ip;
private String account;
private int packet_type;
private int has_dup_traffic;
private String stream_error;
public SessionRecordLog() {
}
public long getUid() {
return uid;
}
public void setUid(long uid) {
this.uid = uid;
}
public int getPolicy_id() {
return policy_id;
}
public void setPolicy_id(int policy_id) {
this.policy_id = policy_id;
}
public long getAction() {
return action;
}
public void setAction(long action) {
this.action = action;
}
public int getStart_time() {
return start_time;
}
public void setStart_time(int start_time) {
this.start_time = start_time;
}
public int getEnd_time() {
return end_time;
}
public void setEnd_time(int end_time) {
this.end_time = end_time;
}
public String getSsl_error() {
return ssl_error;
}
public void setSsl_error(String ssl_error) {
this.ssl_error = ssl_error;
}
public String getApp_proto() {
return app_proto;
}
public void setApp_proto(String app_proto) {
this.app_proto = app_proto;
}
public long getRecv_time() {
return recv_time;
}
public void setRecv_time(long recv_time) {
this.recv_time = recv_time;
}
public String getTrans_proto() {
return trans_proto;
}
public void setTrans_proto(String trans_proto) {
this.trans_proto = trans_proto;
}
public int getAddr_type() {
return addr_type;
}
public void setAddr_type(int addr_type) {
this.addr_type = addr_type;
}
public String getServer_ip() {
return server_ip;
}
public void setServer_ip(String server_ip) {
this.server_ip = server_ip;
}
public String getClient_ip() {
return client_ip;
}
public void setClient_ip(String client_ip) {
this.client_ip = client_ip;
}
public int getServer_port() {
return server_port;
}
public void setServer_port(int server_port) {
this.server_port = server_port;
}
public int getClient_port() {
return client_port;
}
public void setClient_port(int client_port) {
this.client_port = client_port;
}
public int getService() {
return service;
}
public void setService(int service) {
this.service = service;
}
public int getEntrance_id() {
return entrance_id;
}
public void setEntrance_id(int entrance_id) {
this.entrance_id = entrance_id;
}
public int getDevice_id() {
return device_id;
}
public void setDevice_id(int device_id) {
this.device_id = device_id;
}
public int getLink_id() {
return Link_id;
}
public void setLink_id(int link_id) {
Link_id = link_id;
}
public String getIsp() {
return isp;
}
public void setIsp(String isp) {
this.isp = isp;
}
public int getEncap_type() {
return encap_type;
}
public void setEncap_type(int encap_type) {
this.encap_type = encap_type;
}
public int getDirection() {
return direction;
}
public void setDirection(int direction) {
this.direction = direction;
}
public int getStream_dir() {
return stream_dir;
}
public void setStream_dir(int stream_dir) {
this.stream_dir = stream_dir;
}
public String getCap_ip() {
return cap_ip;
}
public void setCap_ip(String cap_ip) {
this.cap_ip = cap_ip;
}
public String getAddr_list() {
return addr_list;
}
public void setAddr_list(String addr_list) {
this.addr_list = addr_list;
}
public String getServer_location() {
return server_location;
}
public void setServer_location(String server_location) {
this.server_location = server_location;
}
public String getClient_location() {
return client_location;
}
public void setClient_location(String client_location) {
this.client_location = client_location;
}
public String getClient_asn() {
return client_asn;
}
public void setClient_asn(String client_asn) {
this.client_asn = client_asn;
}
public String getServer_asn() {
return server_asn;
}
public void setServer_asn(String server_asn) {
this.server_asn = server_asn;
}
public String getSubscribe_id() {
return subscribe_id;
}
public void setSubscribe_id(String subscribe_id) {
this.subscribe_id = subscribe_id;
}
public long getCon_duration_ms() {
return con_duration_ms;
}
public void setCon_duration_ms(long con_duration_ms) {
this.con_duration_ms = con_duration_ms;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public String getDomain() {
return domain;
}
public void setDomain(String domain) {
this.domain = domain;
}
public String getCategory() {
return category;
}
public void setCategory(String category) {
this.category = category;
}
public String getReq_line() {
return req_line;
}
public void setReq_line(String req_line) {
this.req_line = req_line;
}
public String getRes_line() {
return res_line;
}
public void setRes_line(String res_line) {
this.res_line = res_line;
}
public String getCookie() {
return cookie;
}
public void setCookie(String cookie) {
this.cookie = cookie;
}
public String getReferer() {
return referer;
}
public void setReferer(String referer) {
this.referer = referer;
}
public String getUser_agent() {
return user_agent;
}
public void setUser_agent(String user_agent) {
this.user_agent = user_agent;
}
public String getContent_len() {
return content_len;
}
public void setContent_len(String content_len) {
this.content_len = content_len;
}
public String getContent_type() {
return content_type;
}
public void setContent_type(String content_type) {
this.content_type = content_type;
}
public String getSet_cookie() {
return set_cookie;
}
public void setSet_cookie(String set_cookie) {
this.set_cookie = set_cookie;
}
public String getReq_header() {
return req_header;
}
public void setReq_header(String req_header) {
this.req_header = req_header;
}
public String getResp_header() {
return resp_header;
}
public void setResp_header(String resp_header) {
this.resp_header = resp_header;
}
public String getReq_body_key() {
return req_body_key;
}
public void setReq_body_key(String req_body_key) {
this.req_body_key = req_body_key;
}
public String getReq_body() {
return req_body;
}
public void setReq_body(String req_body) {
this.req_body = req_body;
}
public String getRes_body_key() {
return res_body_key;
}
public void setRes_body_key(String res_body_key) {
this.res_body_key = res_body_key;
}
public String getResp_body() {
return resp_body;
}
public void setResp_body(String resp_body) {
this.resp_body = resp_body;
}
public String getVersion() {
return version;
}
public void setVersion(String version) {
this.version = version;
}
public String getSni() {
return sni;
}
public void setSni(String sni) {
this.sni = sni;
}
public String getSan() {
return san;
}
public void setSan(String san) {
this.san = san;
}
public String getCn() {
return cn;
}
public void setCn(String cn) {
this.cn = cn;
}
public int getApp_id() {
return app_id;
}
public void setApp_id(int app_id) {
this.app_id = app_id;
}
public int getProtocol_id() {
return protocol_id;
}
public void setProtocol_id(int protocol_id) {
this.protocol_id = protocol_id;
}
public int getIntercept_state() {
return intercept_state;
}
public void setIntercept_state(int intercept_state) {
this.intercept_state = intercept_state;
}
public long getSsl_server_side_latency() {
return ssl_server_side_latency;
}
public void setSsl_server_side_latency(long ssl_server_side_latency) {
this.ssl_server_side_latency = ssl_server_side_latency;
}
public long getSsl_client_side_latency() {
return ssl_client_side_latency;
}
public void setSsl_client_side_latency(long ssl_client_side_latency) {
this.ssl_client_side_latency = ssl_client_side_latency;
}
public String getSsl_server_side_version() {
return ssl_server_side_version;
}
public void setSsl_server_side_version(String ssl_server_side_version) {
this.ssl_server_side_version = ssl_server_side_version;
}
public String getSsl_client_side_version() {
return ssl_client_side_version;
}
public void setSsl_client_side_version(String ssl_client_side_version) {
this.ssl_client_side_version = ssl_client_side_version;
}
public int getSsl_cert_verify() {
return ssl_cert_verify;
}
public void setSsl_cert_verify(int ssl_cert_verify) {
this.ssl_cert_verify = ssl_cert_verify;
}
public String getStream_trace_id() {
return stream_trace_id;
}
public void setStream_trace_id(String stream_trace_id) {
this.stream_trace_id = stream_trace_id;
}
public long getCon_latency_ms() {
return con_latency_ms;
}
public void setCon_latency_ms(long con_latency_ms) {
this.con_latency_ms = con_latency_ms;
}
public int getPinningst() {
return pinningst;
}
public void setPinningst(int pinningst) {
this.pinningst = pinningst;
}
public long getC2s_pkt_num() {
return c2s_pkt_num;
}
public void setC2s_pkt_num(long c2s_pkt_num) {
this.c2s_pkt_num = c2s_pkt_num;
}
public long getS2c_pkt_num() {
return S2c_pkt_num;
}
public void setS2c_pkt_num(long s2c_pkt_num) {
S2c_pkt_num = s2c_pkt_num;
}
public long getC2s_byte_num() {
return c2s_byte_num;
}
public void setC2s_byte_num(long c2s_byte_num) {
this.c2s_byte_num = c2s_byte_num;
}
public long getS2c_byte_num() {
return s2c_byte_num;
}
public void setS2c_byte_num(long s2c_byte_num) {
this.s2c_byte_num = s2c_byte_num;
}
public String getNas_ip() {
return nas_ip;
}
public void setNas_ip(String nas_ip) {
this.nas_ip = nas_ip;
}
public String getFramed_ip() {
return framed_ip;
}
public void setFramed_ip(String framed_ip) {
this.framed_ip = framed_ip;
}
public String getAccount() {
return account;
}
public void setAccount(String account) {
this.account = account;
}
public int getPacket_type() {
return packet_type;
}
public void setPacket_type(int packet_type) {
this.packet_type = packet_type;
}
public int getHas_dup_traffic() {
return has_dup_traffic;
}
public void setHas_dup_traffic(int has_dup_traffic) {
this.has_dup_traffic = has_dup_traffic;
}
public String getStream_error() {
return stream_error;
}
public void setStream_error(String stream_error) {
this.stream_error = stream_error;
}
}

21
src/main/java/cn/ac/iie/bolt/ConnCompletionBolt.java → src/main/java/cn/ac/iie/bolt/collectProtocol/CollectCompletedBolt.java
View File

@@ -1,7 +1,6 @@
package cn.ac.iie.bolt;
package cn.ac.iie.bolt.collectProtocol;
import cn.ac.iie.common.FlowWriteConfig;
import cn.ac.iie.utils.hbase.HbaseUtils;
import cn.ac.iie.utils.system.TupleUtils;
import com.zdjizhi.utils.StringUtil;
import org.apache.log4j.Logger;
@@ -16,16 +15,18 @@ import org.apache.storm.tuple.Values;
import java.util.HashMap;
import java.util.Map;
import static cn.ac.iie.utils.general.TransFormUtils.getJsonMessage;
import static cn.ac.iie.utils.general.TransFormUtils.getCollectProtocolMessage;
import static cn.ac.iie.utils.hbase.HbaseUtils.change;
/**
* 通联关系日志补全
*
* @author qidaijie
*/
public class ConnCompletionBolt extends BaseBasicBolt {
private static final long serialVersionUID = -1059151670138465894L;
private final static Logger logger = Logger.getLogger(ConnCompletionBolt.class);
@SuppressWarnings("all")
public class CollectCompletedBolt extends BaseBasicBolt {
private static final long serialVersionUID = -105915167013846589L;
private final static Logger logger = Logger.getLogger(CollectCompletedBolt.class);
@Override
public void prepare(Map stormConf, TopologyContext context) {
@@ -36,14 +37,13 @@ public class ConnCompletionBolt extends BaseBasicBolt {
public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
try {
if (TupleUtils.isTick(tuple)) {
// HbaseUtils.change();
} else {
change();
}else {
String message = tuple.getString(0);
if (StringUtil.isNotBlank(message)) {
basicOutputCollector.emit(new Values(getJsonMessage(message)));
basicOutputCollector.emit(new Values(getCollectProtocolMessage(message)));
}
}
} catch (Exception e) {
logger.error("接收解析过程出现异常", e);
}
@@ -57,6 +57,7 @@ public class ConnCompletionBolt extends BaseBasicBolt {
return conf;
}
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
outputFieldsDeclarer.declare(new Fields("connLog"));

65
src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java Normal file
View File

@@ -0,0 +1,65 @@
package cn.ac.iie.bolt.proxy;
import cn.ac.iie.common.FlowWriteConfig;
import cn.ac.iie.utils.system.TupleUtils;
import com.zdjizhi.utils.StringUtil;
import org.apache.log4j.Logger;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
import java.util.HashMap;
import java.util.Map;
import static cn.ac.iie.utils.general.TransFormUtils.getProxyMessage;
import static cn.ac.iie.utils.hbase.HbaseUtils.change;
/**
* 通联关系日志补全
*
* @author qidaijie
*/
@SuppressWarnings("all")
public class ProxyCompletionBolt extends BaseBasicBolt {
private static final long serialVersionUID = -1059151670838465894L;
private final static Logger logger = Logger.getLogger(ProxyCompletionBolt.class);
@Override
public void prepare(Map stormConf, TopologyContext context) {
}
@Override
public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
try {
if (TupleUtils.isTick(tuple)) {
change();
}else {
String message = tuple.getString(0);
if (StringUtil.isNotBlank(message)) {
basicOutputCollector.emit(new Values(getProxyMessage(message)));
}
}
} catch (Exception e) {
logger.error("接收解析过程出现异常", e);
}
}
@Override
public Map<String, Object> getComponentConfiguration() {
Map<String, Object> conf = new HashMap<String, Object>(16);
conf.put(org.apache.storm.Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS,
FlowWriteConfig.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
return conf;
}
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
outputFieldsDeclarer.declare(new Fields("connLog"));
}
}

65
src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java Normal file
View File

@@ -0,0 +1,65 @@
package cn.ac.iie.bolt.security;
import cn.ac.iie.common.FlowWriteConfig;
import cn.ac.iie.utils.hbase.HbaseUtils;
import cn.ac.iie.utils.system.TupleUtils;
import com.zdjizhi.utils.StringUtil;
import org.apache.log4j.Logger;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
import java.util.HashMap;
import java.util.Map;
import static cn.ac.iie.utils.general.TransFormUtils.getSecurityMessage;
/**
* 通联关系日志补全
*
* @author qidaijie
*/
@SuppressWarnings("all")
public class SecurityCompletionBolt extends BaseBasicBolt {
private static final long serialVersionUID = -1059151770138464874L;
private final static Logger logger = Logger.getLogger(SecurityCompletionBolt.class);
@Override
public void prepare(Map stormConf, TopologyContext context) {
}
@Override
public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
try {
if (TupleUtils.isTick(tuple)) {
HbaseUtils.change();
}else {
String message = tuple.getString(0);
if (StringUtil.isNotBlank(message)) {
basicOutputCollector.emit(new Values(getSecurityMessage(message)));
}
}
} catch (Exception e) {
logger.error("接收解析过程出现异常", e);
}
}
@Override
public Map<String, Object> getComponentConfiguration() {
Map<String, Object> conf = new HashMap<String, Object>(16);
conf.put(org.apache.storm.Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS,
FlowWriteConfig.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
return conf;
}
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
outputFieldsDeclarer.declare(new Fields("connLog"));
}
}

52
src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java
View File

@@ -1,9 +1,13 @@
package cn.ac.iie.topology;
import cn.ac.iie.bolt.ConnCompletionBolt;
import cn.ac.iie.bolt.NtcLogSendBolt;
import cn.ac.iie.bolt.collectProtocol.CollectCompletedBolt;
import cn.ac.iie.bolt.radius.RadiusCompletionBolt;
import cn.ac.iie.bolt.security.SecurityCompletionBolt;
import cn.ac.iie.bolt.proxy.ProxyCompletionBolt;
import cn.ac.iie.common.FlowWriteConfig;
import cn.ac.iie.spout.CustomizedKafkaSpout;
import org.apache.log4j.Logger;
@@ -58,15 +62,45 @@ public class LogFlowWriteTopology {
private void buildTopology() {
builder = new TopologyBuilder();
builder.setSpout("LogFlowWriteSpout", new CustomizedKafkaSpout(), FlowWriteConfig.SPOUT_PARALLELISM);
if ("RADIUS-LOG".equals(FlowWriteConfig.KAFKA_TOPIC)) {
builder.setBolt("RadiusCompletionBolt", new RadiusCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout");
builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("RadiusCompletionBolt");
} else {
builder.setBolt("ConnCompletionBolt", new ConnCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout");
builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("ConnCompletionBolt");
}
String topic_name = FlowWriteConfig.KAFKA_TOPIC;
switch(topic_name){
case "PROXY-POLICY-LOG":
builder.setBolt("ProxyCompletionBolt", new ProxyCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout");
builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("ProxyCompletionBolt");
break;
case "COLLECT-RADIUS-RECORD-LOG":
builder.setBolt("RadiusCompletionBolt", new RadiusCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout");
builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("RadiusCompletionBolt");
break;
case "COLLECT-CONNECTION-RECORD-LOG":
builder.setBolt("CollectCompletedBolt", new CollectCompletedBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout");
builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("CollectCompletedBolt");
break;
case "SECURITY-POLICY-LOG":
builder.setBolt("SecurityCompletionBolt", new SecurityCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout");
builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("SecurityCompletionBolt");
break;
}
/*if ("PROXY_POLICY_LOG".equals(FlowWriteConfig.KAFKA_TOPIC) || "COLLECT_HTTP_META_LOG".equals(FlowWriteConfig.KAFKA_TOPIC)) {
builder.setBolt("HttpCompletionBolt", new HttpCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout");
builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("HttpCompletionBolt");
} else if ("COLLECT_RADIUS_RECORD_LOG".equals(FlowWriteConfig.KAFKA_TOPIC)) {
builder.setBolt("RadiusCompletionBolt", new ConnCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout");
builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("RadiusCompletionBolt");
} else {
builder.setBolt("ConnCompletionBolt", new ConnCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout");
builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("ConnCompletionBolt");
}*/
// builder.setBolt("SummaryBolt", new SummaryBolt(), 1).localOrShuffleGrouping("NtcLogSendBolt");
}
}
public static void main(String[] args) throws Exception {
LogFlowWriteTopology csst = null;

165
src/main/java/cn/ac/iie/utils/general/TransFormUtils.java
View File

@@ -1,17 +1,13 @@
package cn.ac.iie.utils.general;
import cn.ac.iie.bean.SessionRecordLog;
import cn.ac.iie.bean.*;
import cn.ac.iie.common.FlowWriteConfig;
import cn.ac.iie.utils.hbase.HbaseUtils;
import cn.ac.iie.utils.redis.RedisPollUtils;
import cn.ac.iie.utils.system.SnowflakeId;
import cn.ac.iie.utils.zookeeper.DistributedLock;
import cn.ac.iie.utils.zookeeper.ZookeeperUtils;
import com.alibaba.fastjson.JSONObject;
import com.zdjizhi.utils.IpLookup;
import com.zdjizhi.utils.StringUtil;
import org.apache.log4j.Logger;
import redis.clients.jedis.Jedis;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -23,6 +19,7 @@ import java.util.regex.Pattern;
* @author qidaijie
* @create 2018-08-13 15:11
*/
@SuppressWarnings("all")
public class TransFormUtils {
private static Logger logger = Logger.getLogger(TransFormUtils.class);
private static Pattern WEB_PATTERN = Pattern.compile("[^\\\\.]+(\\.com\\.cn|\\.net\\.cn|\\.org\\.cn|\\.gov\\.cn|\\.com|\\.net|\\.cn|\\.org|\\.cc|\\.me|\\.tel|\\.mobi|\\.asia|\\.biz|\\.info|\\.name|\\.tv|\\.hk|\\.公司|\\.中国|\\.网络)");
@@ -33,51 +30,25 @@ public class TransFormUtils {
.loadAsnDataFileV6(FlowWriteConfig.IP_LIBRARY + "asn_v6.mmdb")
.build();
/**
* 解析日志，并补全
*
* @param message 原始日志
* @return 补全后的日志
*/
public static String getJsonMessage(String message) {
SessionRecordLog sessionRecordLog = JSONObject.parseObject(message, SessionRecordLog.class);
String serverIp = sessionRecordLog.getServer_ip();
String clientIp = sessionRecordLog.getClient_ip();
try {
sessionRecordLog.setUid(SnowflakeId.generateId());
sessionRecordLog.setServer_location(ipLookup.countryLookup(serverIp));
sessionRecordLog.setClient_location(ipLookup.cityLookupDetail(clientIp));
sessionRecordLog.setClient_asn(ipLookup.asnLookup(clientIp, true));
sessionRecordLog.setServer_asn(ipLookup.asnLookup(serverIp, true));
sessionRecordLog.setDomain(getTopDomain(sessionRecordLog.getSni(), sessionRecordLog.getHost()));
sessionRecordLog.setRecv_time(System.currentTimeMillis() / 1000);
// sessionRecordLog.setSubscribe_id(HbaseUtils.getAccount(clientIp));
return JSONObject.toJSONString(sessionRecordLog);
} catch (Exception e) {
logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e);
return "";
}
}
/**
* 解析日志，并补全
*
* 补subscriber_id，不补domain
* @param message radius原始日志
* @return 补全后的日志
*/
public static String getRadiusMessage(String message) {
SessionRecordLog sessionRecordLog = JSONObject.parseObject(message, SessionRecordLog.class);
String serverIp = sessionRecordLog.getServer_ip();
String clientIp = sessionRecordLog.getClient_ip();
RadiusSessionRecordLog radiusSessionRecordLog = JSONObject.parseObject(message, RadiusSessionRecordLog.class);
String serverIp = radiusSessionRecordLog.getCommon_server_ip();
String clientIp = radiusSessionRecordLog.getCommon_client_ip();
try {
sessionRecordLog.setUid(SnowflakeId.generateId());
sessionRecordLog.setServer_location(ipLookup.countryLookup(serverIp));
sessionRecordLog.setClient_location(ipLookup.cityLookupDetail(clientIp));
sessionRecordLog.setClient_asn(ipLookup.asnLookup(clientIp, true));
sessionRecordLog.setServer_asn(ipLookup.asnLookup(serverIp, true));
sessionRecordLog.setRecv_time(System.currentTimeMillis() / 1000);
return JSONObject.toJSONString(sessionRecordLog);
radiusSessionRecordLog.setCommon_log_id(SnowflakeId.generateId());
radiusSessionRecordLog.setCommon_recv_time((System.currentTimeMillis() / 1000));
radiusSessionRecordLog.setCommon_server_location(ipLookup.countryLookup(serverIp));
radiusSessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp));
radiusSessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true));
radiusSessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true));
return JSONObject.toJSONString(radiusSessionRecordLog);
} catch (Exception e) {
logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e);
return "";
@@ -85,6 +56,96 @@ public class TransFormUtils {
}
/**
* 解析日志，并补全
* 补domain,补subscriber_id
* @param message Proxy原始日志
* @return 补全后的日志
*/
public static String getProxyMessage(String message) {
ProxySessionRecordLog proxySessionRecordLog = JSONObject.parseObject(message, ProxySessionRecordLog.class);
String serverIp = proxySessionRecordLog.getCommon_server_ip();
String clientIp = proxySessionRecordLog.getCommon_client_ip();
try {
proxySessionRecordLog.setCommon_log_id(SnowflakeId.generateId());
proxySessionRecordLog.setCommon_recv_time((System.currentTimeMillis() / 1000));
proxySessionRecordLog.setCommon_server_location(ipLookup.countryLookup(serverIp));
proxySessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp));
proxySessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true));
proxySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true));
//TODO 集成AAA数据subscribe_id -数据端补全
proxySessionRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp));
// proxySessionRecordLog.setCommon_subscriber_id("aaaaaaa");
proxySessionRecordLog.setHttp_domain(getTopDomain(null, proxySessionRecordLog.getHttp_host()));
return JSONObject.toJSONString(proxySessionRecordLog);
} catch (Exception e) {
logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e);
return "";
}
}
/**
* 解析日志，并补全
* 补domain,补subscriber_id
* @param message Security原始日志
* @return 补全后的日志
*/
public static String getSecurityMessage(String message) {
SecurityPolicyLog securitySessionRecordLog = JSONObject.parseObject(message, SecurityPolicyLog.class);
String serverIp = securitySessionRecordLog.getCommon_server_ip();
String clientIp = securitySessionRecordLog.getCommon_client_ip();
try {
securitySessionRecordLog.setCommon_log_id(SnowflakeId.generateId());
securitySessionRecordLog.setCommon_recv_time((System.currentTimeMillis() / 1000));
securitySessionRecordLog.setCommon_server_location(ipLookup.countryLookup(serverIp));
securitySessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp));
securitySessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true));
securitySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true));
//TODO 集成AAA数据subscribe_id -数据端补全
securitySessionRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp));
// securitySessionRecordLog.setCommon_subscriber_id("aaaaa");
securitySessionRecordLog.setHttp_domain(getTopDomain(securitySessionRecordLog.getSsl_sni(), securitySessionRecordLog.getHttp_host()));
return JSONObject.toJSONString(securitySessionRecordLog);
} catch (Exception e) {
logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e);
return "";
}
}
/**
* 解析日志，并补全
* 补domain,补subscriber_id
* @param message Security原始日志
* @return 补全后的日志
*/
public static String getCollectProtocolMessage(String message) {
CollectProtocolRecordLog securitySessionRecordLog = JSONObject.parseObject(message, CollectProtocolRecordLog.class);
String serverIp = securitySessionRecordLog.getCommon_server_ip();
String clientIp = securitySessionRecordLog.getCommon_client_ip();
try {
securitySessionRecordLog.setCommon_log_id(SnowflakeId.generateId());
securitySessionRecordLog.setCommon_recv_time((System.currentTimeMillis() / 1000));
securitySessionRecordLog.setCommon_server_location(ipLookup.countryLookup(serverIp));
securitySessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp));
securitySessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true));
securitySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true));
//TODO 集成AAA数据subscribe_id -数据端补全
securitySessionRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp));
// securitySessionRecordLog.setCommon_subscriber_id("aaaaa");
securitySessionRecordLog.setHttp_domain(getTopDomain(securitySessionRecordLog.getSsl_sni(), securitySessionRecordLog.getHttp_host()));
return JSONObject.toJSONString(securitySessionRecordLog);
} catch (Exception e) {
logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e);
return "";
}
}
/**
* 有sni通过sni获取域名，有hots根据host获取域名
*
@@ -102,24 +163,6 @@ public class TransFormUtils {
}
}
/**
* 获取用户名
*
* @param key Sip
* @return SubscribeId
*/
private static String getSubscribeId(String key) {
String sub = "";
try (Jedis jedis = RedisPollUtils.getJedis()) {
if (jedis != null) {
sub = jedis.get(key);
}
} catch (Exception e) {
logger.error("通过Redis获取用户名出现异常", e);
}
return sub;
}
/**
* 根据url截取顶级域名
@@ -140,7 +183,7 @@ public class TransFormUtils {
}
public static void main(String[] args) {
String s = ipLookup.countryLookup("192.168.10.207");
String s = ipLookup.cityLookupDetail("2001:470:19:790::24");
System.out.println(s);
}

29
src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java
View File

@@ -18,26 +18,27 @@ import java.util.Map;
/**
* @author qidaijie
*/
@SuppressWarnings("all")
public class HbaseUtils {
private final static Logger logger = Logger.getLogger(HbaseUtils.class);
private static Map<String, String> subIdMap = new HashMap<>(16);
private static Connection connection;
private static Long time;
// static {
// // 管理Hbase的配置信息
// Configuration configuration = HBaseConfiguration.create();
// // 设置zookeeper节点
// configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS);
// try {
// connection = ConnectionFactory.createConnection(configuration);
// time = System.currentTimeMillis();
// getAll();
// } catch (IOException e) {
// logger.error("获取HBase连接失败");
// e.printStackTrace();
// }
// }
static {
// 管理Hbase的配置信息
Configuration configuration = HBaseConfiguration.create();
// 设置zookeeper节点
configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS);
try {
connection = ConnectionFactory.createConnection(configuration);
time = System.currentTimeMillis();
getAll();
} catch (IOException e) {
logger.error("获取HBase连接失败");
e.printStackTrace();
}
}
/**
* 更新变量

79
src/main/java/cn/ac/iie/utils/redis/RedisClusterUtils.java
View File

@@ -1,79 +0,0 @@
package cn.ac.iie.utils.redis;
import cn.ac.iie.common.FlowWriteConfig;
import org.apache.log4j.Logger;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisCluster;
import redis.clients.jedis.JedisPoolConfig;
import java.io.IOException;
import java.util.LinkedHashSet;
import java.util.Properties;
import java.util.Set;
/**
* 预用于对准IP对应的用户名的 Redis连接池
*
* @author my
* @date 2018-07-04
*/
public final class RedisClusterUtils {
private static final Logger logger = Logger.getLogger(RedisClusterUtils.class);
private static JedisCluster jedisCluster;
private static Properties props = new Properties();
static {
try {
String redisConfigFile = "redis_config.properties";
props.load(RedisClusterUtils.class.getClassLoader().getResourceAsStream(redisConfigFile));
} catch (IOException e) {
props = null;
logger.error("加载Redis配置文件失败！", e);
}
}
/**
* 不允许通过new创建该类的实例
*/
private RedisClusterUtils() {
}
/**
* 初始化Redis连接池
*/
private static JedisCluster getJedisCluster() {
if (jedisCluster == null) {
JedisPoolConfig poolConfig = new JedisPoolConfig();
poolConfig.setMaxTotal(Integer.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_MAXACTIVE)));
poolConfig.setMaxIdle(Integer.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_MAXIDLE)));
poolConfig.setMaxWaitMillis(Long.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_MAXWAIT)));
poolConfig.setTestOnReturn(Boolean.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_TESTONRETURN)));
poolConfig.setTestOnBorrow(Boolean.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_TESTONBORROW)));
Set<HostAndPort> nodes = new LinkedHashSet<HostAndPort>();
for (String port : props.getProperty(FlowWriteConfig.REDIS_PORT).split(FlowWriteConfig.SEGMENTATION)) {
for (String ip : props.getProperty(FlowWriteConfig.REDIS_IP).split(FlowWriteConfig.SEGMENTATION)) {
nodes.add(new HostAndPort(ip, Integer.parseInt(port)));
}
}
jedisCluster = new JedisCluster(nodes, poolConfig);
}
return jedisCluster;
}
/**
* 获取用户名
*
* @param key service_ip
* @return Subscribe_id
*/
public static String get(String key) {
String s = key.split("\\.")[0];
if (!FlowWriteConfig.CHECK_IP_SCOPE.contains(s)) {
jedisCluster = getJedisCluster();
return jedisCluster.get(key);
}
return "";
}
}

111
src/main/java/cn/ac/iie/utils/redis/RedisPollUtils.java
View File

@@ -1,111 +0,0 @@
package cn.ac.iie.utils.redis;
import cn.ac.iie.common.FlowWriteConfig;
import com.zdjizhi.utils.StringUtil;
import org.apache.commons.lang3.RandomUtils;
import org.apache.log4j.Logger;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import java.util.Properties;
/**
* @author qidaijie
*/
public class RedisPollUtils {
private static final Logger logger = Logger.getLogger(RedisPollUtils.class);
private static JedisPool jedisPool = null;
private static Properties props = new Properties();
private RedisPollUtils() {
}
static {
initialPool();
}
/**
* 初始化Redis连接池
*/
private static void initialPool() {
try {
//加载连接池配置文件
props.load(RedisPollUtils.class.getClassLoader().getResourceAsStream("redis_config.properties"));
// 创建jedis池配置实例
JedisPoolConfig poolConfig = new JedisPoolConfig();
poolConfig.setMaxTotal(Integer.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_MAXACTIVE)));
poolConfig.setMaxIdle(Integer.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_MAXIDLE)));
poolConfig.setMaxWaitMillis(Long.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_MAXWAIT)));
poolConfig.setTestOnReturn(Boolean.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_TESTONRETURN)));
poolConfig.setTestOnBorrow(Boolean.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_TESTONBORROW)));
// 根据配置实例化jedis池
jedisPool = new JedisPool(poolConfig, props.getProperty(FlowWriteConfig.REDIS_IP),
Integer.valueOf(props.getProperty(FlowWriteConfig.REDIS_PORT)), 3000, "123456");
} catch (Exception e) {
logger.error("Redis连接池初始化错误", e);
}
}
/**
* 获取Jedis实例
*
* @return Jedis实例
*/
public static Jedis getJedis() {
Jedis jedis = null;
try {
if (jedisPool == null) {
initialPool();
}
jedis = jedisPool.getResource();
} catch (Exception e) {
logger.error("Redis连接池错误,无法获取连接", e);
}
return jedis;
}
// /**
// * @param key redis key
// * @return value
// */
// public static Integer getWorkerId(String key) {
// int workId = 0;
// int maxId = 32;
// try (Jedis jedis = RedisPollUtils.getJedis()) {
// if (jedis != null) {
// String work = jedis.get(key);
// if (StringUtil.isBlank(work)) {
// jedis.set(key, "0");
// } else {
// workId = Integer.parseInt(work);
// }
// if (workId < maxId) {
// jedis.set(key, String.valueOf(workId + 1));
// } else {
// workId = 0;
// jedis.set(key, "1");
// }
// }
// } catch (Exception e) {
// logger.error("通过Redis获取用户名出现异常", e);
// workId = RandomUtils.nextInt(0, 31);
// }
// return workId;
// }
public static String getWorkerId(String key) {
String sub = "";
try (Jedis jedis = RedisPollUtils.getJedis()) {
sub = jedis.get(key);
} catch (Exception e) {
logger.error("通过Redis获取用户名出现异常", e);
}
return sub;
}
}