diff --git a/pom.xml b/pom.xml index 59a11e9..88624d0 100644 --- a/pom.xml +++ b/pom.xml @@ -67,7 +67,7 @@ properties **/*.properties - + **/*.xml false @@ -132,12 +132,6 @@ ${storm.version} - - redis.clients - jedis - 2.8.1 - - junit junit diff --git a/properties/hbase-site.xml b/properties/hbase-site.xml index b81715c..d26f7df 100644 --- a/properties/hbase-site.xml +++ b/properties/hbase-site.xml @@ -31,13 +31,13 @@ hbase.zookeeper.quorum - 192.168.40.202,192.168.40.203,192.168.40.206 + master,slave1,slave2 hbase.master.info.port 60010 - +#开启schema支持 对应hbase的namespace phoenix.schema.isNamespaceMappingEnabled true @@ -46,36 +46,4 @@ phoenix.schema.mapSystemTablesToNamespace true - - hbase.client.keyvalue.maxsize - 99428800 - - - hbase.server.keyvalue.maxsize - 99428800 - - - hbase.regionserver.wal.codec - org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec - - - phoenix.query.timeoutMs - 1800000 - - - hbase.regionserver.lease.period - 1200000 - - - hbase.rpc.timeout - 1200000 - - - hbase.client.scanner.caching - 1000 - - - hbase.client.scanner.timeout.period - 1200000 - diff --git a/properties/hdfs-site.xml b/properties/hdfs-site.xml index 5bf742b..845b195 100644 --- a/properties/hdfs-site.xml +++ b/properties/hdfs-site.xml @@ -31,7 +31,7 @@ dfs.namenode.secondary.http-address - 192.168.40.202:9001 + master:9001 dfs.webhdfs.enabled @@ -60,27 +60,27 @@ dfs.namenode.rpc-address.ns1.nn1 - 192.168.40.202:9000 + master:9000 dfs.namenode.http-address.ns1.nn1 - 192.168.40.202:50070 + master:50070 dfs.namenode.rpc-address.ns1.nn2 - 192.168.40.203:9000 + slave1:9000 dfs.namenode.http-address.ns1.nn2 - 192.168.40.203:50070 + slave1:50070 dfs.namenode.shared.edits.dir - qjournal://192.168.40.203:8485;192.168.40.206:8485;192.168.40.202:8485/ns1 + qjournal://slave1:8485;slave2:8485;master:8485/ns1 diff --git a/properties/service_flow_config.properties b/properties/service_flow_config.properties index 0e64802..e07af38 100644 --- a/properties/service_flow_config.properties +++ b/properties/service_flow_config.properties @@ -1,11 +1,11 @@ #管理kafka地址 -bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092 +bootstrap.servers=192.168.40.152:9092 #zookeeper 地址 -zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181 +zookeeper.servers=192.168.40.152:2181 #hbase zookeeper地址 -hbase.zookeeper.servers=192.168.40.203:2186 +hbase.zookeeper.servers=192.168.40.203:2181,192.168.40.206:2181 #hbase tablename hbase.table.name=subcriber_info @@ -14,29 +14,29 @@ hbase.table.name=subcriber_info auto.offset.reset=latest #kafka broker下的topic名称 -kafka.topic=SESSION-RECORD-LOG +kafka.topic=SECURITY-POLICY-LOG #读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； group.id=session-completion-program #输出topic -results.output.topic=SESSION-RECORD-COMPLETED-LOG +results.output.topic=SECURITY-POLICY-COMPLETED-LOG #storm topology workers -topology.workers=2 +topology.workers=1 #spout并行度 建议与kafka分区数相同 -spout.parallelism=3 +spout.parallelism=1 #处理补全操作的bolt并行度-worker的倍数 -datacenter.bolt.parallelism=3 +datacenter.bolt.parallelism=1 -#写入kafkad的并行度 -kafka.bolt.parallelism=3 +#写入kafka的并行度 +kafka.bolt.parallelism=1 #定位库地址 -ip.library=/home/ceiec/topology/dat/ - +#ip.library=/home/ceiec/topology/dat/ +ip.library=D:/dat/ #kafka批量条数 batch.insert.num=5000 diff --git a/src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java b/src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java new file mode 100644 index 0000000..6131846 --- /dev/null +++ b/src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java @@ -0,0 +1,677 @@ +package cn.ac.iie.bean; + +public class CollectProtocolRecordLog extends PublicSessionRecordLog{ + + +//TODO HTTP协议属性 21 + + private String http_url; + private String http_host; + private String http_domain; + private String http_request_line; + private String http_response_line; + private String http_request_header; + private String http_response_header; + private String http_request_body; + private String http_response_body; + private String http_request_body_key; + private String http_response_body_key; + private int http_proxy_flag; + private int http_sequence; + private String http_snapshot; + private String http_cookie; + private String http_referer; + private String http_user_agent; + private String http_content_length; + private String http_content_type; + private String http_set_cookie; + private String http_version; + +//TODO MAIL协议属性 9 + + private String mail_protocol_type; + private String mail_sender; + private String mail_receiver; + private String mail_subject; + private String mail_content; + private String mail_attachment_name; + private String mail_attachment_content; + private String mail_eml_file; + private String mail_snapshot; + +//TODO 3DNS协议属性 18 + + private int dns_message_id; + private int dns_qr; + private int dns_opcode; + private int dns_aa; + private int dns_tc; + private int dns_rd; + private int dns_ra; + private int dns_rcode; + private int dns_qdcount; + private int dns_ancount; + private int dns_nscount; + private int dns_arcount; + private String dns_qname; + private int dns_qtype; + private int dns_qclass; + private String dns_cname; + private int dns_sub; + private String dns_rr; + +//TODO SSL协议属性 13 + + + private int ssl_pinningst; + private int ssl_intercept_state; + private int ssl_server_side_latency; + private int ssl_client_side_latency; + private int ssl_cert_verify; + private int ssl_con_latency_ms; + private String ssl_version; + private String ssl_sni; + private String ssl_san; + private String ssl_cn; + private String ssl_server_side_version; + private String ssl_client_side_version; + private String ssl_error; + +//TODO FTP协议属性 2 + private String ftp_url; + private String ftp_content; + + +//TODO BGP协议属性 3 + private int bgp_type; + private String bgp_as_num; + private String bgp_route; + +//TODO VOIP协议属性 4 + private String voip_calling_account; + private String voip_called_account; + private String voip_calling_number; + private String voip_called_number; + +//TODO STREAMING_MEDIA协议属性 2 + private String streaming_media_url; + private String streaming_media_protocol; + + + public String getHttp_url() { + return http_url; + } + + public void setHttp_url(String http_url) { + this.http_url = http_url; + } + + public String getHttp_host() { + return http_host; + } + + public void setHttp_host(String http_host) { + this.http_host = http_host; + } + + public String getHttp_domain() { + return http_domain; + } + + public void setHttp_domain(String http_domain) { + this.http_domain = http_domain; + } + + public String getHttp_request_line() { + return http_request_line; + } + + public void setHttp_request_line(String http_request_line) { + this.http_request_line = http_request_line; + } + + public String getHttp_response_line() { + return http_response_line; + } + + public void setHttp_response_line(String http_response_line) { + this.http_response_line = http_response_line; + } + + public String getHttp_request_header() { + return http_request_header; + } + + public void setHttp_request_header(String http_request_header) { + this.http_request_header = http_request_header; + } + + public String getHttp_response_header() { + return http_response_header; + } + + public void setHttp_response_header(String http_response_header) { + this.http_response_header = http_response_header; + } + + public String getHttp_request_body() { + return http_request_body; + } + + public void setHttp_request_body(String http_request_body) { + this.http_request_body = http_request_body; + } + + public String getHttp_response_body() { + return http_response_body; + } + + public void setHttp_response_body(String http_response_body) { + this.http_response_body = http_response_body; + } + + public String getHttp_request_body_key() { + return http_request_body_key; + } + + public void setHttp_request_body_key(String http_request_body_key) { + this.http_request_body_key = http_request_body_key; + } + + public String getHttp_response_body_key() { + return http_response_body_key; + } + + public void setHttp_response_body_key(String http_response_body_key) { + this.http_response_body_key = http_response_body_key; + } + + public int getHttp_proxy_flag() { + return http_proxy_flag; + } + + public void setHttp_proxy_flag(int http_proxy_flag) { + this.http_proxy_flag = http_proxy_flag; + } + + public int getHttp_sequence() { + return http_sequence; + } + + public void setHttp_sequence(int http_sequence) { + this.http_sequence = http_sequence; + } + + public String getHttp_snapshot() { + return http_snapshot; + } + + public void setHttp_snapshot(String http_snapshot) { + this.http_snapshot = http_snapshot; + } + + public String getHttp_cookie() { + return http_cookie; + } + + public void setHttp_cookie(String http_cookie) { + this.http_cookie = http_cookie; + } + + public String getHttp_referer() { + return http_referer; + } + + public void setHttp_referer(String http_referer) { + this.http_referer = http_referer; + } + + public String getHttp_user_agent() { + return http_user_agent; + } + + public void setHttp_user_agent(String http_user_agent) { + this.http_user_agent = http_user_agent; + } + + public String getHttp_content_length() { + return http_content_length; + } + + public void setHttp_content_length(String http_content_length) { + this.http_content_length = http_content_length; + } + + public String getHttp_content_type() { + return http_content_type; + } + + public void setHttp_content_type(String http_content_type) { + this.http_content_type = http_content_type; + } + + public String getHttp_set_cookie() { + return http_set_cookie; + } + + public void setHttp_set_cookie(String http_set_cookie) { + this.http_set_cookie = http_set_cookie; + } + + public String getHttp_version() { + return http_version; + } + + public void setHttp_version(String http_version) { + this.http_version = http_version; + } + + public String getMail_protocol_type() { + return mail_protocol_type; + } + + public void setMail_protocol_type(String mail_protocol_type) { + this.mail_protocol_type = mail_protocol_type; + } + + public String getMail_sender() { + return mail_sender; + } + + public void setMail_sender(String mail_sender) { + this.mail_sender = mail_sender; + } + + public String getMail_receiver() { + return mail_receiver; + } + + public void setMail_receiver(String mail_receiver) { + this.mail_receiver = mail_receiver; + } + + public String getMail_subject() { + return mail_subject; + } + + public void setMail_subject(String mail_subject) { + this.mail_subject = mail_subject; + } + + public String getMail_content() { + return mail_content; + } + + public void setMail_content(String mail_content) { + this.mail_content = mail_content; + } + + public String getMail_attachment_name() { + return mail_attachment_name; + } + + public void setMail_attachment_name(String mail_attachment_name) { + this.mail_attachment_name = mail_attachment_name; + } + + public String getMail_attachment_content() { + return mail_attachment_content; + } + + public void setMail_attachment_content(String mail_attachment_content) { + this.mail_attachment_content = mail_attachment_content; + } + + public String getMail_eml_file() { + return mail_eml_file; + } + + public void setMail_eml_file(String mail_eml_file) { + this.mail_eml_file = mail_eml_file; + } + + public String getMail_snapshot() { + return mail_snapshot; + } + + public void setMail_snapshot(String mail_snapshot) { + this.mail_snapshot = mail_snapshot; + } + + public int getDns_message_id() { + return dns_message_id; + } + + public void setDns_message_id(int dns_message_id) { + this.dns_message_id = dns_message_id; + } + + public int getDns_qr() { + return dns_qr; + } + + public void setDns_qr(int dns_qr) { + this.dns_qr = dns_qr; + } + + public int getDns_opcode() { + return dns_opcode; + } + + public void setDns_opcode(int dns_opcode) { + this.dns_opcode = dns_opcode; + } + + public int getDns_aa() { + return dns_aa; + } + + public void setDns_aa(int dns_aa) { + this.dns_aa = dns_aa; + } + + public int getDns_tc() { + return dns_tc; + } + + public void setDns_tc(int dns_tc) { + this.dns_tc = dns_tc; + } + + public int getDns_rd() { + return dns_rd; + } + + public void setDns_rd(int dns_rd) { + this.dns_rd = dns_rd; + } + + public int getDns_ra() { + return dns_ra; + } + + public void setDns_ra(int dns_ra) { + this.dns_ra = dns_ra; + } + + public int getDns_rcode() { + return dns_rcode; + } + + public void setDns_rcode(int dns_rcode) { + this.dns_rcode = dns_rcode; + } + + public int getDns_qdcount() { + return dns_qdcount; + } + + public void setDns_qdcount(int dns_qdcount) { + this.dns_qdcount = dns_qdcount; + } + + public int getDns_ancount() { + return dns_ancount; + } + + public void setDns_ancount(int dns_ancount) { + this.dns_ancount = dns_ancount; + } + + public int getDns_nscount() { + return dns_nscount; + } + + public void setDns_nscount(int dns_nscount) { + this.dns_nscount = dns_nscount; + } + + public int getDns_arcount() { + return dns_arcount; + } + + public void setDns_arcount(int dns_arcount) { + this.dns_arcount = dns_arcount; + } + + public String getDns_qname() { + return dns_qname; + } + + public void setDns_qname(String dns_qname) { + this.dns_qname = dns_qname; + } + + public int getDns_qtype() { + return dns_qtype; + } + + public void setDns_qtype(int dns_qtype) { + this.dns_qtype = dns_qtype; + } + + public int getDns_qclass() { + return dns_qclass; + } + + public void setDns_qclass(int dns_qclass) { + this.dns_qclass = dns_qclass; + } + + public String getDns_cname() { + return dns_cname; + } + + public void setDns_cname(String dns_cname) { + this.dns_cname = dns_cname; + } + + public int getDns_sub() { + return dns_sub; + } + + public void setDns_sub(int dns_sub) { + this.dns_sub = dns_sub; + } + + public String getDns_rr() { + return dns_rr; + } + + public void setDns_rr(String dns_rr) { + this.dns_rr = dns_rr; + } + + public int getSsl_pinningst() { + return ssl_pinningst; + } + + public void setSsl_pinningst(int ssl_pinningst) { + this.ssl_pinningst = ssl_pinningst; + } + + public int getSsl_intercept_state() { + return ssl_intercept_state; + } + + public void setSsl_intercept_state(int ssl_intercept_state) { + this.ssl_intercept_state = ssl_intercept_state; + } + + public int getSsl_server_side_latency() { + return ssl_server_side_latency; + } + + public void setSsl_server_side_latency(int ssl_server_side_latency) { + this.ssl_server_side_latency = ssl_server_side_latency; + } + + public int getSsl_client_side_latency() { + return ssl_client_side_latency; + } + + public void setSsl_client_side_latency(int ssl_client_side_latency) { + this.ssl_client_side_latency = ssl_client_side_latency; + } + + public int getSsl_cert_verify() { + return ssl_cert_verify; + } + + public void setSsl_cert_verify(int ssl_cert_verify) { + this.ssl_cert_verify = ssl_cert_verify; + } + + public int getSsl_con_latency_ms() { + return ssl_con_latency_ms; + } + + public void setSsl_con_latency_ms(int ssl_con_latency_ms) { + this.ssl_con_latency_ms = ssl_con_latency_ms; + } + + public String getSsl_version() { + return ssl_version; + } + + public void setSsl_version(String ssl_version) { + this.ssl_version = ssl_version; + } + + public String getSsl_sni() { + return ssl_sni; + } + + public void setSsl_sni(String ssl_sni) { + this.ssl_sni = ssl_sni; + } + + public String getSsl_san() { + return ssl_san; + } + + public void setSsl_san(String ssl_san) { + this.ssl_san = ssl_san; + } + + public String getSsl_cn() { + return ssl_cn; + } + + public void setSsl_cn(String ssl_cn) { + this.ssl_cn = ssl_cn; + } + + public String getSsl_server_side_version() { + return ssl_server_side_version; + } + + public void setSsl_server_side_version(String ssl_server_side_version) { + this.ssl_server_side_version = ssl_server_side_version; + } + + public String getSsl_client_side_version() { + return ssl_client_side_version; + } + + public void setSsl_client_side_version(String ssl_client_side_version) { + this.ssl_client_side_version = ssl_client_side_version; + } + + public String getSsl_error() { + return ssl_error; + } + + public void setSsl_error(String ssl_error) { + this.ssl_error = ssl_error; + } + + public String getFtp_url() { + return ftp_url; + } + + public void setFtp_url(String ftp_url) { + this.ftp_url = ftp_url; + } + + public String getFtp_content() { + return ftp_content; + } + + public void setFtp_content(String ftp_content) { + this.ftp_content = ftp_content; + } + + public int getBgp_type() { + return bgp_type; + } + + public void setBgp_type(int bgp_type) { + this.bgp_type = bgp_type; + } + + public String getBgp_as_num() { + return bgp_as_num; + } + + public void setBgp_as_num(String bgp_as_num) { + this.bgp_as_num = bgp_as_num; + } + + public String getBgp_route() { + return bgp_route; + } + + public void setBgp_route(String bgp_route) { + this.bgp_route = bgp_route; + } + + public String getVoip_calling_account() { + return voip_calling_account; + } + + public void setVoip_calling_account(String voip_calling_account) { + this.voip_calling_account = voip_calling_account; + } + + public String getVoip_called_account() { + return voip_called_account; + } + + public void setVoip_called_account(String voip_called_account) { + this.voip_called_account = voip_called_account; + } + + public String getVoip_calling_number() { + return voip_calling_number; + } + + public void setVoip_calling_number(String voip_calling_number) { + this.voip_calling_number = voip_calling_number; + } + + public String getVoip_called_number() { + return voip_called_number; + } + + public void setVoip_called_number(String voip_called_number) { + this.voip_called_number = voip_called_number; + } + + + public String getStreaming_media_url() { + return streaming_media_url; + } + + public void setStreaming_media_url(String streaming_media_url) { + this.streaming_media_url = streaming_media_url; + } + + public String getStreaming_media_protocol() { + return streaming_media_protocol; + } + + public void setStreaming_media_protocol(String streaming_media_protocol) { + this.streaming_media_protocol = streaming_media_protocol; + } +} diff --git a/src/main/java/cn/ac/iie/bean/ProxySessionRecordLog.java b/src/main/java/cn/ac/iie/bean/ProxySessionRecordLog.java new file mode 100644 index 0000000..53b8bd7 --- /dev/null +++ b/src/main/java/cn/ac/iie/bean/ProxySessionRecordLog.java @@ -0,0 +1,198 @@ +package cn.ac.iie.bean; +import com.alibaba.fastjson.annotation.JSONField; +import com.alibaba.fastjson.support.spring.annotation.FastJsonFilter; +public class ProxySessionRecordLog extends PublicSessionRecordLog{ + + +//TODO HTTP协议属性 21 + + private String http_url; + private String http_host; + private String http_domain; + private String http_request_line; + private String http_response_line; + private String http_request_header; + private String http_response_header; + private String http_request_body; + private String http_response_body; + private String http_request_body_key; + private String http_response_body_key; + private int http_proxy_flag; + private int http_sequence; + private String http_snapshot; + private String http_cookie; + private String http_referer; + private String http_user_agent; + private String http_content_length; + private String http_content_type; + private String http_set_cookie; + private String http_version; + + public String getHttp_url() { + return http_url; + } + + public void setHttp_url(String http_url) { + this.http_url = http_url; + } + + public String getHttp_host() { + return http_host; + } + + public void setHttp_host(String http_host) { + this.http_host = http_host; + } + + public String getHttp_domain() { + return http_domain; + } + + public void setHttp_domain(String http_domain) { + this.http_domain = http_domain; + } + + public String getHttp_request_line() { + return http_request_line; + } + + public void setHttp_request_line(String http_request_line) { + this.http_request_line = http_request_line; + } + + public String getHttp_response_line() { + return http_response_line; + } + + public void setHttp_response_line(String http_response_line) { + this.http_response_line = http_response_line; + } + + public String getHttp_request_header() { + return http_request_header; + } + + public void setHttp_request_header(String http_request_header) { + this.http_request_header = http_request_header; + } + + public String getHttp_response_header() { + return http_response_header; + } + + public void setHttp_response_header(String http_response_header) { + this.http_response_header = http_response_header; + } + + public String getHttp_request_body() { + return http_request_body; + } + + public void setHttp_request_body(String http_request_body) { + this.http_request_body = http_request_body; + } + + public String getHttp_response_body() { + return http_response_body; + } + + public void setHttp_response_body(String http_response_body) { + this.http_response_body = http_response_body; + } + + public String getHttp_request_body_key() { + return http_request_body_key; + } + + public void setHttp_request_body_key(String http_request_body_key) { + this.http_request_body_key = http_request_body_key; + } + + public String getHttp_response_body_key() { + return http_response_body_key; + } + + public void setHttp_response_body_key(String http_response_body_key) { + this.http_response_body_key = http_response_body_key; + } + + public int getHttp_proxy_flag() { + return http_proxy_flag; + } + + public void setHttp_proxy_flag(int http_proxy_flag) { + this.http_proxy_flag = http_proxy_flag; + } + + public int getHttp_sequence() { + return http_sequence; + } + + public void setHttp_sequence(int http_sequence) { + this.http_sequence = http_sequence; + } + + public String getHttp_snapshot() { + return http_snapshot; + } + + public void setHttp_snapshot(String http_snapshot) { + this.http_snapshot = http_snapshot; + } + + public String getHttp_cookie() { + return http_cookie; + } + + public void setHttp_cookie(String http_cookie) { + this.http_cookie = http_cookie; + } + + public String getHttp_referer() { + return http_referer; + } + + public void setHttp_referer(String http_referer) { + this.http_referer = http_referer; + } + + public String getHttp_user_agent() { + return http_user_agent; + } + + public void setHttp_user_agent(String http_user_agent) { + this.http_user_agent = http_user_agent; + } + + public String getHttp_content_length() { + return http_content_length; + } + + public void setHttp_content_length(String http_content_length) { + this.http_content_length = http_content_length; + } + + public String getHttp_content_type() { + return http_content_type; + } + + public void setHttp_content_type(String http_content_type) { + this.http_content_type = http_content_type; + } + + public String getHttp_set_cookie() { + return http_set_cookie; + } + + public void setHttp_set_cookie(String http_set_cookie) { + this.http_set_cookie = http_set_cookie; + } + + public String getHttp_version() { + return http_version; + } + + public void setHttp_version(String http_version) { + this.http_version = http_version; + } +} diff --git a/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java b/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java new file mode 100644 index 0000000..7d853d4 --- /dev/null +++ b/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java @@ -0,0 +1,380 @@ +package cn.ac.iie.bean; + +import com.alibaba.fastjson.annotation.JSONField; +import com.alibaba.fastjson.support.spring.annotation.FastJsonFilter; + +public class PublicSessionRecordLog { + +//TODO 基础属性 40 int22 + private int common_service; + private int common_direction; + private long common_recv_time; + private int common_address_type; + private int common_policy_id; + private int common_action; + private int common_client_port; + private int common_entrance_id; + private int common_link_id; + private int common_encapsulation; + private int common_server_port; + private int common_app_id; + private int common_protocol_id; + private int common_c2s_pkt_num; + private int common_s2c_pkt_num; + private long common_c2s_byte_num; + private long common_s2c_byte_num; + private int common_start_time; + private int common_end_time; + private int common_con_duration_ms; + private int common_stream_dir; + private int common_has_dup_traffic; + private long common_log_id; + + private String common_schema_type; + + private String common_l4_protocol; + private String common_user_tags; + private String common_user_region; + private String common_client_ip; + private String common_device_id; + private String common_isp; + private String common_sled_ip; + private String common_client_location; + private String common_client_asn; + private String common_subscriber_id; + private String common_server_ip; + private String common_server_location; + private String common_server_asn; + private String common_app_label; + private String common_address_list; + private String common_stream_error; + private String common_stream_trace_id; + + public int getCommon_service() { + return common_service; + } + + public void setCommon_service(int common_service) { + this.common_service = common_service; + } + + public int getCommon_direction() { + return common_direction; + } + + public void setCommon_direction(int common_direction) { + this.common_direction = common_direction; + } + + public long getCommon_recv_time() { + return common_recv_time; + } + + public void setCommon_recv_time(long common_recv_time) { + this.common_recv_time = common_recv_time; + } + + public int getCommon_address_type() { + return common_address_type; + } + + public void setCommon_address_type(int common_address_type) { + this.common_address_type = common_address_type; + } + + public int getCommon_policy_id() { + return common_policy_id; + } + + public void setCommon_policy_id(int common_policy_id) { + this.common_policy_id = common_policy_id; + } + + public int getCommon_action() { + return common_action; + } + + public void setCommon_action(int common_action) { + this.common_action = common_action; + } + + public int getCommon_client_port() { + return common_client_port; + } + + public void setCommon_client_port(int common_client_port) { + this.common_client_port = common_client_port; + } + + public int getCommon_entrance_id() { + return common_entrance_id; + } + + public void setCommon_entrance_id(int common_entrance_id) { + this.common_entrance_id = common_entrance_id; + } + + public int getCommon_link_id() { + return common_link_id; + } + + public void setCommon_link_id(int common_link_id) { + this.common_link_id = common_link_id; + } + + public int getCommon_encapsulation() { + return common_encapsulation; + } + + public void setCommon_encapsulation(int common_encapsulation) { + this.common_encapsulation = common_encapsulation; + } + + public int getCommon_server_port() { + return common_server_port; + } + + public void setCommon_server_port(int common_server_port) { + this.common_server_port = common_server_port; + } + + public int getCommon_app_id() { + return common_app_id; + } + + public void setCommon_app_id(int common_app_id) { + this.common_app_id = common_app_id; + } + + public int getCommon_protocol_id() { + return common_protocol_id; + } + + public void setCommon_protocol_id(int common_protocol_id) { + this.common_protocol_id = common_protocol_id; + } + + public int getCommon_c2s_pkt_num() { + return common_c2s_pkt_num; + } + + public void setCommon_c2s_pkt_num(int common_c2s_pkt_num) { + this.common_c2s_pkt_num = common_c2s_pkt_num; + } + + public int getCommon_s2c_pkt_num() { + return common_s2c_pkt_num; + } + + public void setCommon_s2c_pkt_num(int common_s2c_pkt_num) { + this.common_s2c_pkt_num = common_s2c_pkt_num; + } + + public long getCommon_c2s_byte_num() { + return common_c2s_byte_num; + } + + public void setCommon_c2s_byte_num(long common_c2s_byte_num) { + this.common_c2s_byte_num = common_c2s_byte_num; + } + + public long getCommon_s2c_byte_num() { + return common_s2c_byte_num; + } + + public void setCommon_s2c_byte_num(long common_s2c_byte_num) { + this.common_s2c_byte_num = common_s2c_byte_num; + } + + public int getCommon_start_time() { + return common_start_time; + } + + public void setCommon_start_time(int common_start_time) { + this.common_start_time = common_start_time; + } + + public int getCommon_end_time() { + return common_end_time; + } + + public void setCommon_end_time(int common_end_time) { + this.common_end_time = common_end_time; + } + + public int getCommon_con_duration_ms() { + return common_con_duration_ms; + } + + public void setCommon_con_duration_ms(int common_con_duration_ms) { + this.common_con_duration_ms = common_con_duration_ms; + } + + public int getCommon_stream_dir() { + return common_stream_dir; + } + + public void setCommon_stream_dir(int common_stream_dir) { + this.common_stream_dir = common_stream_dir; + } + + public int getCommon_has_dup_traffic() { + return common_has_dup_traffic; + } + + public void setCommon_has_dup_traffic(int common_has_dup_traffic) { + this.common_has_dup_traffic = common_has_dup_traffic; + } + + public String getCommon_schema_type() { + return common_schema_type; + } + + public void setCommon_schema_type(String common_schema_type) { + this.common_schema_type = common_schema_type; + } + + public long getCommon_log_id() { + return common_log_id; + } + + public void setCommon_log_id(long common_log_id) { + this.common_log_id = common_log_id; + } + + public String getCommon_l4_protocol() { + return common_l4_protocol; + } + + public void setCommon_l4_protocol(String common_l4_protocol) { + this.common_l4_protocol = common_l4_protocol; + } + + public String getCommon_user_tags() { + return common_user_tags; + } + + public void setCommon_user_tags(String common_user_tags) { + this.common_user_tags = common_user_tags; + } + + public String getCommon_user_region() { + return common_user_region; + } + + public void setCommon_user_region(String common_user_region) { + this.common_user_region = common_user_region; + } + + public String getCommon_client_ip() { + return common_client_ip; + } + + public void setCommon_client_ip(String common_client_ip) { + this.common_client_ip = common_client_ip; + } + + public String getCommon_device_id() { + return common_device_id; + } + + public void setCommon_device_id(String common_device_id) { + this.common_device_id = common_device_id; + } + + public String getCommon_isp() { + return common_isp; + } + + public void setCommon_isp(String common_isp) { + this.common_isp = common_isp; + } + + public String getCommon_sled_ip() { + return common_sled_ip; + } + + public void setCommon_sled_ip(String common_sled_ip) { + this.common_sled_ip = common_sled_ip; + } + + public String getCommon_client_location() { + return common_client_location; + } + + public void setCommon_client_location(String common_client_location) { + this.common_client_location = common_client_location; + } + + public String getCommon_client_asn() { + return common_client_asn; + } + + public void setCommon_client_asn(String common_client_asn) { + this.common_client_asn = common_client_asn; + } + + public String getCommon_subscriber_id() { + return common_subscriber_id; + } + + public void setCommon_subscriber_id(String common_subscriber_id) { + this.common_subscriber_id = common_subscriber_id; + } + + public String getCommon_server_ip() { + return common_server_ip; + } + + public void setCommon_server_ip(String common_server_ip) { + this.common_server_ip = common_server_ip; + } + + public String getCommon_server_location() { + return common_server_location; + } + + public void setCommon_server_location(String common_server_location) { + this.common_server_location = common_server_location; + } + + public String getCommon_server_asn() { + return common_server_asn; + } + + public void setCommon_server_asn(String common_server_asn) { + this.common_server_asn = common_server_asn; + } + + public String getCommon_app_label() { + return common_app_label; + } + + public void setCommon_app_label(String common_app_label) { + this.common_app_label = common_app_label; + } + + public String getCommon_address_list() { + return common_address_list; + } + + public void setCommon_address_list(String common_address_list) { + this.common_address_list = common_address_list; + } + + public String getCommon_stream_error() { + return common_stream_error; + } + + public void setCommon_stream_error(String common_stream_error) { + this.common_stream_error = common_stream_error; + } + + public String getCommon_stream_trace_id() { + return common_stream_trace_id; + } + + public void setCommon_stream_trace_id(String common_stream_trace_id) { + this.common_stream_trace_id = common_stream_trace_id; + } +} diff --git a/src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java b/src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java new file mode 100644 index 0000000..d467afc --- /dev/null +++ b/src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java @@ -0,0 +1,84 @@ +package cn.ac.iie.bean; +import com.alibaba.fastjson.annotation.JSONField; +import com.alibaba.fastjson.support.spring.annotation.FastJsonFilter; +public class RadiusSessionRecordLog extends PublicSessionRecordLog{ + + +//TODO RADIUS协议属性 4 + + private int radius_session_timeout; + private int radius_idle_timeout; + private int radius_acct_status_type; + private int radius_acct_terminate_cause; + + public int getRadius_session_timeout() { + return radius_session_timeout; + } + + public void setRadius_session_timeout(int radius_session_timeout) { + this.radius_session_timeout = radius_session_timeout; + } + + public int getRadius_idle_timeout() { + return radius_idle_timeout; + } + + public void setRadius_idle_timeout(int radius_idle_timeout) { + this.radius_idle_timeout = radius_idle_timeout; + } + + public int getRadius_acct_status_type() { + return radius_acct_status_type; + } + + public void setRadius_acct_status_type(int radius_acct_status_type) { + this.radius_acct_status_type = radius_acct_status_type; + } + + public int getRadius_acct_terminate_cause() { + return radius_acct_terminate_cause; + } + + public void setRadius_acct_terminate_cause(int radius_acct_terminate_cause) { + this.radius_acct_terminate_cause = radius_acct_terminate_cause; + } + + private int radius_packet_type; + + private String radius_nas_ip; + private String radius_framed_ip; + private String radius_account; + + + public int getRadius_packet_type() { + return radius_packet_type; + } + + public void setRadius_packet_type(int radius_packet_type) { + this.radius_packet_type = radius_packet_type; + } + + public String getRadius_nas_ip() { + return radius_nas_ip; + } + + public void setRadius_nas_ip(String radius_nas_ip) { + this.radius_nas_ip = radius_nas_ip; + } + + public String getRadius_framed_ip() { + return radius_framed_ip; + } + + public void setRadius_framed_ip(String radius_framed_ip) { + this.radius_framed_ip = radius_framed_ip; + } + + public String getRadius_account() { + return radius_account; + } + + public void setRadius_account(String radius_account) { + this.radius_account = radius_account; + } +} diff --git a/src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java b/src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java new file mode 100644 index 0000000..775f1c7 --- /dev/null +++ b/src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java @@ -0,0 +1,677 @@ +package cn.ac.iie.bean; + +public class SecurityPolicyLog extends PublicSessionRecordLog{ + + +//TODO HTTP协议属性 21 + + private String http_url; + private String http_host; + private String http_domain; + private String http_request_line; + private String http_response_line; + private String http_request_header; + private String http_response_header; + private String http_request_body; + private String http_response_body; + private String http_request_body_key; + private String http_response_body_key; + private int http_proxy_flag; + private int http_sequence; + private String http_snapshot; + private String http_cookie; + private String http_referer; + private String http_user_agent; + private String http_content_length; + private String http_content_type; + private String http_set_cookie; + private String http_version; + +//TODO MAIL协议属性 9 + + private String mail_protocol_type; + private String mail_sender; + private String mail_receiver; + private String mail_subject; + private String mail_content; + private String mail_attachment_name; + private String mail_attachment_content; + private String mail_eml_file; + private String mail_snapshot; + +//TODO 3DNS协议属性 18 + + private int dns_message_id; + private int dns_qr; + private int dns_opcode; + private int dns_aa; + private int dns_tc; + private int dns_rd; + private int dns_ra; + private int dns_rcode; + private int dns_qdcount; + private int dns_ancount; + private int dns_nscount; + private int dns_arcount; + private String dns_qname; + private int dns_qtype; + private int dns_qclass; + private String dns_cname; + private int dns_sub; + private String dns_rr; + +//TODO SSL协议属性 13 + + + private int ssl_pinningst; + private int ssl_intercept_state; + private int ssl_server_side_latency; + private int ssl_client_side_latency; + private int ssl_cert_verify; + private int ssl_con_latency_ms; + private String ssl_version; + private String ssl_sni; + private String ssl_san; + private String ssl_cn; + private String ssl_server_side_version; + private String ssl_client_side_version; + private String ssl_error; + +//TODO FTP协议属性 2 + private String ftp_url; + private String ftp_content; + + +//TODO BGP协议属性 3 + private int bgp_type; + private String bgp_as_num; + private String bgp_route; + +//TODO VOIP协议属性 4 + private String voip_calling_account; + private String voip_called_account; + private String voip_calling_number; + private String voip_called_number; + +//TODO STREAMING_MEDIA协议属性 2 + private String streaming_media_url; + private String streaming_media_protocol; + + + public String getHttp_url() { + return http_url; + } + + public void setHttp_url(String http_url) { + this.http_url = http_url; + } + + public String getHttp_host() { + return http_host; + } + + public void setHttp_host(String http_host) { + this.http_host = http_host; + } + + public String getHttp_domain() { + return http_domain; + } + + public void setHttp_domain(String http_domain) { + this.http_domain = http_domain; + } + + public String getHttp_request_line() { + return http_request_line; + } + + public void setHttp_request_line(String http_request_line) { + this.http_request_line = http_request_line; + } + + public String getHttp_response_line() { + return http_response_line; + } + + public void setHttp_response_line(String http_response_line) { + this.http_response_line = http_response_line; + } + + public String getHttp_request_header() { + return http_request_header; + } + + public void setHttp_request_header(String http_request_header) { + this.http_request_header = http_request_header; + } + + public String getHttp_response_header() { + return http_response_header; + } + + public void setHttp_response_header(String http_response_header) { + this.http_response_header = http_response_header; + } + + public String getHttp_request_body() { + return http_request_body; + } + + public void setHttp_request_body(String http_request_body) { + this.http_request_body = http_request_body; + } + + public String getHttp_response_body() { + return http_response_body; + } + + public void setHttp_response_body(String http_response_body) { + this.http_response_body = http_response_body; + } + + public String getHttp_request_body_key() { + return http_request_body_key; + } + + public void setHttp_request_body_key(String http_request_body_key) { + this.http_request_body_key = http_request_body_key; + } + + public String getHttp_response_body_key() { + return http_response_body_key; + } + + public void setHttp_response_body_key(String http_response_body_key) { + this.http_response_body_key = http_response_body_key; + } + + public int getHttp_proxy_flag() { + return http_proxy_flag; + } + + public void setHttp_proxy_flag(int http_proxy_flag) { + this.http_proxy_flag = http_proxy_flag; + } + + public int getHttp_sequence() { + return http_sequence; + } + + public void setHttp_sequence(int http_sequence) { + this.http_sequence = http_sequence; + } + + public String getHttp_snapshot() { + return http_snapshot; + } + + public void setHttp_snapshot(String http_snapshot) { + this.http_snapshot = http_snapshot; + } + + public String getHttp_cookie() { + return http_cookie; + } + + public void setHttp_cookie(String http_cookie) { + this.http_cookie = http_cookie; + } + + public String getHttp_referer() { + return http_referer; + } + + public void setHttp_referer(String http_referer) { + this.http_referer = http_referer; + } + + public String getHttp_user_agent() { + return http_user_agent; + } + + public void setHttp_user_agent(String http_user_agent) { + this.http_user_agent = http_user_agent; + } + + public String getHttp_content_length() { + return http_content_length; + } + + public void setHttp_content_length(String http_content_length) { + this.http_content_length = http_content_length; + } + + public String getHttp_content_type() { + return http_content_type; + } + + public void setHttp_content_type(String http_content_type) { + this.http_content_type = http_content_type; + } + + public String getHttp_set_cookie() { + return http_set_cookie; + } + + public void setHttp_set_cookie(String http_set_cookie) { + this.http_set_cookie = http_set_cookie; + } + + public String getHttp_version() { + return http_version; + } + + public void setHttp_version(String http_version) { + this.http_version = http_version; + } + + public String getMail_protocol_type() { + return mail_protocol_type; + } + + public void setMail_protocol_type(String mail_protocol_type) { + this.mail_protocol_type = mail_protocol_type; + } + + public String getMail_sender() { + return mail_sender; + } + + public void setMail_sender(String mail_sender) { + this.mail_sender = mail_sender; + } + + public String getMail_receiver() { + return mail_receiver; + } + + public void setMail_receiver(String mail_receiver) { + this.mail_receiver = mail_receiver; + } + + public String getMail_subject() { + return mail_subject; + } + + public void setMail_subject(String mail_subject) { + this.mail_subject = mail_subject; + } + + public String getMail_content() { + return mail_content; + } + + public void setMail_content(String mail_content) { + this.mail_content = mail_content; + } + + public String getMail_attachment_name() { + return mail_attachment_name; + } + + public void setMail_attachment_name(String mail_attachment_name) { + this.mail_attachment_name = mail_attachment_name; + } + + public String getMail_attachment_content() { + return mail_attachment_content; + } + + public void setMail_attachment_content(String mail_attachment_content) { + this.mail_attachment_content = mail_attachment_content; + } + + public String getMail_eml_file() { + return mail_eml_file; + } + + public void setMail_eml_file(String mail_eml_file) { + this.mail_eml_file = mail_eml_file; + } + + public String getMail_snapshot() { + return mail_snapshot; + } + + public void setMail_snapshot(String mail_snapshot) { + this.mail_snapshot = mail_snapshot; + } + + public int getDns_message_id() { + return dns_message_id; + } + + public void setDns_message_id(int dns_message_id) { + this.dns_message_id = dns_message_id; + } + + public int getDns_qr() { + return dns_qr; + } + + public void setDns_qr(int dns_qr) { + this.dns_qr = dns_qr; + } + + public int getDns_opcode() { + return dns_opcode; + } + + public void setDns_opcode(int dns_opcode) { + this.dns_opcode = dns_opcode; + } + + public int getDns_aa() { + return dns_aa; + } + + public void setDns_aa(int dns_aa) { + this.dns_aa = dns_aa; + } + + public int getDns_tc() { + return dns_tc; + } + + public void setDns_tc(int dns_tc) { + this.dns_tc = dns_tc; + } + + public int getDns_rd() { + return dns_rd; + } + + public void setDns_rd(int dns_rd) { + this.dns_rd = dns_rd; + } + + public int getDns_ra() { + return dns_ra; + } + + public void setDns_ra(int dns_ra) { + this.dns_ra = dns_ra; + } + + public int getDns_rcode() { + return dns_rcode; + } + + public void setDns_rcode(int dns_rcode) { + this.dns_rcode = dns_rcode; + } + + public int getDns_qdcount() { + return dns_qdcount; + } + + public void setDns_qdcount(int dns_qdcount) { + this.dns_qdcount = dns_qdcount; + } + + public int getDns_ancount() { + return dns_ancount; + } + + public void setDns_ancount(int dns_ancount) { + this.dns_ancount = dns_ancount; + } + + public int getDns_nscount() { + return dns_nscount; + } + + public void setDns_nscount(int dns_nscount) { + this.dns_nscount = dns_nscount; + } + + public int getDns_arcount() { + return dns_arcount; + } + + public void setDns_arcount(int dns_arcount) { + this.dns_arcount = dns_arcount; + } + + public String getDns_qname() { + return dns_qname; + } + + public void setDns_qname(String dns_qname) { + this.dns_qname = dns_qname; + } + + public int getDns_qtype() { + return dns_qtype; + } + + public void setDns_qtype(int dns_qtype) { + this.dns_qtype = dns_qtype; + } + + public int getDns_qclass() { + return dns_qclass; + } + + public void setDns_qclass(int dns_qclass) { + this.dns_qclass = dns_qclass; + } + + public String getDns_cname() { + return dns_cname; + } + + public void setDns_cname(String dns_cname) { + this.dns_cname = dns_cname; + } + + public int getDns_sub() { + return dns_sub; + } + + public void setDns_sub(int dns_sub) { + this.dns_sub = dns_sub; + } + + public String getDns_rr() { + return dns_rr; + } + + public void setDns_rr(String dns_rr) { + this.dns_rr = dns_rr; + } + + public int getSsl_pinningst() { + return ssl_pinningst; + } + + public void setSsl_pinningst(int ssl_pinningst) { + this.ssl_pinningst = ssl_pinningst; + } + + public int getSsl_intercept_state() { + return ssl_intercept_state; + } + + public void setSsl_intercept_state(int ssl_intercept_state) { + this.ssl_intercept_state = ssl_intercept_state; + } + + public int getSsl_server_side_latency() { + return ssl_server_side_latency; + } + + public void setSsl_server_side_latency(int ssl_server_side_latency) { + this.ssl_server_side_latency = ssl_server_side_latency; + } + + public int getSsl_client_side_latency() { + return ssl_client_side_latency; + } + + public void setSsl_client_side_latency(int ssl_client_side_latency) { + this.ssl_client_side_latency = ssl_client_side_latency; + } + + public int getSsl_cert_verify() { + return ssl_cert_verify; + } + + public void setSsl_cert_verify(int ssl_cert_verify) { + this.ssl_cert_verify = ssl_cert_verify; + } + + public int getSsl_con_latency_ms() { + return ssl_con_latency_ms; + } + + public void setSsl_con_latency_ms(int ssl_con_latency_ms) { + this.ssl_con_latency_ms = ssl_con_latency_ms; + } + + public String getSsl_version() { + return ssl_version; + } + + public void setSsl_version(String ssl_version) { + this.ssl_version = ssl_version; + } + + public String getSsl_sni() { + return ssl_sni; + } + + public void setSsl_sni(String ssl_sni) { + this.ssl_sni = ssl_sni; + } + + public String getSsl_san() { + return ssl_san; + } + + public void setSsl_san(String ssl_san) { + this.ssl_san = ssl_san; + } + + public String getSsl_cn() { + return ssl_cn; + } + + public void setSsl_cn(String ssl_cn) { + this.ssl_cn = ssl_cn; + } + + public String getSsl_server_side_version() { + return ssl_server_side_version; + } + + public void setSsl_server_side_version(String ssl_server_side_version) { + this.ssl_server_side_version = ssl_server_side_version; + } + + public String getSsl_client_side_version() { + return ssl_client_side_version; + } + + public void setSsl_client_side_version(String ssl_client_side_version) { + this.ssl_client_side_version = ssl_client_side_version; + } + + public String getSsl_error() { + return ssl_error; + } + + public void setSsl_error(String ssl_error) { + this.ssl_error = ssl_error; + } + + public String getFtp_url() { + return ftp_url; + } + + public void setFtp_url(String ftp_url) { + this.ftp_url = ftp_url; + } + + public String getFtp_content() { + return ftp_content; + } + + public void setFtp_content(String ftp_content) { + this.ftp_content = ftp_content; + } + + public int getBgp_type() { + return bgp_type; + } + + public void setBgp_type(int bgp_type) { + this.bgp_type = bgp_type; + } + + public String getBgp_as_num() { + return bgp_as_num; + } + + public void setBgp_as_num(String bgp_as_num) { + this.bgp_as_num = bgp_as_num; + } + + public String getBgp_route() { + return bgp_route; + } + + public void setBgp_route(String bgp_route) { + this.bgp_route = bgp_route; + } + + public String getVoip_calling_account() { + return voip_calling_account; + } + + public void setVoip_calling_account(String voip_calling_account) { + this.voip_calling_account = voip_calling_account; + } + + public String getVoip_called_account() { + return voip_called_account; + } + + public void setVoip_called_account(String voip_called_account) { + this.voip_called_account = voip_called_account; + } + + public String getVoip_calling_number() { + return voip_calling_number; + } + + public void setVoip_calling_number(String voip_calling_number) { + this.voip_calling_number = voip_calling_number; + } + + public String getVoip_called_number() { + return voip_called_number; + } + + public void setVoip_called_number(String voip_called_number) { + this.voip_called_number = voip_called_number; + } + + + public String getStreaming_media_url() { + return streaming_media_url; + } + + public void setStreaming_media_url(String streaming_media_url) { + this.streaming_media_url = streaming_media_url; + } + + public String getStreaming_media_protocol() { + return streaming_media_protocol; + } + + public void setStreaming_media_protocol(String streaming_media_protocol) { + this.streaming_media_protocol = streaming_media_protocol; + } +} diff --git a/src/main/java/cn/ac/iie/bean/SessionRecordLog.java b/src/main/java/cn/ac/iie/bean/SessionRecordLog.java deleted file mode 100644 index baa3f81..0000000 --- a/src/main/java/cn/ac/iie/bean/SessionRecordLog.java +++ /dev/null @@ -1,672 +0,0 @@ -package cn.ac.iie.bean; - -import com.alibaba.fastjson.annotation.JSONField; -import com.alibaba.fastjson.support.spring.annotation.FastJsonFilter; - -/** - * @author qidaijie - */ -public class SessionRecordLog { - private long uid; - private int policy_id; - private long action; - private int start_time; - private int end_time; - private long recv_time; - private String trans_proto; - private String app_proto; - private int addr_type; - private String server_ip; - private String client_ip; - private int server_port; - private int client_port; - private int service; - private int entrance_id; - private int device_id; - private int Link_id; - private String isp; - private int encap_type; - private int direction; - private int stream_dir; - private String cap_ip; - private String addr_list; - private String server_location; - private String client_location; - private String client_asn; - private String server_asn; - private String subscribe_id; - private long con_duration_ms; - private String url; - private String host; - private String domain; - private String category; - private String req_line; - private String res_line; - private String cookie; - private String referer; - private String user_agent; - private String content_len; - private String content_type; - private String set_cookie; - private String req_header; - private String resp_header; - private String req_body_key; - private String req_body; - private String res_body_key; - private String resp_body; - private String version; - private String sni; - private String san; - private String cn; - private int app_id; - private int protocol_id; - private long con_latency_ms; - private int pinningst; - private int intercept_state; - private long ssl_server_side_latency; - private long ssl_client_side_latency; - private String ssl_server_side_version; - private String ssl_client_side_version; - private int ssl_cert_verify; - private String stream_trace_id; - private String ssl_error; - private long c2s_pkt_num; - private long S2c_pkt_num; - private long c2s_byte_num; - private long s2c_byte_num; - private String nas_ip; - private String framed_ip; - private String account; - private int packet_type; - private int has_dup_traffic; - private String stream_error; - - - public SessionRecordLog() { - } - - public long getUid() { - return uid; - } - - public void setUid(long uid) { - this.uid = uid; - } - - public int getPolicy_id() { - return policy_id; - } - - public void setPolicy_id(int policy_id) { - this.policy_id = policy_id; - } - - public long getAction() { - return action; - } - - public void setAction(long action) { - this.action = action; - } - - public int getStart_time() { - return start_time; - } - - public void setStart_time(int start_time) { - this.start_time = start_time; - } - - public int getEnd_time() { - return end_time; - } - - public void setEnd_time(int end_time) { - this.end_time = end_time; - } - - public String getSsl_error() { - return ssl_error; - } - - public void setSsl_error(String ssl_error) { - this.ssl_error = ssl_error; - } - - public String getApp_proto() { - return app_proto; - } - - public void setApp_proto(String app_proto) { - this.app_proto = app_proto; - } - - public long getRecv_time() { - return recv_time; - } - - public void setRecv_time(long recv_time) { - this.recv_time = recv_time; - } - - public String getTrans_proto() { - return trans_proto; - } - - public void setTrans_proto(String trans_proto) { - this.trans_proto = trans_proto; - } - - public int getAddr_type() { - return addr_type; - } - - public void setAddr_type(int addr_type) { - this.addr_type = addr_type; - } - - public String getServer_ip() { - return server_ip; - } - - public void setServer_ip(String server_ip) { - this.server_ip = server_ip; - } - - public String getClient_ip() { - return client_ip; - } - - public void setClient_ip(String client_ip) { - this.client_ip = client_ip; - } - - public int getServer_port() { - return server_port; - } - - public void setServer_port(int server_port) { - this.server_port = server_port; - } - - public int getClient_port() { - return client_port; - } - - public void setClient_port(int client_port) { - this.client_port = client_port; - } - - public int getService() { - return service; - } - - public void setService(int service) { - this.service = service; - } - - public int getEntrance_id() { - return entrance_id; - } - - public void setEntrance_id(int entrance_id) { - this.entrance_id = entrance_id; - } - - public int getDevice_id() { - return device_id; - } - - public void setDevice_id(int device_id) { - this.device_id = device_id; - } - - public int getLink_id() { - return Link_id; - } - - public void setLink_id(int link_id) { - Link_id = link_id; - } - - public String getIsp() { - return isp; - } - - public void setIsp(String isp) { - this.isp = isp; - } - - public int getEncap_type() { - return encap_type; - } - - public void setEncap_type(int encap_type) { - this.encap_type = encap_type; - } - - public int getDirection() { - return direction; - } - - public void setDirection(int direction) { - this.direction = direction; - } - - public int getStream_dir() { - return stream_dir; - } - - public void setStream_dir(int stream_dir) { - this.stream_dir = stream_dir; - } - - public String getCap_ip() { - return cap_ip; - } - - public void setCap_ip(String cap_ip) { - this.cap_ip = cap_ip; - } - - public String getAddr_list() { - return addr_list; - } - - public void setAddr_list(String addr_list) { - this.addr_list = addr_list; - } - - public String getServer_location() { - return server_location; - } - - public void setServer_location(String server_location) { - this.server_location = server_location; - } - - public String getClient_location() { - return client_location; - } - - public void setClient_location(String client_location) { - this.client_location = client_location; - } - - public String getClient_asn() { - return client_asn; - } - - public void setClient_asn(String client_asn) { - this.client_asn = client_asn; - } - - public String getServer_asn() { - return server_asn; - } - - public void setServer_asn(String server_asn) { - this.server_asn = server_asn; - } - - public String getSubscribe_id() { - return subscribe_id; - } - - public void setSubscribe_id(String subscribe_id) { - this.subscribe_id = subscribe_id; - } - - public long getCon_duration_ms() { - return con_duration_ms; - } - - public void setCon_duration_ms(long con_duration_ms) { - this.con_duration_ms = con_duration_ms; - } - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - - public String getHost() { - return host; - } - - public void setHost(String host) { - this.host = host; - } - - public String getDomain() { - return domain; - } - - public void setDomain(String domain) { - this.domain = domain; - } - - public String getCategory() { - return category; - } - - public void setCategory(String category) { - this.category = category; - } - - public String getReq_line() { - return req_line; - } - - public void setReq_line(String req_line) { - this.req_line = req_line; - } - - public String getRes_line() { - return res_line; - } - - public void setRes_line(String res_line) { - this.res_line = res_line; - } - - public String getCookie() { - return cookie; - } - - public void setCookie(String cookie) { - this.cookie = cookie; - } - - public String getReferer() { - return referer; - } - - public void setReferer(String referer) { - this.referer = referer; - } - - public String getUser_agent() { - return user_agent; - } - - public void setUser_agent(String user_agent) { - this.user_agent = user_agent; - } - - public String getContent_len() { - return content_len; - } - - public void setContent_len(String content_len) { - this.content_len = content_len; - } - - public String getContent_type() { - return content_type; - } - - public void setContent_type(String content_type) { - this.content_type = content_type; - } - - public String getSet_cookie() { - return set_cookie; - } - - public void setSet_cookie(String set_cookie) { - this.set_cookie = set_cookie; - } - - public String getReq_header() { - return req_header; - } - - public void setReq_header(String req_header) { - this.req_header = req_header; - } - - public String getResp_header() { - return resp_header; - } - - public void setResp_header(String resp_header) { - this.resp_header = resp_header; - } - - public String getReq_body_key() { - return req_body_key; - } - - public void setReq_body_key(String req_body_key) { - this.req_body_key = req_body_key; - } - - public String getReq_body() { - return req_body; - } - - public void setReq_body(String req_body) { - this.req_body = req_body; - } - - public String getRes_body_key() { - return res_body_key; - } - - public void setRes_body_key(String res_body_key) { - this.res_body_key = res_body_key; - } - - public String getResp_body() { - return resp_body; - } - - public void setResp_body(String resp_body) { - this.resp_body = resp_body; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - - public String getSni() { - return sni; - } - - public void setSni(String sni) { - this.sni = sni; - } - - public String getSan() { - return san; - } - - public void setSan(String san) { - this.san = san; - } - - public String getCn() { - return cn; - } - - public void setCn(String cn) { - this.cn = cn; - } - - public int getApp_id() { - return app_id; - } - - public void setApp_id(int app_id) { - this.app_id = app_id; - } - - public int getProtocol_id() { - return protocol_id; - } - - public void setProtocol_id(int protocol_id) { - this.protocol_id = protocol_id; - } - - public int getIntercept_state() { - return intercept_state; - } - - public void setIntercept_state(int intercept_state) { - this.intercept_state = intercept_state; - } - - public long getSsl_server_side_latency() { - return ssl_server_side_latency; - } - - public void setSsl_server_side_latency(long ssl_server_side_latency) { - this.ssl_server_side_latency = ssl_server_side_latency; - } - - public long getSsl_client_side_latency() { - return ssl_client_side_latency; - } - - public void setSsl_client_side_latency(long ssl_client_side_latency) { - this.ssl_client_side_latency = ssl_client_side_latency; - } - - public String getSsl_server_side_version() { - return ssl_server_side_version; - } - - public void setSsl_server_side_version(String ssl_server_side_version) { - this.ssl_server_side_version = ssl_server_side_version; - } - - public String getSsl_client_side_version() { - return ssl_client_side_version; - } - - public void setSsl_client_side_version(String ssl_client_side_version) { - this.ssl_client_side_version = ssl_client_side_version; - } - - public int getSsl_cert_verify() { - return ssl_cert_verify; - } - - public void setSsl_cert_verify(int ssl_cert_verify) { - this.ssl_cert_verify = ssl_cert_verify; - } - - public String getStream_trace_id() { - return stream_trace_id; - } - - public void setStream_trace_id(String stream_trace_id) { - this.stream_trace_id = stream_trace_id; - } - - public long getCon_latency_ms() { - return con_latency_ms; - } - - public void setCon_latency_ms(long con_latency_ms) { - this.con_latency_ms = con_latency_ms; - } - - public int getPinningst() { - return pinningst; - } - - public void setPinningst(int pinningst) { - this.pinningst = pinningst; - } - - - public long getC2s_pkt_num() { - return c2s_pkt_num; - } - - public void setC2s_pkt_num(long c2s_pkt_num) { - this.c2s_pkt_num = c2s_pkt_num; - } - - public long getS2c_pkt_num() { - return S2c_pkt_num; - } - - public void setS2c_pkt_num(long s2c_pkt_num) { - S2c_pkt_num = s2c_pkt_num; - } - - public long getC2s_byte_num() { - return c2s_byte_num; - } - - public void setC2s_byte_num(long c2s_byte_num) { - this.c2s_byte_num = c2s_byte_num; - } - - public long getS2c_byte_num() { - return s2c_byte_num; - } - - public void setS2c_byte_num(long s2c_byte_num) { - this.s2c_byte_num = s2c_byte_num; - } - - public String getNas_ip() { - return nas_ip; - } - - public void setNas_ip(String nas_ip) { - this.nas_ip = nas_ip; - } - - public String getFramed_ip() { - return framed_ip; - } - - public void setFramed_ip(String framed_ip) { - this.framed_ip = framed_ip; - } - - public String getAccount() { - return account; - } - - public void setAccount(String account) { - this.account = account; - } - - public int getPacket_type() { - return packet_type; - } - - public void setPacket_type(int packet_type) { - this.packet_type = packet_type; - } - - public int getHas_dup_traffic() { - return has_dup_traffic; - } - - public void setHas_dup_traffic(int has_dup_traffic) { - this.has_dup_traffic = has_dup_traffic; - } - - public String getStream_error() { - return stream_error; - } - - public void setStream_error(String stream_error) { - this.stream_error = stream_error; - } -} diff --git a/src/main/java/cn/ac/iie/bolt/ConnCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/collectProtocol/CollectCompletedBolt.java similarity index 76% rename from src/main/java/cn/ac/iie/bolt/ConnCompletionBolt.java rename to src/main/java/cn/ac/iie/bolt/collectProtocol/CollectCompletedBolt.java index 109f856..1cd40fa 100644 --- a/src/main/java/cn/ac/iie/bolt/ConnCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/collectProtocol/CollectCompletedBolt.java @@ -1,7 +1,6 @@ -package cn.ac.iie.bolt; +package cn.ac.iie.bolt.collectProtocol; import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.hbase.HbaseUtils; import cn.ac.iie.utils.system.TupleUtils; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; @@ -16,16 +15,18 @@ import org.apache.storm.tuple.Values; import java.util.HashMap; import java.util.Map; -import static cn.ac.iie.utils.general.TransFormUtils.getJsonMessage; +import static cn.ac.iie.utils.general.TransFormUtils.getCollectProtocolMessage; +import static cn.ac.iie.utils.hbase.HbaseUtils.change; /** * 通联关系日志补全 * * @author qidaijie */ -public class ConnCompletionBolt extends BaseBasicBolt { - private static final long serialVersionUID = -1059151670138465894L; - private final static Logger logger = Logger.getLogger(ConnCompletionBolt.class); +@SuppressWarnings("all") +public class CollectCompletedBolt extends BaseBasicBolt { + private static final long serialVersionUID = -105915167013846589L; + private final static Logger logger = Logger.getLogger(CollectCompletedBolt.class); @Override public void prepare(Map stormConf, TopologyContext context) { @@ -36,14 +37,13 @@ public class ConnCompletionBolt extends BaseBasicBolt { public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { try { if (TupleUtils.isTick(tuple)) { -// HbaseUtils.change(); - } else { + change(); + }else { String message = tuple.getString(0); if (StringUtil.isNotBlank(message)) { - basicOutputCollector.emit(new Values(getJsonMessage(message))); + basicOutputCollector.emit(new Values(getCollectProtocolMessage(message))); } } - } catch (Exception e) { logger.error("接收解析过程出现异常", e); } @@ -57,6 +57,7 @@ public class ConnCompletionBolt extends BaseBasicBolt { return conf; } + @Override public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { outputFieldsDeclarer.declare(new Fields("connLog")); diff --git a/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java new file mode 100644 index 0000000..123095a --- /dev/null +++ b/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java @@ -0,0 +1,65 @@ +package cn.ac.iie.bolt.proxy; + +import cn.ac.iie.common.FlowWriteConfig; +import cn.ac.iie.utils.system.TupleUtils; +import com.zdjizhi.utils.StringUtil; +import org.apache.log4j.Logger; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.BasicOutputCollector; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseBasicBolt; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; + +import java.util.HashMap; +import java.util.Map; + +import static cn.ac.iie.utils.general.TransFormUtils.getProxyMessage; +import static cn.ac.iie.utils.hbase.HbaseUtils.change; + +/** + * 通联关系日志补全 + * + * @author qidaijie + */ +@SuppressWarnings("all") +public class ProxyCompletionBolt extends BaseBasicBolt { + private static final long serialVersionUID = -1059151670838465894L; + private final static Logger logger = Logger.getLogger(ProxyCompletionBolt.class); + + @Override + public void prepare(Map stormConf, TopologyContext context) { + + } + + @Override + public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { + try { + if (TupleUtils.isTick(tuple)) { + change(); + }else { + String message = tuple.getString(0); + if (StringUtil.isNotBlank(message)) { + basicOutputCollector.emit(new Values(getProxyMessage(message))); + } + } + } catch (Exception e) { + logger.error("接收解析过程出现异常", e); + } + } + + @Override + public Map getComponentConfiguration() { + Map conf = new HashMap(16); + conf.put(org.apache.storm.Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, + FlowWriteConfig.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + return conf; + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { + outputFieldsDeclarer.declare(new Fields("connLog")); + } + +} diff --git a/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java new file mode 100644 index 0000000..ebb33b7 --- /dev/null +++ b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java @@ -0,0 +1,65 @@ +package cn.ac.iie.bolt.security; + +import cn.ac.iie.common.FlowWriteConfig; +import cn.ac.iie.utils.hbase.HbaseUtils; +import cn.ac.iie.utils.system.TupleUtils; +import com.zdjizhi.utils.StringUtil; +import org.apache.log4j.Logger; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.BasicOutputCollector; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseBasicBolt; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; + +import java.util.HashMap; +import java.util.Map; + +import static cn.ac.iie.utils.general.TransFormUtils.getSecurityMessage; + +/** + * 通联关系日志补全 + * + * @author qidaijie + */ +@SuppressWarnings("all") +public class SecurityCompletionBolt extends BaseBasicBolt { + private static final long serialVersionUID = -1059151770138464874L; + private final static Logger logger = Logger.getLogger(SecurityCompletionBolt.class); + + @Override + public void prepare(Map stormConf, TopologyContext context) { + + } + + @Override + public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { + try { + if (TupleUtils.isTick(tuple)) { + HbaseUtils.change(); + }else { + String message = tuple.getString(0); + if (StringUtil.isNotBlank(message)) { + basicOutputCollector.emit(new Values(getSecurityMessage(message))); + } + } + } catch (Exception e) { + logger.error("接收解析过程出现异常", e); + } + } + + @Override + public Map getComponentConfiguration() { + Map conf = new HashMap(16); + conf.put(org.apache.storm.Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, + FlowWriteConfig.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + return conf; + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { + outputFieldsDeclarer.declare(new Fields("connLog")); + } + +} diff --git a/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java b/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java index 6b677d8..80ecdb8 100644 --- a/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java +++ b/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java @@ -1,9 +1,13 @@ package cn.ac.iie.topology; -import cn.ac.iie.bolt.ConnCompletionBolt; import cn.ac.iie.bolt.NtcLogSendBolt; +import cn.ac.iie.bolt.collectProtocol.CollectCompletedBolt; import cn.ac.iie.bolt.radius.RadiusCompletionBolt; +import cn.ac.iie.bolt.security.SecurityCompletionBolt; + +import cn.ac.iie.bolt.proxy.ProxyCompletionBolt; + import cn.ac.iie.common.FlowWriteConfig; import cn.ac.iie.spout.CustomizedKafkaSpout; import org.apache.log4j.Logger; @@ -58,15 +62,45 @@ public class LogFlowWriteTopology { private void buildTopology() { builder = new TopologyBuilder(); builder.setSpout("LogFlowWriteSpout", new CustomizedKafkaSpout(), FlowWriteConfig.SPOUT_PARALLELISM); - if ("RADIUS-LOG".equals(FlowWriteConfig.KAFKA_TOPIC)) { - builder.setBolt("RadiusCompletionBolt", new RadiusCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); - builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("RadiusCompletionBolt"); - } else { - builder.setBolt("ConnCompletionBolt", new ConnCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); - builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("ConnCompletionBolt"); - } + + String topic_name = FlowWriteConfig.KAFKA_TOPIC; + + switch(topic_name){ + case "PROXY-POLICY-LOG": + builder.setBolt("ProxyCompletionBolt", new ProxyCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); + builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("ProxyCompletionBolt"); + break; + + case "COLLECT-RADIUS-RECORD-LOG": + builder.setBolt("RadiusCompletionBolt", new RadiusCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); + builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("RadiusCompletionBolt"); + break; + + case "COLLECT-CONNECTION-RECORD-LOG": + builder.setBolt("CollectCompletedBolt", new CollectCompletedBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); + builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("CollectCompletedBolt"); + break; + + case "SECURITY-POLICY-LOG": + builder.setBolt("SecurityCompletionBolt", new SecurityCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); + builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("SecurityCompletionBolt"); + break; + + + } + + /*if ("PROXY_POLICY_LOG".equals(FlowWriteConfig.KAFKA_TOPIC) || "COLLECT_HTTP_META_LOG".equals(FlowWriteConfig.KAFKA_TOPIC)) { + builder.setBolt("HttpCompletionBolt", new HttpCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); + builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("HttpCompletionBolt"); + } else if ("COLLECT_RADIUS_RECORD_LOG".equals(FlowWriteConfig.KAFKA_TOPIC)) { + builder.setBolt("RadiusCompletionBolt", new ConnCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); + builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("RadiusCompletionBolt"); + } else { + builder.setBolt("ConnCompletionBolt", new ConnCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); + builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("ConnCompletionBolt"); + }*/ // builder.setBolt("SummaryBolt", new SummaryBolt(), 1).localOrShuffleGrouping("NtcLogSendBolt"); - } + } public static void main(String[] args) throws Exception { LogFlowWriteTopology csst = null; diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java index 3790c35..c515689 100644 --- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java @@ -1,17 +1,13 @@ package cn.ac.iie.utils.general; -import cn.ac.iie.bean.SessionRecordLog; +import cn.ac.iie.bean.*; import cn.ac.iie.common.FlowWriteConfig; import cn.ac.iie.utils.hbase.HbaseUtils; -import cn.ac.iie.utils.redis.RedisPollUtils; import cn.ac.iie.utils.system.SnowflakeId; -import cn.ac.iie.utils.zookeeper.DistributedLock; -import cn.ac.iie.utils.zookeeper.ZookeeperUtils; import com.alibaba.fastjson.JSONObject; import com.zdjizhi.utils.IpLookup; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; -import redis.clients.jedis.Jedis; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -23,6 +19,7 @@ import java.util.regex.Pattern; * @author qidaijie * @create 2018-08-13 15:11 */ +@SuppressWarnings("all") public class TransFormUtils { private static Logger logger = Logger.getLogger(TransFormUtils.class); private static Pattern WEB_PATTERN = Pattern.compile("[^\\\\.]+(\\.com\\.cn|\\.net\\.cn|\\.org\\.cn|\\.gov\\.cn|\\.com|\\.net|\\.cn|\\.org|\\.cc|\\.me|\\.tel|\\.mobi|\\.asia|\\.biz|\\.info|\\.name|\\.tv|\\.hk|\\.公司|\\.中国|\\.网络)"); @@ -33,51 +30,25 @@ public class TransFormUtils { .loadAsnDataFileV6(FlowWriteConfig.IP_LIBRARY + "asn_v6.mmdb") .build(); - /** - * 解析日志，并补全 - * - * @param message 原始日志 - * @return 补全后的日志 - */ - public static String getJsonMessage(String message) { - SessionRecordLog sessionRecordLog = JSONObject.parseObject(message, SessionRecordLog.class); - String serverIp = sessionRecordLog.getServer_ip(); - String clientIp = sessionRecordLog.getClient_ip(); - try { - sessionRecordLog.setUid(SnowflakeId.generateId()); - sessionRecordLog.setServer_location(ipLookup.countryLookup(serverIp)); - sessionRecordLog.setClient_location(ipLookup.cityLookupDetail(clientIp)); - sessionRecordLog.setClient_asn(ipLookup.asnLookup(clientIp, true)); - sessionRecordLog.setServer_asn(ipLookup.asnLookup(serverIp, true)); - sessionRecordLog.setDomain(getTopDomain(sessionRecordLog.getSni(), sessionRecordLog.getHost())); - sessionRecordLog.setRecv_time(System.currentTimeMillis() / 1000); -// sessionRecordLog.setSubscribe_id(HbaseUtils.getAccount(clientIp)); - return JSONObject.toJSONString(sessionRecordLog); - } catch (Exception e) { - logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e); - return ""; - } - } - /** * 解析日志，并补全 - * + * 补subscriber_id，不补domain * @param message radius原始日志 * @return 补全后的日志 */ public static String getRadiusMessage(String message) { - SessionRecordLog sessionRecordLog = JSONObject.parseObject(message, SessionRecordLog.class); - String serverIp = sessionRecordLog.getServer_ip(); - String clientIp = sessionRecordLog.getClient_ip(); + RadiusSessionRecordLog radiusSessionRecordLog = JSONObject.parseObject(message, RadiusSessionRecordLog.class); + String serverIp = radiusSessionRecordLog.getCommon_server_ip(); + String clientIp = radiusSessionRecordLog.getCommon_client_ip(); try { - sessionRecordLog.setUid(SnowflakeId.generateId()); - sessionRecordLog.setServer_location(ipLookup.countryLookup(serverIp)); - sessionRecordLog.setClient_location(ipLookup.cityLookupDetail(clientIp)); - sessionRecordLog.setClient_asn(ipLookup.asnLookup(clientIp, true)); - sessionRecordLog.setServer_asn(ipLookup.asnLookup(serverIp, true)); - sessionRecordLog.setRecv_time(System.currentTimeMillis() / 1000); - return JSONObject.toJSONString(sessionRecordLog); + radiusSessionRecordLog.setCommon_log_id(SnowflakeId.generateId()); + radiusSessionRecordLog.setCommon_recv_time((System.currentTimeMillis() / 1000)); + radiusSessionRecordLog.setCommon_server_location(ipLookup.countryLookup(serverIp)); + radiusSessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp)); + radiusSessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); + radiusSessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); + return JSONObject.toJSONString(radiusSessionRecordLog); } catch (Exception e) { logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e); return ""; @@ -85,6 +56,96 @@ public class TransFormUtils { } + + + + + /** + * 解析日志，并补全 + * 补domain,补subscriber_id + * @param message Proxy原始日志 + * @return 补全后的日志 + */ + public static String getProxyMessage(String message) { + ProxySessionRecordLog proxySessionRecordLog = JSONObject.parseObject(message, ProxySessionRecordLog.class); + String serverIp = proxySessionRecordLog.getCommon_server_ip(); + String clientIp = proxySessionRecordLog.getCommon_client_ip(); + try { + proxySessionRecordLog.setCommon_log_id(SnowflakeId.generateId()); + proxySessionRecordLog.setCommon_recv_time((System.currentTimeMillis() / 1000)); + proxySessionRecordLog.setCommon_server_location(ipLookup.countryLookup(serverIp)); + proxySessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp)); + proxySessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); + proxySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); + //TODO 集成AAA数据subscribe_id -数据端补全 + proxySessionRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp)); +// proxySessionRecordLog.setCommon_subscriber_id("aaaaaaa"); + proxySessionRecordLog.setHttp_domain(getTopDomain(null, proxySessionRecordLog.getHttp_host())); + return JSONObject.toJSONString(proxySessionRecordLog); + } catch (Exception e) { + logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e); + return ""; + } + } + + + + /** + * 解析日志，并补全 + * 补domain,补subscriber_id + * @param message Security原始日志 + * @return 补全后的日志 + */ + public static String getSecurityMessage(String message) { + SecurityPolicyLog securitySessionRecordLog = JSONObject.parseObject(message, SecurityPolicyLog.class); + String serverIp = securitySessionRecordLog.getCommon_server_ip(); + String clientIp = securitySessionRecordLog.getCommon_client_ip(); + try { + securitySessionRecordLog.setCommon_log_id(SnowflakeId.generateId()); + securitySessionRecordLog.setCommon_recv_time((System.currentTimeMillis() / 1000)); + securitySessionRecordLog.setCommon_server_location(ipLookup.countryLookup(serverIp)); + securitySessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp)); + securitySessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); + securitySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); + //TODO 集成AAA数据subscribe_id -数据端补全 + securitySessionRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp)); +// securitySessionRecordLog.setCommon_subscriber_id("aaaaa"); + securitySessionRecordLog.setHttp_domain(getTopDomain(securitySessionRecordLog.getSsl_sni(), securitySessionRecordLog.getHttp_host())); + return JSONObject.toJSONString(securitySessionRecordLog); + } catch (Exception e) { + logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e); + return ""; + } + } + + /** + * 解析日志，并补全 + * 补domain,补subscriber_id + * @param message Security原始日志 + * @return 补全后的日志 + */ + public static String getCollectProtocolMessage(String message) { + CollectProtocolRecordLog securitySessionRecordLog = JSONObject.parseObject(message, CollectProtocolRecordLog.class); + String serverIp = securitySessionRecordLog.getCommon_server_ip(); + String clientIp = securitySessionRecordLog.getCommon_client_ip(); + try { + securitySessionRecordLog.setCommon_log_id(SnowflakeId.generateId()); + securitySessionRecordLog.setCommon_recv_time((System.currentTimeMillis() / 1000)); + securitySessionRecordLog.setCommon_server_location(ipLookup.countryLookup(serverIp)); + securitySessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp)); + securitySessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); + securitySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); + //TODO 集成AAA数据subscribe_id -数据端补全 + securitySessionRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp)); +// securitySessionRecordLog.setCommon_subscriber_id("aaaaa"); + securitySessionRecordLog.setHttp_domain(getTopDomain(securitySessionRecordLog.getSsl_sni(), securitySessionRecordLog.getHttp_host())); + return JSONObject.toJSONString(securitySessionRecordLog); + } catch (Exception e) { + logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e); + return ""; + } + } + /** * 有sni通过sni获取域名，有hots根据host获取域名 * @@ -102,24 +163,6 @@ public class TransFormUtils { } } - /** - * 获取用户名 - * - * @param key Sip - * @return SubscribeId - */ - private static String getSubscribeId(String key) { - String sub = ""; - try (Jedis jedis = RedisPollUtils.getJedis()) { - if (jedis != null) { - sub = jedis.get(key); - } - } catch (Exception e) { - logger.error("通过Redis获取用户名出现异常", e); - } - return sub; - } - /** * 根据url截取顶级域名 @@ -140,7 +183,7 @@ public class TransFormUtils { } public static void main(String[] args) { - String s = ipLookup.countryLookup("192.168.10.207"); + String s = ipLookup.cityLookupDetail("2001:470:19:790::24"); System.out.println(s); } diff --git a/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java b/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java index c7b05e3..b8fecda 100644 --- a/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java +++ b/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java @@ -18,26 +18,27 @@ import java.util.Map; /** * @author qidaijie */ +@SuppressWarnings("all") public class HbaseUtils { private final static Logger logger = Logger.getLogger(HbaseUtils.class); private static Map subIdMap = new HashMap<>(16); private static Connection connection; private static Long time; -// static { -// // 管理Hbase的配置信息 -// Configuration configuration = HBaseConfiguration.create(); -// // 设置zookeeper节点 -// configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS); -// try { -// connection = ConnectionFactory.createConnection(configuration); -// time = System.currentTimeMillis(); -// getAll(); -// } catch (IOException e) { -// logger.error("获取HBase连接失败"); -// e.printStackTrace(); -// } -// } + static { + // 管理Hbase的配置信息 + Configuration configuration = HBaseConfiguration.create(); + // 设置zookeeper节点 + configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS); + try { + connection = ConnectionFactory.createConnection(configuration); + time = System.currentTimeMillis(); + getAll(); + } catch (IOException e) { + logger.error("获取HBase连接失败"); + e.printStackTrace(); + } + } /** * 更新变量 diff --git a/src/main/java/cn/ac/iie/utils/redis/RedisClusterUtils.java b/src/main/java/cn/ac/iie/utils/redis/RedisClusterUtils.java deleted file mode 100644 index e7f67d9..0000000 --- a/src/main/java/cn/ac/iie/utils/redis/RedisClusterUtils.java +++ /dev/null @@ -1,79 +0,0 @@ -package cn.ac.iie.utils.redis; - -import cn.ac.iie.common.FlowWriteConfig; -import org.apache.log4j.Logger; -import redis.clients.jedis.HostAndPort; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.JedisPoolConfig; - -import java.io.IOException; -import java.util.LinkedHashSet; -import java.util.Properties; -import java.util.Set; - -/** - * 预用于对准IP对应的用户名的 Redis连接池 - * - * @author my - * @date 2018-07-04 - */ -public final class RedisClusterUtils { - private static final Logger logger = Logger.getLogger(RedisClusterUtils.class); - private static JedisCluster jedisCluster; - private static Properties props = new Properties(); - - static { - try { - String redisConfigFile = "redis_config.properties"; - props.load(RedisClusterUtils.class.getClassLoader().getResourceAsStream(redisConfigFile)); - } catch (IOException e) { - props = null; - logger.error("加载Redis配置文件失败！", e); - } - } - - /** - * 不允许通过new创建该类的实例 - */ - private RedisClusterUtils() { - } - - /** - * 初始化Redis连接池 - */ - private static JedisCluster getJedisCluster() { - if (jedisCluster == null) { - JedisPoolConfig poolConfig = new JedisPoolConfig(); - poolConfig.setMaxTotal(Integer.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_MAXACTIVE))); - poolConfig.setMaxIdle(Integer.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_MAXIDLE))); - poolConfig.setMaxWaitMillis(Long.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_MAXWAIT))); - poolConfig.setTestOnReturn(Boolean.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_TESTONRETURN))); - poolConfig.setTestOnBorrow(Boolean.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_TESTONBORROW))); - Set nodes = new LinkedHashSet(); - for (String port : props.getProperty(FlowWriteConfig.REDIS_PORT).split(FlowWriteConfig.SEGMENTATION)) { - for (String ip : props.getProperty(FlowWriteConfig.REDIS_IP).split(FlowWriteConfig.SEGMENTATION)) { - nodes.add(new HostAndPort(ip, Integer.parseInt(port))); - } - } - jedisCluster = new JedisCluster(nodes, poolConfig); - } - return jedisCluster; - } - - /** - * 获取用户名 - * - * @param key service_ip - * @return Subscribe_id - */ - public static String get(String key) { - String s = key.split("\\.")[0]; - if (!FlowWriteConfig.CHECK_IP_SCOPE.contains(s)) { - jedisCluster = getJedisCluster(); - return jedisCluster.get(key); - } - return ""; - } - - -} diff --git a/src/main/java/cn/ac/iie/utils/redis/RedisPollUtils.java b/src/main/java/cn/ac/iie/utils/redis/RedisPollUtils.java deleted file mode 100644 index 0394378..0000000 --- a/src/main/java/cn/ac/iie/utils/redis/RedisPollUtils.java +++ /dev/null @@ -1,111 +0,0 @@ -package cn.ac.iie.utils.redis; - -import cn.ac.iie.common.FlowWriteConfig; -import com.zdjizhi.utils.StringUtil; -import org.apache.commons.lang3.RandomUtils; -import org.apache.log4j.Logger; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; -import redis.clients.jedis.JedisPoolConfig; - -import java.util.Properties; - -/** - * @author qidaijie - */ -public class RedisPollUtils { - private static final Logger logger = Logger.getLogger(RedisPollUtils.class); - private static JedisPool jedisPool = null; - private static Properties props = new Properties(); - - - private RedisPollUtils() { - } - - static { - initialPool(); - - } - - /** - * 初始化Redis连接池 - */ - private static void initialPool() { - try { - //加载连接池配置文件 - props.load(RedisPollUtils.class.getClassLoader().getResourceAsStream("redis_config.properties")); - // 创建jedis池配置实例 - JedisPoolConfig poolConfig = new JedisPoolConfig(); - poolConfig.setMaxTotal(Integer.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_MAXACTIVE))); - poolConfig.setMaxIdle(Integer.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_MAXIDLE))); - poolConfig.setMaxWaitMillis(Long.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_MAXWAIT))); - poolConfig.setTestOnReturn(Boolean.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_TESTONRETURN))); - poolConfig.setTestOnBorrow(Boolean.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_TESTONBORROW))); - // 根据配置实例化jedis池 - jedisPool = new JedisPool(poolConfig, props.getProperty(FlowWriteConfig.REDIS_IP), - Integer.valueOf(props.getProperty(FlowWriteConfig.REDIS_PORT)), 3000, "123456"); - } catch (Exception e) { - logger.error("Redis连接池初始化错误", e); - } - } - - /** - * 获取Jedis实例 - * - * @return Jedis实例 - */ - public static Jedis getJedis() { - Jedis jedis = null; - try { - if (jedisPool == null) { - initialPool(); - } - jedis = jedisPool.getResource(); - } catch (Exception e) { - logger.error("Redis连接池错误,无法获取连接", e); - } - return jedis; - } - -// /** -// * @param key redis key -// * @return value -// */ -// public static Integer getWorkerId(String key) { -// int workId = 0; -// int maxId = 32; -// try (Jedis jedis = RedisPollUtils.getJedis()) { -// if (jedis != null) { -// String work = jedis.get(key); -// if (StringUtil.isBlank(work)) { -// jedis.set(key, "0"); -// } else { -// workId = Integer.parseInt(work); -// } -// if (workId < maxId) { -// jedis.set(key, String.valueOf(workId + 1)); -// } else { -// workId = 0; -// jedis.set(key, "1"); -// } -// } -// } catch (Exception e) { -// logger.error("通过Redis获取用户名出现异常", e); -// workId = RandomUtils.nextInt(0, 31); -// } -// return workId; -// } - - public static String getWorkerId(String key) { - String sub = ""; - try (Jedis jedis = RedisPollUtils.getJedis()) { - sub = jedis.get(key); - } catch (Exception e) { - logger.error("通过Redis获取用户名出现异常", e); - - } - return sub; - } - - -}