diff --git a/pom.xml b/pom.xml index 88624d0..def6d01 100644 --- a/pom.xml +++ b/pom.xml @@ -142,7 +142,7 @@ com.alibaba fastjson - 1.2.47 + 1.2.59 diff --git a/properties/core-site.xml b/properties/core-site.xml new file mode 100644 index 0000000..93dfb1d --- /dev/null +++ b/properties/core-site.xml @@ -0,0 +1,71 @@ + + + + + + + + + fs.defaultFS + hdfs://ns1 + + + hadoop.tmp.dir + file:/opt/hadoop/tmp + + + io.file.buffer.size + 131702 + + + hadoop.proxyuser.root.hosts + * + + + hadoop.proxyuser.root.groups + * + + + hadoop.logfile.size + 10000000 + The max size of each log file + + + + hadoop.logfile.count + 1 + The max number of log files + + + ha.zookeeper.quorum + master:2181,slave1:2181,slave2:2181 + + +    +     fs.hdfs.impl   +     org.apache.hadoop.hdfs.DistributedFileSystem   +     The FileSystem for hdfs: uris.   + + + +io.compression.codecs +com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec + + +io.compression.codec.lzo.class +com.hadoop.compression.lzo.LzoCodec + + + diff --git a/properties/hbase-site.xml b/properties/hbase-site.xml index d26f7df..54554e4 100644 --- a/properties/hbase-site.xml +++ b/properties/hbase-site.xml @@ -23,7 +23,7 @@ hbase.rootdir - hdfs://ns1/hbase/hbase-1.4.9 + hdfs://ns1/hbase-1.4.9 hbase.cluster.distributed @@ -31,13 +31,13 @@ hbase.zookeeper.quorum - master,slave1,slave2 + 192.168.40.119,192.168.40.122,192.168.40.123 hbase.master.info.port 60010 -#开启schema支持 对应hbase的namespace + phoenix.schema.isNamespaceMappingEnabled true @@ -46,4 +46,32 @@ phoenix.schema.mapSystemTablesToNamespace true + + hbase.client.keyvalue.maxsize + 99428800 + + + hbase.server.keyvalue.maxsize + 99428800 + + + hbase.regionserver.wal.codec + org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec + + + phoenix.query.timeoutMs + 1800000 + + + hbase.rpc.timeout + 1200000 + + + hbase.client.scanner.caching + 1000 + + + hbase.client.scanner.timeout.period + 1200000 + diff --git a/properties/hdfs-site.xml b/properties/hdfs-site.xml index 845b195..1e148b7 100644 --- a/properties/hdfs-site.xml +++ b/properties/hdfs-site.xml @@ -17,13 +17,13 @@ - + dfs.namenode.name.dir - file:/home/bigdata/hadoop/dfs/name + file:/home/ceiec/hadoop/dfs/name dfs.datanode.data.dir - file:/home/bigdata/hadoop/dfs/data + file:/home/ceiec/hadoop/dfs/data dfs.replication @@ -31,11 +31,11 @@ dfs.namenode.secondary.http-address - master:9001 + 192.168.40.119:9001 - dfs.webhdfs.enabled - true + dfs.webhdfs.enabled + true dfs.permissions @@ -60,32 +60,32 @@ dfs.namenode.rpc-address.ns1.nn1 - master:9000 + 192.168.40.119:8020 dfs.namenode.http-address.ns1.nn1 - master:50070 + 192.168.40.119:50070 dfs.namenode.rpc-address.ns1.nn2 - slave1:9000 + 192.168.40.122:8020 dfs.namenode.http-address.ns1.nn2 - slave1:50070 + 192.168.40.122:50070 dfs.namenode.shared.edits.dir - qjournal://slave1:8485;slave2:8485;master:8485/ns1 + qjournal://192.168.40.119:8485;192.168.40.122:8485;192.168.40.123:8485/ns1 dfs.journalnode.edits.dir - /home/bigdata/hadoop/journal + /home/ceiec/hadoop/journal diff --git a/properties/service_flow_config.properties b/properties/service_flow_config.properties index fb65910..8646d78 100644 --- a/properties/service_flow_config.properties +++ b/properties/service_flow_config.properties @@ -1,11 +1,11 @@ #管理kafka地址 -bootstrap.servers=192.168.40.152:9092 +bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092 #zookeeper 地址 -zookeeper.servers=192.168.40.152:2181 +zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181 #hbase zookeeper地址 -hbase.zookeeper.servers=192.168.40.203:2181,192.168.40.206:2181 +hbase.zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181 #hbase tablename hbase.table.name=subcriber_info @@ -14,25 +14,25 @@ hbase.table.name=subcriber_info auto.offset.reset=latest #kafka broker下的topic名称 -kafka.topic=SECURITY-POLICY-LOG +kafka.topic=SECURITY-EVENT-LOG #读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； group.id=security-policy-191112 #输出topic -results.output.topic=SECURITY-POLICY-COMPLETED-LOG +results.output.topic=SECURITY-EVENT-COMPLETED-LOG #storm topology workers -topology.workers=1 +topology.workers=2 #spout并行度 建议与kafka分区数相同 -spout.parallelism=1 +spout.parallelism=3 #处理补全操作的bolt并行度-worker的倍数 -datacenter.bolt.parallelism=1 +datacenter.bolt.parallelism=10 -#写入kafka的并行度 -kafka.bolt.parallelism=1 +#写入kafka的并行度10 +kafka.bolt.parallelism=10 #定位库地址 ip.library=/home/ceiec/topology/dat/ diff --git a/src/main/java/cn/ac/iie/bean/connection/ConnectionRecordLog.java b/src/main/java/cn/ac/iie/bean/connection/ConnectionRecordLog.java new file mode 100644 index 0000000..e73c2d2 --- /dev/null +++ b/src/main/java/cn/ac/iie/bean/connection/ConnectionRecordLog.java @@ -0,0 +1,684 @@ +package cn.ac.iie.bean.connection; + +import cn.ac.iie.bean.PublicSessionRecordLog; + +/** + * 除radius之外 + * + * @author qidaijie + */ +public class ConnectionRecordLog extends PublicSessionRecordLog { + + +//TODO HTTP协议属性 21 + + private String http_url; + private String http_host; + private String http_domain; + private String http_request_line; + private String http_response_line; + private String http_request_header; + private String http_response_header; + private String http_request_body; + private String http_response_body; + private String http_request_body_key; + private String http_response_body_key; + private int http_proxy_flag; + private int http_sequence; + private String http_snapshot; + private String http_cookie; + private String http_referer; + private String http_user_agent; + private String http_content_length; + private String http_content_type; + private String http_set_cookie; + private String http_version; + +//TODO MAIL协议属性 9 + + private String mail_protocol_type; + private String mail_sender; + private String mail_receiver; + private String mail_subject; + private String mail_content; + private String mail_attachment_name; + private String mail_attachment_content; + private String mail_eml_file; + private String mail_snapshot; + +//TODO 3DNS协议属性 18 + + private int dns_message_id; + private int dns_qr; + private int dns_opcode; + private int dns_aa; + private int dns_tc; + private int dns_rd; + private int dns_ra; + private int dns_rcode; + private int dns_qdcount; + private int dns_ancount; + private int dns_nscount; + private int dns_arcount; + private String dns_qname; + private int dns_qtype; + private int dns_qclass; + private String dns_cname; + private int dns_sub; + private String dns_rr; + +//TODO SSL协议属性 13 + + + private int ssl_pinningst; + private int ssl_intercept_state; + private int ssl_server_side_latency; + private int ssl_client_side_latency; + private int ssl_cert_verify; + private int ssl_con_latency_ms; + private String ssl_version; + private String ssl_sni; + private String ssl_san; + private String ssl_cn; + private String ssl_server_side_version; + private String ssl_client_side_version; + private String ssl_error; + + //TODO FTP协议属性 2 + private String ftp_url; + private String ftp_content; + + + //TODO BGP协议属性 3 + private int bgp_type; + private String bgp_as_num; + private String bgp_route; + + //TODO VOIP协议属性 4 + private String voip_calling_account; + private String voip_called_account; + private String voip_calling_number; + private String voip_called_number; + + //TODO STREAMING_MEDIA协议属性 2 + private String streaming_media_url; + private String streaming_media_protocol; + + + public String getHttp_url() { + return http_url; + } + + public void setHttp_url(String http_url) { + this.http_url = http_url; + } + + public String getHttp_host() { + return http_host; + } + + public void setHttp_host(String http_host) { + this.http_host = http_host; + } + + public String getHttp_domain() { + return http_domain; + } + + public void setHttp_domain(String http_domain) { + this.http_domain = http_domain; + } + + public String getHttp_request_line() { + return http_request_line; + } + + public void setHttp_request_line(String http_request_line) { + this.http_request_line = http_request_line; + } + + public String getHttp_response_line() { + return http_response_line; + } + + public void setHttp_response_line(String http_response_line) { + this.http_response_line = http_response_line; + } + + public String getHttp_request_header() { + return http_request_header; + } + + public void setHttp_request_header(String http_request_header) { + this.http_request_header = http_request_header; + } + + public String getHttp_response_header() { + return http_response_header; + } + + public void setHttp_response_header(String http_response_header) { + this.http_response_header = http_response_header; + } + + public String getHttp_request_body() { + return http_request_body; + } + + public void setHttp_request_body(String http_request_body) { + this.http_request_body = http_request_body; + } + + public String getHttp_response_body() { + return http_response_body; + } + + public void setHttp_response_body(String http_response_body) { + this.http_response_body = http_response_body; + } + + public String getHttp_request_body_key() { + return http_request_body_key; + } + + public void setHttp_request_body_key(String http_request_body_key) { + this.http_request_body_key = http_request_body_key; + } + + public String getHttp_response_body_key() { + return http_response_body_key; + } + + public void setHttp_response_body_key(String http_response_body_key) { + this.http_response_body_key = http_response_body_key; + } + + public int getHttp_proxy_flag() { + return http_proxy_flag; + } + + public void setHttp_proxy_flag(int http_proxy_flag) { + this.http_proxy_flag = http_proxy_flag; + } + + public int getHttp_sequence() { + return http_sequence; + } + + public void setHttp_sequence(int http_sequence) { + this.http_sequence = http_sequence; + } + + public String getHttp_snapshot() { + return http_snapshot; + } + + public void setHttp_snapshot(String http_snapshot) { + this.http_snapshot = http_snapshot; + } + + public String getHttp_cookie() { + return http_cookie; + } + + public void setHttp_cookie(String http_cookie) { + this.http_cookie = http_cookie; + } + + public String getHttp_referer() { + return http_referer; + } + + public void setHttp_referer(String http_referer) { + this.http_referer = http_referer; + } + + public String getHttp_user_agent() { + return http_user_agent; + } + + public void setHttp_user_agent(String http_user_agent) { + this.http_user_agent = http_user_agent; + } + + public String getHttp_content_length() { + return http_content_length; + } + + public void setHttp_content_length(String http_content_length) { + this.http_content_length = http_content_length; + } + + public String getHttp_content_type() { + return http_content_type; + } + + public void setHttp_content_type(String http_content_type) { + this.http_content_type = http_content_type; + } + + public String getHttp_set_cookie() { + return http_set_cookie; + } + + public void setHttp_set_cookie(String http_set_cookie) { + this.http_set_cookie = http_set_cookie; + } + + public String getHttp_version() { + return http_version; + } + + public void setHttp_version(String http_version) { + this.http_version = http_version; + } + + public String getMail_protocol_type() { + return mail_protocol_type; + } + + public void setMail_protocol_type(String mail_protocol_type) { + this.mail_protocol_type = mail_protocol_type; + } + + public String getMail_sender() { + return mail_sender; + } + + public void setMail_sender(String mail_sender) { + this.mail_sender = mail_sender; + } + + public String getMail_receiver() { + return mail_receiver; + } + + public void setMail_receiver(String mail_receiver) { + this.mail_receiver = mail_receiver; + } + + public String getMail_subject() { + return mail_subject; + } + + public void setMail_subject(String mail_subject) { + this.mail_subject = mail_subject; + } + + public String getMail_content() { + return mail_content; + } + + public void setMail_content(String mail_content) { + this.mail_content = mail_content; + } + + public String getMail_attachment_name() { + return mail_attachment_name; + } + + public void setMail_attachment_name(String mail_attachment_name) { + this.mail_attachment_name = mail_attachment_name; + } + + public String getMail_attachment_content() { + return mail_attachment_content; + } + + public void setMail_attachment_content(String mail_attachment_content) { + this.mail_attachment_content = mail_attachment_content; + } + + public String getMail_eml_file() { + return mail_eml_file; + } + + public void setMail_eml_file(String mail_eml_file) { + this.mail_eml_file = mail_eml_file; + } + + public String getMail_snapshot() { + return mail_snapshot; + } + + public void setMail_snapshot(String mail_snapshot) { + this.mail_snapshot = mail_snapshot; + } + + public int getDns_message_id() { + return dns_message_id; + } + + public void setDns_message_id(int dns_message_id) { + this.dns_message_id = dns_message_id; + } + + public int getDns_qr() { + return dns_qr; + } + + public void setDns_qr(int dns_qr) { + this.dns_qr = dns_qr; + } + + public int getDns_opcode() { + return dns_opcode; + } + + public void setDns_opcode(int dns_opcode) { + this.dns_opcode = dns_opcode; + } + + public int getDns_aa() { + return dns_aa; + } + + public void setDns_aa(int dns_aa) { + this.dns_aa = dns_aa; + } + + public int getDns_tc() { + return dns_tc; + } + + public void setDns_tc(int dns_tc) { + this.dns_tc = dns_tc; + } + + public int getDns_rd() { + return dns_rd; + } + + public void setDns_rd(int dns_rd) { + this.dns_rd = dns_rd; + } + + public int getDns_ra() { + return dns_ra; + } + + public void setDns_ra(int dns_ra) { + this.dns_ra = dns_ra; + } + + public int getDns_rcode() { + return dns_rcode; + } + + public void setDns_rcode(int dns_rcode) { + this.dns_rcode = dns_rcode; + } + + public int getDns_qdcount() { + return dns_qdcount; + } + + public void setDns_qdcount(int dns_qdcount) { + this.dns_qdcount = dns_qdcount; + } + + public int getDns_ancount() { + return dns_ancount; + } + + public void setDns_ancount(int dns_ancount) { + this.dns_ancount = dns_ancount; + } + + public int getDns_nscount() { + return dns_nscount; + } + + public void setDns_nscount(int dns_nscount) { + this.dns_nscount = dns_nscount; + } + + public int getDns_arcount() { + return dns_arcount; + } + + public void setDns_arcount(int dns_arcount) { + this.dns_arcount = dns_arcount; + } + + public String getDns_qname() { + return dns_qname; + } + + public void setDns_qname(String dns_qname) { + this.dns_qname = dns_qname; + } + + public int getDns_qtype() { + return dns_qtype; + } + + public void setDns_qtype(int dns_qtype) { + this.dns_qtype = dns_qtype; + } + + public int getDns_qclass() { + return dns_qclass; + } + + public void setDns_qclass(int dns_qclass) { + this.dns_qclass = dns_qclass; + } + + public String getDns_cname() { + return dns_cname; + } + + public void setDns_cname(String dns_cname) { + this.dns_cname = dns_cname; + } + + public int getDns_sub() { + return dns_sub; + } + + public void setDns_sub(int dns_sub) { + this.dns_sub = dns_sub; + } + + public String getDns_rr() { + return dns_rr; + } + + public void setDns_rr(String dns_rr) { + this.dns_rr = dns_rr; + } + + public int getSsl_pinningst() { + return ssl_pinningst; + } + + public void setSsl_pinningst(int ssl_pinningst) { + this.ssl_pinningst = ssl_pinningst; + } + + public int getSsl_intercept_state() { + return ssl_intercept_state; + } + + public void setSsl_intercept_state(int ssl_intercept_state) { + this.ssl_intercept_state = ssl_intercept_state; + } + + public int getSsl_server_side_latency() { + return ssl_server_side_latency; + } + + public void setSsl_server_side_latency(int ssl_server_side_latency) { + this.ssl_server_side_latency = ssl_server_side_latency; + } + + public int getSsl_client_side_latency() { + return ssl_client_side_latency; + } + + public void setSsl_client_side_latency(int ssl_client_side_latency) { + this.ssl_client_side_latency = ssl_client_side_latency; + } + + public int getSsl_cert_verify() { + return ssl_cert_verify; + } + + public void setSsl_cert_verify(int ssl_cert_verify) { + this.ssl_cert_verify = ssl_cert_verify; + } + + public int getSsl_con_latency_ms() { + return ssl_con_latency_ms; + } + + public void setSsl_con_latency_ms(int ssl_con_latency_ms) { + this.ssl_con_latency_ms = ssl_con_latency_ms; + } + + public String getSsl_version() { + return ssl_version; + } + + public void setSsl_version(String ssl_version) { + this.ssl_version = ssl_version; + } + + public String getSsl_sni() { + return ssl_sni; + } + + public void setSsl_sni(String ssl_sni) { + this.ssl_sni = ssl_sni; + } + + public String getSsl_san() { + return ssl_san; + } + + public void setSsl_san(String ssl_san) { + this.ssl_san = ssl_san; + } + + public String getSsl_cn() { + return ssl_cn; + } + + public void setSsl_cn(String ssl_cn) { + this.ssl_cn = ssl_cn; + } + + public String getSsl_server_side_version() { + return ssl_server_side_version; + } + + public void setSsl_server_side_version(String ssl_server_side_version) { + this.ssl_server_side_version = ssl_server_side_version; + } + + public String getSsl_client_side_version() { + return ssl_client_side_version; + } + + public void setSsl_client_side_version(String ssl_client_side_version) { + this.ssl_client_side_version = ssl_client_side_version; + } + + public String getSsl_error() { + return ssl_error; + } + + public void setSsl_error(String ssl_error) { + this.ssl_error = ssl_error; + } + + public String getFtp_url() { + return ftp_url; + } + + public void setFtp_url(String ftp_url) { + this.ftp_url = ftp_url; + } + + public String getFtp_content() { + return ftp_content; + } + + public void setFtp_content(String ftp_content) { + this.ftp_content = ftp_content; + } + + public int getBgp_type() { + return bgp_type; + } + + public void setBgp_type(int bgp_type) { + this.bgp_type = bgp_type; + } + + public String getBgp_as_num() { + return bgp_as_num; + } + + public void setBgp_as_num(String bgp_as_num) { + this.bgp_as_num = bgp_as_num; + } + + public String getBgp_route() { + return bgp_route; + } + + public void setBgp_route(String bgp_route) { + this.bgp_route = bgp_route; + } + + public String getVoip_calling_account() { + return voip_calling_account; + } + + public void setVoip_calling_account(String voip_calling_account) { + this.voip_calling_account = voip_calling_account; + } + + public String getVoip_called_account() { + return voip_called_account; + } + + public void setVoip_called_account(String voip_called_account) { + this.voip_called_account = voip_called_account; + } + + public String getVoip_calling_number() { + return voip_calling_number; + } + + public void setVoip_calling_number(String voip_calling_number) { + this.voip_calling_number = voip_calling_number; + } + + public String getVoip_called_number() { + return voip_called_number; + } + + public void setVoip_called_number(String voip_called_number) { + this.voip_called_number = voip_called_number; + } + + + public String getStreaming_media_url() { + return streaming_media_url; + } + + public void setStreaming_media_url(String streaming_media_url) { + this.streaming_media_url = streaming_media_url; + } + + public String getStreaming_media_protocol() { + return streaming_media_protocol; + } + + public void setStreaming_media_protocol(String streaming_media_protocol) { + this.streaming_media_protocol = streaming_media_protocol; + } +} diff --git a/src/main/java/cn/ac/iie/bean/proxy/ProxySessionRecordLog.java b/src/main/java/cn/ac/iie/bean/proxy/ProxySessionRecordLog.java new file mode 100644 index 0000000..ba5dc85 --- /dev/null +++ b/src/main/java/cn/ac/iie/bean/proxy/ProxySessionRecordLog.java @@ -0,0 +1,203 @@ +package cn.ac.iie.bean.proxy; + + +import cn.ac.iie.bean.PublicSessionRecordLog; + +/** + * @author qidaijie + */ +public class ProxySessionRecordLog extends PublicSessionRecordLog { + + +//TODO HTTP协议属性 21 + + private String http_url; + private String http_host; + private String http_domain; + private String http_request_line; + private String http_response_line; + private String http_request_header; + private String http_response_header; + private String http_request_body; + private String http_response_body; + private String http_request_body_key; + private String http_response_body_key; + private int http_proxy_flag; + private int http_sequence; + private String http_snapshot; + private String http_cookie; + private String http_referer; + private String http_user_agent; + private String http_content_length; + private String http_content_type; + private String http_set_cookie; + private String http_version; + + public String getHttp_url() { + return http_url; + } + + public void setHttp_url(String http_url) { + this.http_url = http_url; + } + + public String getHttp_host() { + return http_host; + } + + public void setHttp_host(String http_host) { + this.http_host = http_host; + } + + public String getHttp_domain() { + return http_domain; + } + + public void setHttp_domain(String http_domain) { + this.http_domain = http_domain; + } + + public String getHttp_request_line() { + return http_request_line; + } + + public void setHttp_request_line(String http_request_line) { + this.http_request_line = http_request_line; + } + + public String getHttp_response_line() { + return http_response_line; + } + + public void setHttp_response_line(String http_response_line) { + this.http_response_line = http_response_line; + } + + public String getHttp_request_header() { + return http_request_header; + } + + public void setHttp_request_header(String http_request_header) { + this.http_request_header = http_request_header; + } + + public String getHttp_response_header() { + return http_response_header; + } + + public void setHttp_response_header(String http_response_header) { + this.http_response_header = http_response_header; + } + + public String getHttp_request_body() { + return http_request_body; + } + + public void setHttp_request_body(String http_request_body) { + this.http_request_body = http_request_body; + } + + public String getHttp_response_body() { + return http_response_body; + } + + public void setHttp_response_body(String http_response_body) { + this.http_response_body = http_response_body; + } + + public String getHttp_request_body_key() { + return http_request_body_key; + } + + public void setHttp_request_body_key(String http_request_body_key) { + this.http_request_body_key = http_request_body_key; + } + + public String getHttp_response_body_key() { + return http_response_body_key; + } + + public void setHttp_response_body_key(String http_response_body_key) { + this.http_response_body_key = http_response_body_key; + } + + public int getHttp_proxy_flag() { + return http_proxy_flag; + } + + public void setHttp_proxy_flag(int http_proxy_flag) { + this.http_proxy_flag = http_proxy_flag; + } + + public int getHttp_sequence() { + return http_sequence; + } + + public void setHttp_sequence(int http_sequence) { + this.http_sequence = http_sequence; + } + + public String getHttp_snapshot() { + return http_snapshot; + } + + public void setHttp_snapshot(String http_snapshot) { + this.http_snapshot = http_snapshot; + } + + public String getHttp_cookie() { + return http_cookie; + } + + public void setHttp_cookie(String http_cookie) { + this.http_cookie = http_cookie; + } + + public String getHttp_referer() { + return http_referer; + } + + public void setHttp_referer(String http_referer) { + this.http_referer = http_referer; + } + + public String getHttp_user_agent() { + return http_user_agent; + } + + public void setHttp_user_agent(String http_user_agent) { + this.http_user_agent = http_user_agent; + } + + public String getHttp_content_length() { + return http_content_length; + } + + public void setHttp_content_length(String http_content_length) { + this.http_content_length = http_content_length; + } + + public String getHttp_content_type() { + return http_content_type; + } + + public void setHttp_content_type(String http_content_type) { + this.http_content_type = http_content_type; + } + + public String getHttp_set_cookie() { + return http_set_cookie; + } + + public void setHttp_set_cookie(String http_set_cookie) { + this.http_set_cookie = http_set_cookie; + } + + public String getHttp_version() { + return http_version; + } + + public void setHttp_version(String http_version) { + this.http_version = http_version; + } +} diff --git a/src/main/java/cn/ac/iie/bean/radius/RadiusSessionRecordLog.java b/src/main/java/cn/ac/iie/bean/radius/RadiusSessionRecordLog.java new file mode 100644 index 0000000..4dd11a9 --- /dev/null +++ b/src/main/java/cn/ac/iie/bean/radius/RadiusSessionRecordLog.java @@ -0,0 +1,89 @@ +package cn.ac.iie.bean.radius; + + +import cn.ac.iie.bean.PublicSessionRecordLog; + +/** + * Radius 日志 + * + * @author qidaijie + */ +public class RadiusSessionRecordLog extends PublicSessionRecordLog { + + +//TODO RADIUS协议属性 8 + + private int radius_session_timeout; + private int radius_idle_timeout; + private int radius_acct_status_type; + private int radius_acct_terminate_cause; + private int radius_packet_type; + private String radius_nas_ip; + private String radius_framed_ip; + private String radius_account; + + + public int getRadius_session_timeout() { + return radius_session_timeout; + } + + public void setRadius_session_timeout(int radius_session_timeout) { + this.radius_session_timeout = radius_session_timeout; + } + + public int getRadius_idle_timeout() { + return radius_idle_timeout; + } + + public void setRadius_idle_timeout(int radius_idle_timeout) { + this.radius_idle_timeout = radius_idle_timeout; + } + + public int getRadius_acct_status_type() { + return radius_acct_status_type; + } + + public void setRadius_acct_status_type(int radius_acct_status_type) { + this.radius_acct_status_type = radius_acct_status_type; + } + + public int getRadius_acct_terminate_cause() { + return radius_acct_terminate_cause; + } + + public void setRadius_acct_terminate_cause(int radius_acct_terminate_cause) { + this.radius_acct_terminate_cause = radius_acct_terminate_cause; + } + + public int getRadius_packet_type() { + return radius_packet_type; + } + + public void setRadius_packet_type(int radius_packet_type) { + this.radius_packet_type = radius_packet_type; + } + + public String getRadius_nas_ip() { + return radius_nas_ip; + } + + public void setRadius_nas_ip(String radius_nas_ip) { + this.radius_nas_ip = radius_nas_ip; + } + + public String getRadius_framed_ip() { + return radius_framed_ip; + } + + public void setRadius_framed_ip(String radius_framed_ip) { + this.radius_framed_ip = radius_framed_ip; + } + + public String getRadius_account() { + return radius_account; + } + + public void setRadius_account(String radius_account) { + this.radius_account = radius_account; + } +} diff --git a/src/main/java/cn/ac/iie/bean/security/SecurityPolicyLog.java b/src/main/java/cn/ac/iie/bean/security/SecurityPolicyLog.java new file mode 100644 index 0000000..e266b5e --- /dev/null +++ b/src/main/java/cn/ac/iie/bean/security/SecurityPolicyLog.java @@ -0,0 +1,684 @@ +package cn.ac.iie.bean.security; + +import cn.ac.iie.bean.PublicSessionRecordLog; + +/** + * 策略 + * + * @author qidaijie + */ +public class SecurityPolicyLog extends PublicSessionRecordLog { + + +//TODO HTTP协议属性 21 + + private String http_url; + private String http_host; + private String http_domain; + private String http_request_line; + private String http_response_line; + private String http_request_header; + private String http_response_header; + private String http_request_body; + private String http_response_body; + private String http_request_body_key; + private String http_response_body_key; + private int http_proxy_flag; + private int http_sequence; + private String http_snapshot; + private String http_cookie; + private String http_referer; + private String http_user_agent; + private String http_content_length; + private String http_content_type; + private String http_set_cookie; + private String http_version; + +//TODO MAIL协议属性 9 + + private String mail_protocol_type; + private String mail_sender; + private String mail_receiver; + private String mail_subject; + private String mail_content; + private String mail_attachment_name; + private String mail_attachment_content; + private String mail_eml_file; + private String mail_snapshot; + +//TODO 3DNS协议属性 18 + + private int dns_message_id; + private int dns_qr; + private int dns_opcode; + private int dns_aa; + private int dns_tc; + private int dns_rd; + private int dns_ra; + private int dns_rcode; + private int dns_qdcount; + private int dns_ancount; + private int dns_nscount; + private int dns_arcount; + private String dns_qname; + private int dns_qtype; + private int dns_qclass; + private String dns_cname; + private int dns_sub; + private String dns_rr; + +//TODO SSL协议属性 13 + + + private int ssl_pinningst; + private int ssl_intercept_state; + private int ssl_server_side_latency; + private int ssl_client_side_latency; + private int ssl_cert_verify; + private int ssl_con_latency_ms; + private String ssl_version; + private String ssl_sni; + private String ssl_san; + private String ssl_cn; + private String ssl_server_side_version; + private String ssl_client_side_version; + private String ssl_error; + + //TODO FTP协议属性 2 + private String ftp_url; + private String ftp_content; + + + //TODO BGP协议属性 3 + private int bgp_type; + private String bgp_as_num; + private String bgp_route; + + //TODO VOIP协议属性 4 + private String voip_calling_account; + private String voip_called_account; + private String voip_calling_number; + private String voip_called_number; + + //TODO STREAMING_MEDIA协议属性 2 + private String streaming_media_url; + private String streaming_media_protocol; + + + public String getHttp_url() { + return http_url; + } + + public void setHttp_url(String http_url) { + this.http_url = http_url; + } + + public String getHttp_host() { + return http_host; + } + + public void setHttp_host(String http_host) { + this.http_host = http_host; + } + + public String getHttp_domain() { + return http_domain; + } + + public void setHttp_domain(String http_domain) { + this.http_domain = http_domain; + } + + public String getHttp_request_line() { + return http_request_line; + } + + public void setHttp_request_line(String http_request_line) { + this.http_request_line = http_request_line; + } + + public String getHttp_response_line() { + return http_response_line; + } + + public void setHttp_response_line(String http_response_line) { + this.http_response_line = http_response_line; + } + + public String getHttp_request_header() { + return http_request_header; + } + + public void setHttp_request_header(String http_request_header) { + this.http_request_header = http_request_header; + } + + public String getHttp_response_header() { + return http_response_header; + } + + public void setHttp_response_header(String http_response_header) { + this.http_response_header = http_response_header; + } + + public String getHttp_request_body() { + return http_request_body; + } + + public void setHttp_request_body(String http_request_body) { + this.http_request_body = http_request_body; + } + + public String getHttp_response_body() { + return http_response_body; + } + + public void setHttp_response_body(String http_response_body) { + this.http_response_body = http_response_body; + } + + public String getHttp_request_body_key() { + return http_request_body_key; + } + + public void setHttp_request_body_key(String http_request_body_key) { + this.http_request_body_key = http_request_body_key; + } + + public String getHttp_response_body_key() { + return http_response_body_key; + } + + public void setHttp_response_body_key(String http_response_body_key) { + this.http_response_body_key = http_response_body_key; + } + + public int getHttp_proxy_flag() { + return http_proxy_flag; + } + + public void setHttp_proxy_flag(int http_proxy_flag) { + this.http_proxy_flag = http_proxy_flag; + } + + public int getHttp_sequence() { + return http_sequence; + } + + public void setHttp_sequence(int http_sequence) { + this.http_sequence = http_sequence; + } + + public String getHttp_snapshot() { + return http_snapshot; + } + + public void setHttp_snapshot(String http_snapshot) { + this.http_snapshot = http_snapshot; + } + + public String getHttp_cookie() { + return http_cookie; + } + + public void setHttp_cookie(String http_cookie) { + this.http_cookie = http_cookie; + } + + public String getHttp_referer() { + return http_referer; + } + + public void setHttp_referer(String http_referer) { + this.http_referer = http_referer; + } + + public String getHttp_user_agent() { + return http_user_agent; + } + + public void setHttp_user_agent(String http_user_agent) { + this.http_user_agent = http_user_agent; + } + + public String getHttp_content_length() { + return http_content_length; + } + + public void setHttp_content_length(String http_content_length) { + this.http_content_length = http_content_length; + } + + public String getHttp_content_type() { + return http_content_type; + } + + public void setHttp_content_type(String http_content_type) { + this.http_content_type = http_content_type; + } + + public String getHttp_set_cookie() { + return http_set_cookie; + } + + public void setHttp_set_cookie(String http_set_cookie) { + this.http_set_cookie = http_set_cookie; + } + + public String getHttp_version() { + return http_version; + } + + public void setHttp_version(String http_version) { + this.http_version = http_version; + } + + public String getMail_protocol_type() { + return mail_protocol_type; + } + + public void setMail_protocol_type(String mail_protocol_type) { + this.mail_protocol_type = mail_protocol_type; + } + + public String getMail_sender() { + return mail_sender; + } + + public void setMail_sender(String mail_sender) { + this.mail_sender = mail_sender; + } + + public String getMail_receiver() { + return mail_receiver; + } + + public void setMail_receiver(String mail_receiver) { + this.mail_receiver = mail_receiver; + } + + public String getMail_subject() { + return mail_subject; + } + + public void setMail_subject(String mail_subject) { + this.mail_subject = mail_subject; + } + + public String getMail_content() { + return mail_content; + } + + public void setMail_content(String mail_content) { + this.mail_content = mail_content; + } + + public String getMail_attachment_name() { + return mail_attachment_name; + } + + public void setMail_attachment_name(String mail_attachment_name) { + this.mail_attachment_name = mail_attachment_name; + } + + public String getMail_attachment_content() { + return mail_attachment_content; + } + + public void setMail_attachment_content(String mail_attachment_content) { + this.mail_attachment_content = mail_attachment_content; + } + + public String getMail_eml_file() { + return mail_eml_file; + } + + public void setMail_eml_file(String mail_eml_file) { + this.mail_eml_file = mail_eml_file; + } + + public String getMail_snapshot() { + return mail_snapshot; + } + + public void setMail_snapshot(String mail_snapshot) { + this.mail_snapshot = mail_snapshot; + } + + public int getDns_message_id() { + return dns_message_id; + } + + public void setDns_message_id(int dns_message_id) { + this.dns_message_id = dns_message_id; + } + + public int getDns_qr() { + return dns_qr; + } + + public void setDns_qr(int dns_qr) { + this.dns_qr = dns_qr; + } + + public int getDns_opcode() { + return dns_opcode; + } + + public void setDns_opcode(int dns_opcode) { + this.dns_opcode = dns_opcode; + } + + public int getDns_aa() { + return dns_aa; + } + + public void setDns_aa(int dns_aa) { + this.dns_aa = dns_aa; + } + + public int getDns_tc() { + return dns_tc; + } + + public void setDns_tc(int dns_tc) { + this.dns_tc = dns_tc; + } + + public int getDns_rd() { + return dns_rd; + } + + public void setDns_rd(int dns_rd) { + this.dns_rd = dns_rd; + } + + public int getDns_ra() { + return dns_ra; + } + + public void setDns_ra(int dns_ra) { + this.dns_ra = dns_ra; + } + + public int getDns_rcode() { + return dns_rcode; + } + + public void setDns_rcode(int dns_rcode) { + this.dns_rcode = dns_rcode; + } + + public int getDns_qdcount() { + return dns_qdcount; + } + + public void setDns_qdcount(int dns_qdcount) { + this.dns_qdcount = dns_qdcount; + } + + public int getDns_ancount() { + return dns_ancount; + } + + public void setDns_ancount(int dns_ancount) { + this.dns_ancount = dns_ancount; + } + + public int getDns_nscount() { + return dns_nscount; + } + + public void setDns_nscount(int dns_nscount) { + this.dns_nscount = dns_nscount; + } + + public int getDns_arcount() { + return dns_arcount; + } + + public void setDns_arcount(int dns_arcount) { + this.dns_arcount = dns_arcount; + } + + public String getDns_qname() { + return dns_qname; + } + + public void setDns_qname(String dns_qname) { + this.dns_qname = dns_qname; + } + + public int getDns_qtype() { + return dns_qtype; + } + + public void setDns_qtype(int dns_qtype) { + this.dns_qtype = dns_qtype; + } + + public int getDns_qclass() { + return dns_qclass; + } + + public void setDns_qclass(int dns_qclass) { + this.dns_qclass = dns_qclass; + } + + public String getDns_cname() { + return dns_cname; + } + + public void setDns_cname(String dns_cname) { + this.dns_cname = dns_cname; + } + + public int getDns_sub() { + return dns_sub; + } + + public void setDns_sub(int dns_sub) { + this.dns_sub = dns_sub; + } + + public String getDns_rr() { + return dns_rr; + } + + public void setDns_rr(String dns_rr) { + this.dns_rr = dns_rr; + } + + public int getSsl_pinningst() { + return ssl_pinningst; + } + + public void setSsl_pinningst(int ssl_pinningst) { + this.ssl_pinningst = ssl_pinningst; + } + + public int getSsl_intercept_state() { + return ssl_intercept_state; + } + + public void setSsl_intercept_state(int ssl_intercept_state) { + this.ssl_intercept_state = ssl_intercept_state; + } + + public int getSsl_server_side_latency() { + return ssl_server_side_latency; + } + + public void setSsl_server_side_latency(int ssl_server_side_latency) { + this.ssl_server_side_latency = ssl_server_side_latency; + } + + public int getSsl_client_side_latency() { + return ssl_client_side_latency; + } + + public void setSsl_client_side_latency(int ssl_client_side_latency) { + this.ssl_client_side_latency = ssl_client_side_latency; + } + + public int getSsl_cert_verify() { + return ssl_cert_verify; + } + + public void setSsl_cert_verify(int ssl_cert_verify) { + this.ssl_cert_verify = ssl_cert_verify; + } + + public int getSsl_con_latency_ms() { + return ssl_con_latency_ms; + } + + public void setSsl_con_latency_ms(int ssl_con_latency_ms) { + this.ssl_con_latency_ms = ssl_con_latency_ms; + } + + public String getSsl_version() { + return ssl_version; + } + + public void setSsl_version(String ssl_version) { + this.ssl_version = ssl_version; + } + + public String getSsl_sni() { + return ssl_sni; + } + + public void setSsl_sni(String ssl_sni) { + this.ssl_sni = ssl_sni; + } + + public String getSsl_san() { + return ssl_san; + } + + public void setSsl_san(String ssl_san) { + this.ssl_san = ssl_san; + } + + public String getSsl_cn() { + return ssl_cn; + } + + public void setSsl_cn(String ssl_cn) { + this.ssl_cn = ssl_cn; + } + + public String getSsl_server_side_version() { + return ssl_server_side_version; + } + + public void setSsl_server_side_version(String ssl_server_side_version) { + this.ssl_server_side_version = ssl_server_side_version; + } + + public String getSsl_client_side_version() { + return ssl_client_side_version; + } + + public void setSsl_client_side_version(String ssl_client_side_version) { + this.ssl_client_side_version = ssl_client_side_version; + } + + public String getSsl_error() { + return ssl_error; + } + + public void setSsl_error(String ssl_error) { + this.ssl_error = ssl_error; + } + + public String getFtp_url() { + return ftp_url; + } + + public void setFtp_url(String ftp_url) { + this.ftp_url = ftp_url; + } + + public String getFtp_content() { + return ftp_content; + } + + public void setFtp_content(String ftp_content) { + this.ftp_content = ftp_content; + } + + public int getBgp_type() { + return bgp_type; + } + + public void setBgp_type(int bgp_type) { + this.bgp_type = bgp_type; + } + + public String getBgp_as_num() { + return bgp_as_num; + } + + public void setBgp_as_num(String bgp_as_num) { + this.bgp_as_num = bgp_as_num; + } + + public String getBgp_route() { + return bgp_route; + } + + public void setBgp_route(String bgp_route) { + this.bgp_route = bgp_route; + } + + public String getVoip_calling_account() { + return voip_calling_account; + } + + public void setVoip_calling_account(String voip_calling_account) { + this.voip_calling_account = voip_calling_account; + } + + public String getVoip_called_account() { + return voip_called_account; + } + + public void setVoip_called_account(String voip_called_account) { + this.voip_called_account = voip_called_account; + } + + public String getVoip_calling_number() { + return voip_calling_number; + } + + public void setVoip_calling_number(String voip_calling_number) { + this.voip_calling_number = voip_calling_number; + } + + public String getVoip_called_number() { + return voip_called_number; + } + + public void setVoip_called_number(String voip_called_number) { + this.voip_called_number = voip_called_number; + } + + + public String getStreaming_media_url() { + return streaming_media_url; + } + + public void setStreaming_media_url(String streaming_media_url) { + this.streaming_media_url = streaming_media_url; + } + + public String getStreaming_media_protocol() { + return streaming_media_protocol; + } + + public void setStreaming_media_protocol(String streaming_media_protocol) { + this.streaming_media_protocol = streaming_media_protocol; + } +} diff --git a/src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java b/src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java index 2e08d2e..6f923d5 100644 --- a/src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java +++ b/src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java @@ -1,6 +1,7 @@ package cn.ac.iie.bolt.collect; import cn.ac.iie.common.FlowWriteConfig; +import cn.ac.iie.utils.hbase.HBaseUtils; import cn.ac.iie.utils.system.TupleUtils; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; @@ -16,21 +17,19 @@ import java.util.HashMap; import java.util.Map; import static cn.ac.iie.utils.general.TransFormUtils.getCollectProtocolMessage; -import static cn.ac.iie.utils.hbase.HbaseUtils.change; +import static cn.ac.iie.utils.hbase.HBaseUtils.change; /** * 通联关系日志补全 * * @author qidaijie */ - public class CollectCompletedBolt extends BaseBasicBolt { private final static Logger logger = Logger.getLogger(CollectCompletedBolt.class); private static final long serialVersionUID = 4682827168247333522L; @Override public void prepare(Map stormConf, TopologyContext context) { - } @Override diff --git a/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java index 838702f..fc65431 100644 --- a/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java @@ -16,7 +16,7 @@ import java.util.HashMap; import java.util.Map; import static cn.ac.iie.utils.general.TransFormUtils.getProxyMessage; -import static cn.ac.iie.utils.hbase.HbaseUtils.change; +import static cn.ac.iie.utils.hbase.HBaseUtils.change; /** * 通联关系日志补全 diff --git a/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java index 4947ccd..ca52c03 100644 --- a/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java @@ -1,7 +1,7 @@ package cn.ac.iie.bolt.security; import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.hbase.HbaseUtils; +import cn.ac.iie.utils.hbase.HBaseUtils; import cn.ac.iie.utils.system.TupleUtils; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; @@ -38,7 +38,7 @@ public class SecurityCompletionBolt extends BaseBasicBolt { public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { try { if (TupleUtils.isTick(tuple)) { - HbaseUtils.change(); + HBaseUtils.change(); } else { String message = tuple.getString(0); if (StringUtil.isNotBlank(message)) { diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java index 66f5c49..0340776 100644 --- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java @@ -1,8 +1,11 @@ package cn.ac.iie.utils.general; -import cn.ac.iie.bean.*; +import cn.ac.iie.bean.connection.ConnectionRecordLog; +import cn.ac.iie.bean.proxy.ProxySessionRecordLog; +import cn.ac.iie.bean.radius.RadiusSessionRecordLog; +import cn.ac.iie.bean.security.SecurityPolicyLog; import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.hbase.HbaseUtils; +import cn.ac.iie.utils.hbase.HBaseUtils; import cn.ac.iie.utils.system.SnowflakeId; import com.alibaba.fastjson.JSONObject; import com.zdjizhi.utils.IpLookup; @@ -77,7 +80,7 @@ public class TransFormUtils { proxySessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); proxySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); //TODO 集成AAA数据subscribe_id -数据端补全 - proxySessionRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp)); + proxySessionRecordLog.setCommon_subscriber_id(HBaseUtils.getAccount(clientIp)); proxySessionRecordLog.setHttp_domain(getTopDomain(null, proxySessionRecordLog.getHttp_host())); return JSONObject.toJSONString(proxySessionRecordLog); } catch (Exception e) { @@ -107,7 +110,7 @@ public class TransFormUtils { securitySessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); securitySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); //TODO 集成AAA数据subscribe_id -数据端补全 - securitySessionRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp)); + securitySessionRecordLog.setCommon_subscriber_id(HBaseUtils.getAccount(clientIp)); securitySessionRecordLog.setHttp_domain(getTopDomain(securitySessionRecordLog.getSsl_sni(), securitySessionRecordLog.getHttp_host())); return JSONObject.toJSONString(securitySessionRecordLog); } catch (Exception e) { @@ -125,7 +128,7 @@ public class TransFormUtils { * @return 补全后的日志 */ public static String getCollectProtocolMessage(String message) { - CollectProtocolRecordLog collectProtocolRecordLog = JSONObject.parseObject(message, CollectProtocolRecordLog.class); + ConnectionRecordLog collectProtocolRecordLog = JSONObject.parseObject(message, ConnectionRecordLog.class); String serverIp = collectProtocolRecordLog.getCommon_server_ip(); String clientIp = collectProtocolRecordLog.getCommon_client_ip(); try { @@ -136,7 +139,7 @@ public class TransFormUtils { collectProtocolRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); collectProtocolRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); //TODO 集成AAA数据subscribe_id -数据端补全 - collectProtocolRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp)); + collectProtocolRecordLog.setCommon_subscriber_id(HBaseUtils.getAccount(clientIp)); collectProtocolRecordLog.setHttp_domain(getTopDomain(collectProtocolRecordLog.getSsl_sni(), collectProtocolRecordLog.getHttp_host())); return JSONObject.toJSONString(collectProtocolRecordLog); } catch (Exception e) { @@ -152,7 +155,7 @@ public class TransFormUtils { * @param host host * @return 顶级域名 */ - public static String getTopDomain(String sni, String host) { + private static String getTopDomain(String sni, String host) { if (StringUtil.isNotBlank(sni)) { return getDomain(sni); } else if (StringUtil.isNotBlank(host)) { @@ -181,10 +184,5 @@ public class TransFormUtils { return ""; } - public static void main(String[] args) { - String s = ipLookup.cityLookupDetail("2001:470:19:790::24"); - System.out.println(s); - } - } diff --git a/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java b/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java index b8fecda..f1db9aa 100644 --- a/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java +++ b/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java @@ -16,12 +16,14 @@ import java.util.HashMap; import java.util.Map; /** + * HBase 工具类 + * * @author qidaijie */ -@SuppressWarnings("all") -public class HbaseUtils { - private final static Logger logger = Logger.getLogger(HbaseUtils.class); - private static Map subIdMap = new HashMap<>(16); + +public class HBaseUtils { + private final static Logger logger = Logger.getLogger(HBaseUtils.class); + private static Map subIdMap = new HashMap<>(32); private static Connection connection; private static Long time; @@ -55,29 +57,30 @@ public class HbaseUtils { * @param endTime 结束时间 */ private static void timestampsFilter(Long startTime, Long endTime) { + Long begin = System.currentTimeMillis(); Table table = null; - TableName tableName = TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME); - Admin admin = null; + ResultScanner scanner = null; + Scan scan2 = new Scan(); try { table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME)); - Scan scan2 = new Scan(); scan2.setTimeRange(startTime, endTime); - ResultScanner scanner = table.getScanner(scan2); + scanner = table.getScanner(scan2); + logger.warn("读取HBase变量耗时：" + (System.currentTimeMillis() - begin)); for (Result result : scanner) { Cell[] cells = result.rawCells(); for (Cell cell : cells) { subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); } } - admin = connection.getAdmin(); - admin.flush(tableName); logger.warn("当前集合长度" + subIdMap.keySet().size()); - logger.warn("更新后集合keys：" + subIdMap.keySet()); + logger.warn("更新缓存耗时：" + (System.currentTimeMillis() - begin)); time = endTime; - scanner.close(); } catch (IOException e) { e.printStackTrace(); } finally { + if (scanner != null) { + scanner.close(); + } if (table != null) { try { table.close(); @@ -92,6 +95,7 @@ public class HbaseUtils { * 获取所有的 key value */ private static void getAll() { + Long begin = System.currentTimeMillis(); try { Table table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME)); Scan scan2 = new Scan(); @@ -102,18 +106,20 @@ public class HbaseUtils { subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); } } - logger.warn("获取全量后集合keys：" + subIdMap.keySet()); + logger.warn("获取全量后集合长度：" + subIdMap.size()); + logger.warn("获取全量耗时：" + (System.currentTimeMillis() - begin)); scanner.close(); } catch (IOException e) { e.printStackTrace(); } } + /** * 获取 account * - * @param ip - * @return + * @param ip client_ip + * @return account */ public static String getAccount(String ip) { if (StringUtil.isNotBlank(ip)) { diff --git a/src/main/java/log4j.properties b/src/main/java/log4j.properties index c355401..17c0e9a 100644 --- a/src/main/java/log4j.properties +++ b/src/main/java/log4j.properties @@ -12,7 +12,7 @@ log4j.appender.file.Threshold=error log4j.appender.file.encoding=UTF-8 log4j.appender.file.Append=true #路径请用相对路径，做好相关测试输出到应用目下 -log4j.appender.file.file=galaxy-name.log +log4j.appender.file.file=storm-topology.log log4j.appender.file.DatePattern='.'yyyy-MM-dd log4j.appender.file.layout=org.apache.log4j.PatternLayout #log4j.appender.file.layout.ConversionPattern=%d{HH:mm:ss} %X{ip} [%t] %5p %c{1} %m%n diff --git a/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java b/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java new file mode 100644 index 0000000..5af4124 --- /dev/null +++ b/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java @@ -0,0 +1,140 @@ +package cn.ac.iie.test.hbase; + +import cn.ac.iie.common.FlowWriteConfig; +import cn.ac.iie.utils.hbase.HBaseUtils; +import com.zdjizhi.utils.StringUtil; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.log4j.Logger; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class HBaseTest { + private final static Logger logger = Logger.getLogger(HBaseTest.class); + private static Map subIdMap = new HashMap<>(16); + private static Connection connection; + private static Long time; + + + static { + // 管理Hbase的配置信息 + Configuration configuration = HBaseConfiguration.create(); + // 设置zookeeper节点 + configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS); +// configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS); +// configuration.setInt("zookeeper.session.timeout", 2000); + try { + connection = ConnectionFactory.createConnection(configuration); + time = System.currentTimeMillis(); + } catch (IOException e) { + logger.error("获取HBase连接失败" + e); + e.printStackTrace(); + } + } + +// @Before +// public void connHBase() { +// +// } + + @Test + public void change() { + Long begin = System.currentTimeMillis(); + System.gc(); + Long start = Runtime.getRuntime().freeMemory(); + System.out.println("开始内存"+start); + getAll(); + System.gc(); + Long end = Runtime.getRuntime().freeMemory(); + System.out.println("结束内存"+end); + System.out.println( "一个HashMap对象占内存: " + (end - start)); + System.out.println(System.currentTimeMillis() - begin); + } + + /** + * 获取变更内容 + * + * @param startTime 开始时间 + * @param endTime 结束时间 + */ + private static void timestampsFilter(Long startTime, Long endTime) { + Table table = null; + TableName tableName = TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME); + Admin admin = null; + try { + table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME)); + Scan scan2 = new Scan(); +// scan2.setCaching(500); + scan2.setTimeRange(startTime, endTime); + ResultScanner scanner = table.getScanner(scan2); + scanner.next().isEmpty(); + for (Result result : scanner) { + Cell[] cells = result.rawCells(); + for (Cell cell : cells) { + subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); + } + } + admin = connection.getAdmin(); + admin.flush(tableName); + logger.warn("当前集合长度" + subIdMap.keySet().size()); + logger.warn("更新后集合keys：" + subIdMap.keySet()); + time = endTime; + scanner.close(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + if (table != null) { + try { + table.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } + + /** + * 获取所有的 key value + */ + public static void getAll() { + try { + Table table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME)); + Scan scan2 = new Scan(); + ResultScanner scanner = table.getScanner(scan2); + for (Result result : scanner) { + Cell[] cells = result.rawCells(); + for (Cell cell : cells) { + subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); + } + } + logger.warn("获取全量后集合size：" + subIdMap.size()); + scanner.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + + /** + * 获取 account + * + * @param ip client_ip + * @return account + */ + public static String getAccount(String ip) { + if (StringUtil.isNotBlank(ip)) { + return subIdMap.get(ip); + } else { + return ""; + } + } +}