gfwleak/galaxy-tsg-olap-storm-log-stream-completion
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

更新hbase配置

Browse Source
This commit is contained in:
qidaijie
2019-11-15 09:49:51 +08:00
parent 692eabd056
commit 8f59751da9
16 changed files with 1962 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@@ -142,7 +142,7 @@
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.47</version>
<version>1.2.59</version>
</dependency>
<dependency>

71
properties/core-site.xml Normal file
View File

@@ -0,0 +1,71 @@
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://ns1</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>file:/opt/hadoop/tmp</value>
</property>
<property>
<name>io.file.buffer.size</name>
<value>131702</value>
</property>
<property>
<name>hadoop.proxyuser.root.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.root.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.logfile.size</name>
<value>10000000</value>
<description>The max size of each log file</description>
</property>
<property>
<name>hadoop.logfile.count</name>
<value>1</value>
<description>The max number of log files</description>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>master:2181,slave1:2181,slave2:2181</value>
</property>
<property>  
    <name>fs.hdfs.impl</name>  
    <value>org.apache.hadoop.hdfs.DistributedFileSystem</value>  
    <description>The FileSystem for hdfs: uris.</description>  
</property>
<property>
<name>io.compression.codecs</name>
<value>com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec</value>
</property>
<property>
<name>io.compression.codec.lzo.class</name>
<value>com.hadoop.compression.lzo.LzoCodec</value>
</property>
</configuration>

34
properties/hbase-site.xml
View File

@@ -23,7 +23,7 @@
<configuration>
<property>
<name>hbase.rootdir</name>
<value>hdfs://ns1/hbase/hbase-1.4.9</value>
<value>hdfs://ns1/hbase-1.4.9</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
@@ -31,13 +31,13 @@
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>master,slave1,slave2</value>
<value>192.168.40.119,192.168.40.122,192.168.40.123</value>
</property>
<property>
<name>hbase.master.info.port</name>
<value>60010</value>
</property>
#开启schema支持 对应hbase的namespace
<!-- 开启启schema支持 对应hbase的namespace -->
<property>
<name>phoenix.schema.isNamespaceMappingEnabled</name>
<value>true</value>
@@ -46,4 +46,32 @@
<name>phoenix.schema.mapSystemTablesToNamespace</name>
<value>true</value>
</property>
<property>
<name>hbase.client.keyvalue.maxsize</name>
<value>99428800</value>
</property>
<property>
<name>hbase.server.keyvalue.maxsize</name>
<value>99428800</value>
</property>
<property>
<name>hbase.regionserver.wal.codec</name>
<value>org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec</value>
</property>
<property>
<name>phoenix.query.timeoutMs</name>
<value>1800000</value>
</property>
<property>
<name>hbase.rpc.timeout</name>
<value>1200000</value>
</property>
<property>
<name>hbase.client.scanner.caching</name>
<value>1000</value>
</property>
<property>
<name>hbase.client.scanner.timeout.period</name>
<value>1200000</value>
</property>
</configuration>

24
properties/hdfs-site.xml
View File

@@ -17,13 +17,13 @@
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/home/bigdata/hadoop/dfs/name</value>
<value>file:/home/ceiec/hadoop/dfs/name</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/home/bigdata/hadoop/dfs/data</value>
<value>file:/home/ceiec/hadoop/dfs/data</value>
</property>
<property>
<name>dfs.replication</name>
@@ -31,11 +31,11 @@
</property>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>master:9001</value>
<value>192.168.40.119:9001</value>
</property>
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
</property>
<property>
<name>dfs.permissions</name>
@@ -60,32 +60,32 @@
<!-- nn1的RPC通信地址，nn1所在地址 -->
<property>
<name>dfs.namenode.rpc-address.ns1.nn1</name>
<value>master:9000</value>
<value>192.168.40.119:8020</value>
</property>
<!-- nn1的http通信地址，外部访问地址 -->
<property>
<name>dfs.namenode.http-address.ns1.nn1</name>
<value>master:50070</value>
<value>192.168.40.119:50070</value>
</property>
<!-- nn2的RPC通信地址，nn2所在地址 -->
<property>
<name>dfs.namenode.rpc-address.ns1.nn2</name>
<value>slave1:9000</value>
<value>192.168.40.122:8020</value>
</property>
<!-- nn2的http通信地址，外部访问地址 -->
<property>
<name>dfs.namenode.http-address.ns1.nn2</name>
<value>slave1:50070</value>
<value>192.168.40.122:50070</value>
</property>
<!-- 指定NameNode的元数据在JournalNode日志上的存放位置(一般和zookeeper部署在一起) -->
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://slave1:8485;slave2:8485;master:8485/ns1</value>
<value>qjournal://192.168.40.119:8485;192.168.40.122:8485;192.168.40.123:8485/ns1</value>
</property>
<!-- 指定JournalNode在本地磁盘存放数据的位置 -->
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/home/bigdata/hadoop/journal</value>
<value>/home/ceiec/hadoop/journal</value>
</property>
<!--客户端通过代理访问namenode，访问文件系统，HDFS 客户端与Active 节点通信的Java 类，使用其确定Active 节点是否活跃 -->
<property>

20
properties/service_flow_config.properties
View File

@@ -1,11 +1,11 @@
#管理kafka地址
bootstrap.servers=192.168.40.152:9092
bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092
#zookeeper 地址
zookeeper.servers=192.168.40.152:2181
zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181
#hbase zookeeper地址
hbase.zookeeper.servers=192.168.40.203:2181,192.168.40.206:2181
hbase.zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181
#hbase tablename
hbase.table.name=subcriber_info
@@ -14,25 +14,25 @@ hbase.table.name=subcriber_info
auto.offset.reset=latest
#kafka broker下的topic名称
kafka.topic=SECURITY-POLICY-LOG
kafka.topic=SECURITY-EVENT-LOG
#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据；
group.id=security-policy-191112
#输出topic
results.output.topic=SECURITY-POLICY-COMPLETED-LOG
results.output.topic=SECURITY-EVENT-COMPLETED-LOG
#storm topology workers
topology.workers=1
topology.workers=2
#spout并行度 建议与kafka分区数相同
spout.parallelism=1
spout.parallelism=3
#处理补全操作的bolt并行度-worker的倍数
datacenter.bolt.parallelism=1
datacenter.bolt.parallelism=10
#写入kafka的并行度
kafka.bolt.parallelism=1
#写入kafka的并行度10
kafka.bolt.parallelism=10
#定位库地址
ip.library=/home/ceiec/topology/dat/

684
src/main/java/cn/ac/iie/bean/connection/ConnectionRecordLog.java Normal file
View File

@@ -0,0 +1,684 @@
package cn.ac.iie.bean.connection;
import cn.ac.iie.bean.PublicSessionRecordLog;
/**
* 除radius之外
*
* @author qidaijie
*/
public class ConnectionRecordLog extends PublicSessionRecordLog {
//TODO HTTP协议属性 21
private String http_url;
private String http_host;
private String http_domain;
private String http_request_line;
private String http_response_line;
private String http_request_header;
private String http_response_header;
private String http_request_body;
private String http_response_body;
private String http_request_body_key;
private String http_response_body_key;
private int http_proxy_flag;
private int http_sequence;
private String http_snapshot;
private String http_cookie;
private String http_referer;
private String http_user_agent;
private String http_content_length;
private String http_content_type;
private String http_set_cookie;
private String http_version;
//TODO MAIL协议属性 9
private String mail_protocol_type;
private String mail_sender;
private String mail_receiver;
private String mail_subject;
private String mail_content;
private String mail_attachment_name;
private String mail_attachment_content;
private String mail_eml_file;
private String mail_snapshot;
//TODO 3DNS协议属性 18
private int dns_message_id;
private int dns_qr;
private int dns_opcode;
private int dns_aa;
private int dns_tc;
private int dns_rd;
private int dns_ra;
private int dns_rcode;
private int dns_qdcount;
private int dns_ancount;
private int dns_nscount;
private int dns_arcount;
private String dns_qname;
private int dns_qtype;
private int dns_qclass;
private String dns_cname;
private int dns_sub;
private String dns_rr;
//TODO SSL协议属性 13
private int ssl_pinningst;
private int ssl_intercept_state;
private int ssl_server_side_latency;
private int ssl_client_side_latency;
private int ssl_cert_verify;
private int ssl_con_latency_ms;
private String ssl_version;
private String ssl_sni;
private String ssl_san;
private String ssl_cn;
private String ssl_server_side_version;
private String ssl_client_side_version;
private String ssl_error;
//TODO FTP协议属性 2
private String ftp_url;
private String ftp_content;
//TODO BGP协议属性 3
private int bgp_type;
private String bgp_as_num;
private String bgp_route;
//TODO VOIP协议属性 4
private String voip_calling_account;
private String voip_called_account;
private String voip_calling_number;
private String voip_called_number;
//TODO STREAMING_MEDIA协议属性 2
private String streaming_media_url;
private String streaming_media_protocol;
public String getHttp_url() {
return http_url;
}
public void setHttp_url(String http_url) {
this.http_url = http_url;
}
public String getHttp_host() {
return http_host;
}
public void setHttp_host(String http_host) {
this.http_host = http_host;
}
public String getHttp_domain() {
return http_domain;
}
public void setHttp_domain(String http_domain) {
this.http_domain = http_domain;
}
public String getHttp_request_line() {
return http_request_line;
}
public void setHttp_request_line(String http_request_line) {
this.http_request_line = http_request_line;
}
public String getHttp_response_line() {
return http_response_line;
}
public void setHttp_response_line(String http_response_line) {
this.http_response_line = http_response_line;
}
public String getHttp_request_header() {
return http_request_header;
}
public void setHttp_request_header(String http_request_header) {
this.http_request_header = http_request_header;
}
public String getHttp_response_header() {
return http_response_header;
}
public void setHttp_response_header(String http_response_header) {
this.http_response_header = http_response_header;
}
public String getHttp_request_body() {
return http_request_body;
}
public void setHttp_request_body(String http_request_body) {
this.http_request_body = http_request_body;
}
public String getHttp_response_body() {
return http_response_body;
}
public void setHttp_response_body(String http_response_body) {
this.http_response_body = http_response_body;
}
public String getHttp_request_body_key() {
return http_request_body_key;
}
public void setHttp_request_body_key(String http_request_body_key) {
this.http_request_body_key = http_request_body_key;
}
public String getHttp_response_body_key() {
return http_response_body_key;
}
public void setHttp_response_body_key(String http_response_body_key) {
this.http_response_body_key = http_response_body_key;
}
public int getHttp_proxy_flag() {
return http_proxy_flag;
}
public void setHttp_proxy_flag(int http_proxy_flag) {
this.http_proxy_flag = http_proxy_flag;
}
public int getHttp_sequence() {
return http_sequence;
}
public void setHttp_sequence(int http_sequence) {
this.http_sequence = http_sequence;
}
public String getHttp_snapshot() {
return http_snapshot;
}
public void setHttp_snapshot(String http_snapshot) {
this.http_snapshot = http_snapshot;
}
public String getHttp_cookie() {
return http_cookie;
}
public void setHttp_cookie(String http_cookie) {
this.http_cookie = http_cookie;
}
public String getHttp_referer() {
return http_referer;
}
public void setHttp_referer(String http_referer) {
this.http_referer = http_referer;
}
public String getHttp_user_agent() {
return http_user_agent;
}
public void setHttp_user_agent(String http_user_agent) {
this.http_user_agent = http_user_agent;
}
public String getHttp_content_length() {
return http_content_length;
}
public void setHttp_content_length(String http_content_length) {
this.http_content_length = http_content_length;
}
public String getHttp_content_type() {
return http_content_type;
}
public void setHttp_content_type(String http_content_type) {
this.http_content_type = http_content_type;
}
public String getHttp_set_cookie() {
return http_set_cookie;
}
public void setHttp_set_cookie(String http_set_cookie) {
this.http_set_cookie = http_set_cookie;
}
public String getHttp_version() {
return http_version;
}
public void setHttp_version(String http_version) {
this.http_version = http_version;
}
public String getMail_protocol_type() {
return mail_protocol_type;
}
public void setMail_protocol_type(String mail_protocol_type) {
this.mail_protocol_type = mail_protocol_type;
}
public String getMail_sender() {
return mail_sender;
}
public void setMail_sender(String mail_sender) {
this.mail_sender = mail_sender;
}
public String getMail_receiver() {
return mail_receiver;
}
public void setMail_receiver(String mail_receiver) {
this.mail_receiver = mail_receiver;
}
public String getMail_subject() {
return mail_subject;
}
public void setMail_subject(String mail_subject) {
this.mail_subject = mail_subject;
}
public String getMail_content() {
return mail_content;
}
public void setMail_content(String mail_content) {
this.mail_content = mail_content;
}
public String getMail_attachment_name() {
return mail_attachment_name;
}
public void setMail_attachment_name(String mail_attachment_name) {
this.mail_attachment_name = mail_attachment_name;
}
public String getMail_attachment_content() {
return mail_attachment_content;
}
public void setMail_attachment_content(String mail_attachment_content) {
this.mail_attachment_content = mail_attachment_content;
}
public String getMail_eml_file() {
return mail_eml_file;
}
public void setMail_eml_file(String mail_eml_file) {
this.mail_eml_file = mail_eml_file;
}
public String getMail_snapshot() {
return mail_snapshot;
}
public void setMail_snapshot(String mail_snapshot) {
this.mail_snapshot = mail_snapshot;
}
public int getDns_message_id() {
return dns_message_id;
}
public void setDns_message_id(int dns_message_id) {
this.dns_message_id = dns_message_id;
}
public int getDns_qr() {
return dns_qr;
}
public void setDns_qr(int dns_qr) {
this.dns_qr = dns_qr;
}
public int getDns_opcode() {
return dns_opcode;
}
public void setDns_opcode(int dns_opcode) {
this.dns_opcode = dns_opcode;
}
public int getDns_aa() {
return dns_aa;
}
public void setDns_aa(int dns_aa) {
this.dns_aa = dns_aa;
}
public int getDns_tc() {
return dns_tc;
}
public void setDns_tc(int dns_tc) {
this.dns_tc = dns_tc;
}
public int getDns_rd() {
return dns_rd;
}
public void setDns_rd(int dns_rd) {
this.dns_rd = dns_rd;
}
public int getDns_ra() {
return dns_ra;
}
public void setDns_ra(int dns_ra) {
this.dns_ra = dns_ra;
}
public int getDns_rcode() {
return dns_rcode;
}
public void setDns_rcode(int dns_rcode) {
this.dns_rcode = dns_rcode;
}
public int getDns_qdcount() {
return dns_qdcount;
}
public void setDns_qdcount(int dns_qdcount) {
this.dns_qdcount = dns_qdcount;
}
public int getDns_ancount() {
return dns_ancount;
}
public void setDns_ancount(int dns_ancount) {
this.dns_ancount = dns_ancount;
}
public int getDns_nscount() {
return dns_nscount;
}
public void setDns_nscount(int dns_nscount) {
this.dns_nscount = dns_nscount;
}
public int getDns_arcount() {
return dns_arcount;
}
public void setDns_arcount(int dns_arcount) {
this.dns_arcount = dns_arcount;
}
public String getDns_qname() {
return dns_qname;
}
public void setDns_qname(String dns_qname) {
this.dns_qname = dns_qname;
}
public int getDns_qtype() {
return dns_qtype;
}
public void setDns_qtype(int dns_qtype) {
this.dns_qtype = dns_qtype;
}
public int getDns_qclass() {
return dns_qclass;
}
public void setDns_qclass(int dns_qclass) {
this.dns_qclass = dns_qclass;
}
public String getDns_cname() {
return dns_cname;
}
public void setDns_cname(String dns_cname) {
this.dns_cname = dns_cname;
}
public int getDns_sub() {
return dns_sub;
}
public void setDns_sub(int dns_sub) {
this.dns_sub = dns_sub;
}
public String getDns_rr() {
return dns_rr;
}
public void setDns_rr(String dns_rr) {
this.dns_rr = dns_rr;
}
public int getSsl_pinningst() {
return ssl_pinningst;
}
public void setSsl_pinningst(int ssl_pinningst) {
this.ssl_pinningst = ssl_pinningst;
}
public int getSsl_intercept_state() {
return ssl_intercept_state;
}
public void setSsl_intercept_state(int ssl_intercept_state) {
this.ssl_intercept_state = ssl_intercept_state;
}
public int getSsl_server_side_latency() {
return ssl_server_side_latency;
}
public void setSsl_server_side_latency(int ssl_server_side_latency) {
this.ssl_server_side_latency = ssl_server_side_latency;
}
public int getSsl_client_side_latency() {
return ssl_client_side_latency;
}
public void setSsl_client_side_latency(int ssl_client_side_latency) {
this.ssl_client_side_latency = ssl_client_side_latency;
}
public int getSsl_cert_verify() {
return ssl_cert_verify;
}
public void setSsl_cert_verify(int ssl_cert_verify) {
this.ssl_cert_verify = ssl_cert_verify;
}
public int getSsl_con_latency_ms() {
return ssl_con_latency_ms;
}
public void setSsl_con_latency_ms(int ssl_con_latency_ms) {
this.ssl_con_latency_ms = ssl_con_latency_ms;
}
public String getSsl_version() {
return ssl_version;
}
public void setSsl_version(String ssl_version) {
this.ssl_version = ssl_version;
}
public String getSsl_sni() {
return ssl_sni;
}
public void setSsl_sni(String ssl_sni) {
this.ssl_sni = ssl_sni;
}
public String getSsl_san() {
return ssl_san;
}
public void setSsl_san(String ssl_san) {
this.ssl_san = ssl_san;
}
public String getSsl_cn() {
return ssl_cn;
}
public void setSsl_cn(String ssl_cn) {
this.ssl_cn = ssl_cn;
}
public String getSsl_server_side_version() {
return ssl_server_side_version;
}
public void setSsl_server_side_version(String ssl_server_side_version) {
this.ssl_server_side_version = ssl_server_side_version;
}
public String getSsl_client_side_version() {
return ssl_client_side_version;
}
public void setSsl_client_side_version(String ssl_client_side_version) {
this.ssl_client_side_version = ssl_client_side_version;
}
public String getSsl_error() {
return ssl_error;
}
public void setSsl_error(String ssl_error) {
this.ssl_error = ssl_error;
}
public String getFtp_url() {
return ftp_url;
}
public void setFtp_url(String ftp_url) {
this.ftp_url = ftp_url;
}
public String getFtp_content() {
return ftp_content;
}
public void setFtp_content(String ftp_content) {
this.ftp_content = ftp_content;
}
public int getBgp_type() {
return bgp_type;
}
public void setBgp_type(int bgp_type) {
this.bgp_type = bgp_type;
}
public String getBgp_as_num() {
return bgp_as_num;
}
public void setBgp_as_num(String bgp_as_num) {
this.bgp_as_num = bgp_as_num;
}
public String getBgp_route() {
return bgp_route;
}
public void setBgp_route(String bgp_route) {
this.bgp_route = bgp_route;
}
public String getVoip_calling_account() {
return voip_calling_account;
}
public void setVoip_calling_account(String voip_calling_account) {
this.voip_calling_account = voip_calling_account;
}
public String getVoip_called_account() {
return voip_called_account;
}
public void setVoip_called_account(String voip_called_account) {
this.voip_called_account = voip_called_account;
}
public String getVoip_calling_number() {
return voip_calling_number;
}
public void setVoip_calling_number(String voip_calling_number) {
this.voip_calling_number = voip_calling_number;
}
public String getVoip_called_number() {
return voip_called_number;
}
public void setVoip_called_number(String voip_called_number) {
this.voip_called_number = voip_called_number;
}
public String getStreaming_media_url() {
return streaming_media_url;
}
public void setStreaming_media_url(String streaming_media_url) {
this.streaming_media_url = streaming_media_url;
}
public String getStreaming_media_protocol() {
return streaming_media_protocol;
}
public void setStreaming_media_protocol(String streaming_media_protocol) {
this.streaming_media_protocol = streaming_media_protocol;
}
}

203
src/main/java/cn/ac/iie/bean/proxy/ProxySessionRecordLog.java Normal file
View File

@@ -0,0 +1,203 @@
package cn.ac.iie.bean.proxy;
import cn.ac.iie.bean.PublicSessionRecordLog;
/**
* @author qidaijie
*/
public class ProxySessionRecordLog extends PublicSessionRecordLog {
//TODO HTTP协议属性 21
private String http_url;
private String http_host;
private String http_domain;
private String http_request_line;
private String http_response_line;
private String http_request_header;
private String http_response_header;
private String http_request_body;
private String http_response_body;
private String http_request_body_key;
private String http_response_body_key;
private int http_proxy_flag;
private int http_sequence;
private String http_snapshot;
private String http_cookie;
private String http_referer;
private String http_user_agent;
private String http_content_length;
private String http_content_type;
private String http_set_cookie;
private String http_version;
public String getHttp_url() {
return http_url;
}
public void setHttp_url(String http_url) {
this.http_url = http_url;
}
public String getHttp_host() {
return http_host;
}
public void setHttp_host(String http_host) {
this.http_host = http_host;
}
public String getHttp_domain() {
return http_domain;
}
public void setHttp_domain(String http_domain) {
this.http_domain = http_domain;
}
public String getHttp_request_line() {
return http_request_line;
}
public void setHttp_request_line(String http_request_line) {
this.http_request_line = http_request_line;
}
public String getHttp_response_line() {
return http_response_line;
}
public void setHttp_response_line(String http_response_line) {
this.http_response_line = http_response_line;
}
public String getHttp_request_header() {
return http_request_header;
}
public void setHttp_request_header(String http_request_header) {
this.http_request_header = http_request_header;
}
public String getHttp_response_header() {
return http_response_header;
}
public void setHttp_response_header(String http_response_header) {
this.http_response_header = http_response_header;
}
public String getHttp_request_body() {
return http_request_body;
}
public void setHttp_request_body(String http_request_body) {
this.http_request_body = http_request_body;
}
public String getHttp_response_body() {
return http_response_body;
}
public void setHttp_response_body(String http_response_body) {
this.http_response_body = http_response_body;
}
public String getHttp_request_body_key() {
return http_request_body_key;
}
public void setHttp_request_body_key(String http_request_body_key) {
this.http_request_body_key = http_request_body_key;
}
public String getHttp_response_body_key() {
return http_response_body_key;
}
public void setHttp_response_body_key(String http_response_body_key) {
this.http_response_body_key = http_response_body_key;
}
public int getHttp_proxy_flag() {
return http_proxy_flag;
}
public void setHttp_proxy_flag(int http_proxy_flag) {
this.http_proxy_flag = http_proxy_flag;
}
public int getHttp_sequence() {
return http_sequence;
}
public void setHttp_sequence(int http_sequence) {
this.http_sequence = http_sequence;
}
public String getHttp_snapshot() {
return http_snapshot;
}
public void setHttp_snapshot(String http_snapshot) {
this.http_snapshot = http_snapshot;
}
public String getHttp_cookie() {
return http_cookie;
}
public void setHttp_cookie(String http_cookie) {
this.http_cookie = http_cookie;
}
public String getHttp_referer() {
return http_referer;
}
public void setHttp_referer(String http_referer) {
this.http_referer = http_referer;
}
public String getHttp_user_agent() {
return http_user_agent;
}
public void setHttp_user_agent(String http_user_agent) {
this.http_user_agent = http_user_agent;
}
public String getHttp_content_length() {
return http_content_length;
}
public void setHttp_content_length(String http_content_length) {
this.http_content_length = http_content_length;
}
public String getHttp_content_type() {
return http_content_type;
}
public void setHttp_content_type(String http_content_type) {
this.http_content_type = http_content_type;
}
public String getHttp_set_cookie() {
return http_set_cookie;
}
public void setHttp_set_cookie(String http_set_cookie) {
this.http_set_cookie = http_set_cookie;
}
public String getHttp_version() {
return http_version;
}
public void setHttp_version(String http_version) {
this.http_version = http_version;
}
}

89
src/main/java/cn/ac/iie/bean/radius/RadiusSessionRecordLog.java Normal file
View File

@@ -0,0 +1,89 @@
package cn.ac.iie.bean.radius;
import cn.ac.iie.bean.PublicSessionRecordLog;
/**
* Radius 日志
*
* @author qidaijie
*/
public class RadiusSessionRecordLog extends PublicSessionRecordLog {
//TODO RADIUS协议属性 8
private int radius_session_timeout;
private int radius_idle_timeout;
private int radius_acct_status_type;
private int radius_acct_terminate_cause;
private int radius_packet_type;
private String radius_nas_ip;
private String radius_framed_ip;
private String radius_account;
public int getRadius_session_timeout() {
return radius_session_timeout;
}
public void setRadius_session_timeout(int radius_session_timeout) {
this.radius_session_timeout = radius_session_timeout;
}
public int getRadius_idle_timeout() {
return radius_idle_timeout;
}
public void setRadius_idle_timeout(int radius_idle_timeout) {
this.radius_idle_timeout = radius_idle_timeout;
}
public int getRadius_acct_status_type() {
return radius_acct_status_type;
}
public void setRadius_acct_status_type(int radius_acct_status_type) {
this.radius_acct_status_type = radius_acct_status_type;
}
public int getRadius_acct_terminate_cause() {
return radius_acct_terminate_cause;
}
public void setRadius_acct_terminate_cause(int radius_acct_terminate_cause) {
this.radius_acct_terminate_cause = radius_acct_terminate_cause;
}
public int getRadius_packet_type() {
return radius_packet_type;
}
public void setRadius_packet_type(int radius_packet_type) {
this.radius_packet_type = radius_packet_type;
}
public String getRadius_nas_ip() {
return radius_nas_ip;
}
public void setRadius_nas_ip(String radius_nas_ip) {
this.radius_nas_ip = radius_nas_ip;
}
public String getRadius_framed_ip() {
return radius_framed_ip;
}
public void setRadius_framed_ip(String radius_framed_ip) {
this.radius_framed_ip = radius_framed_ip;
}
public String getRadius_account() {
return radius_account;
}
public void setRadius_account(String radius_account) {
this.radius_account = radius_account;
}
}

684
src/main/java/cn/ac/iie/bean/security/SecurityPolicyLog.java Normal file
View File

@@ -0,0 +1,684 @@
package cn.ac.iie.bean.security;
import cn.ac.iie.bean.PublicSessionRecordLog;
/**
* 策略
*
* @author qidaijie
*/
public class SecurityPolicyLog extends PublicSessionRecordLog {
//TODO HTTP协议属性 21
private String http_url;
private String http_host;
private String http_domain;
private String http_request_line;
private String http_response_line;
private String http_request_header;
private String http_response_header;
private String http_request_body;
private String http_response_body;
private String http_request_body_key;
private String http_response_body_key;
private int http_proxy_flag;
private int http_sequence;
private String http_snapshot;
private String http_cookie;
private String http_referer;
private String http_user_agent;
private String http_content_length;
private String http_content_type;
private String http_set_cookie;
private String http_version;
//TODO MAIL协议属性 9
private String mail_protocol_type;
private String mail_sender;
private String mail_receiver;
private String mail_subject;
private String mail_content;
private String mail_attachment_name;
private String mail_attachment_content;
private String mail_eml_file;
private String mail_snapshot;
//TODO 3DNS协议属性 18
private int dns_message_id;
private int dns_qr;
private int dns_opcode;
private int dns_aa;
private int dns_tc;
private int dns_rd;
private int dns_ra;
private int dns_rcode;
private int dns_qdcount;
private int dns_ancount;
private int dns_nscount;
private int dns_arcount;
private String dns_qname;
private int dns_qtype;
private int dns_qclass;
private String dns_cname;
private int dns_sub;
private String dns_rr;
//TODO SSL协议属性 13
private int ssl_pinningst;
private int ssl_intercept_state;
private int ssl_server_side_latency;
private int ssl_client_side_latency;
private int ssl_cert_verify;
private int ssl_con_latency_ms;
private String ssl_version;
private String ssl_sni;
private String ssl_san;
private String ssl_cn;
private String ssl_server_side_version;
private String ssl_client_side_version;
private String ssl_error;
//TODO FTP协议属性 2
private String ftp_url;
private String ftp_content;
//TODO BGP协议属性 3
private int bgp_type;
private String bgp_as_num;
private String bgp_route;
//TODO VOIP协议属性 4
private String voip_calling_account;
private String voip_called_account;
private String voip_calling_number;
private String voip_called_number;
//TODO STREAMING_MEDIA协议属性 2
private String streaming_media_url;
private String streaming_media_protocol;
public String getHttp_url() {
return http_url;
}
public void setHttp_url(String http_url) {
this.http_url = http_url;
}
public String getHttp_host() {
return http_host;
}
public void setHttp_host(String http_host) {
this.http_host = http_host;
}
public String getHttp_domain() {
return http_domain;
}
public void setHttp_domain(String http_domain) {
this.http_domain = http_domain;
}
public String getHttp_request_line() {
return http_request_line;
}
public void setHttp_request_line(String http_request_line) {
this.http_request_line = http_request_line;
}
public String getHttp_response_line() {
return http_response_line;
}
public void setHttp_response_line(String http_response_line) {
this.http_response_line = http_response_line;
}
public String getHttp_request_header() {
return http_request_header;
}
public void setHttp_request_header(String http_request_header) {
this.http_request_header = http_request_header;
}
public String getHttp_response_header() {
return http_response_header;
}
public void setHttp_response_header(String http_response_header) {
this.http_response_header = http_response_header;
}
public String getHttp_request_body() {
return http_request_body;
}
public void setHttp_request_body(String http_request_body) {
this.http_request_body = http_request_body;
}
public String getHttp_response_body() {
return http_response_body;
}
public void setHttp_response_body(String http_response_body) {
this.http_response_body = http_response_body;
}
public String getHttp_request_body_key() {
return http_request_body_key;
}
public void setHttp_request_body_key(String http_request_body_key) {
this.http_request_body_key = http_request_body_key;
}
public String getHttp_response_body_key() {
return http_response_body_key;
}
public void setHttp_response_body_key(String http_response_body_key) {
this.http_response_body_key = http_response_body_key;
}
public int getHttp_proxy_flag() {
return http_proxy_flag;
}
public void setHttp_proxy_flag(int http_proxy_flag) {
this.http_proxy_flag = http_proxy_flag;
}
public int getHttp_sequence() {
return http_sequence;
}
public void setHttp_sequence(int http_sequence) {
this.http_sequence = http_sequence;
}
public String getHttp_snapshot() {
return http_snapshot;
}
public void setHttp_snapshot(String http_snapshot) {
this.http_snapshot = http_snapshot;
}
public String getHttp_cookie() {
return http_cookie;
}
public void setHttp_cookie(String http_cookie) {
this.http_cookie = http_cookie;
}
public String getHttp_referer() {
return http_referer;
}
public void setHttp_referer(String http_referer) {
this.http_referer = http_referer;
}
public String getHttp_user_agent() {
return http_user_agent;
}
public void setHttp_user_agent(String http_user_agent) {
this.http_user_agent = http_user_agent;
}
public String getHttp_content_length() {
return http_content_length;
}
public void setHttp_content_length(String http_content_length) {
this.http_content_length = http_content_length;
}
public String getHttp_content_type() {
return http_content_type;
}
public void setHttp_content_type(String http_content_type) {
this.http_content_type = http_content_type;
}
public String getHttp_set_cookie() {
return http_set_cookie;
}
public void setHttp_set_cookie(String http_set_cookie) {
this.http_set_cookie = http_set_cookie;
}
public String getHttp_version() {
return http_version;
}
public void setHttp_version(String http_version) {
this.http_version = http_version;
}
public String getMail_protocol_type() {
return mail_protocol_type;
}
public void setMail_protocol_type(String mail_protocol_type) {
this.mail_protocol_type = mail_protocol_type;
}
public String getMail_sender() {
return mail_sender;
}
public void setMail_sender(String mail_sender) {
this.mail_sender = mail_sender;
}
public String getMail_receiver() {
return mail_receiver;
}
public void setMail_receiver(String mail_receiver) {
this.mail_receiver = mail_receiver;
}
public String getMail_subject() {
return mail_subject;
}
public void setMail_subject(String mail_subject) {
this.mail_subject = mail_subject;
}
public String getMail_content() {
return mail_content;
}
public void setMail_content(String mail_content) {
this.mail_content = mail_content;
}
public String getMail_attachment_name() {
return mail_attachment_name;
}
public void setMail_attachment_name(String mail_attachment_name) {
this.mail_attachment_name = mail_attachment_name;
}
public String getMail_attachment_content() {
return mail_attachment_content;
}
public void setMail_attachment_content(String mail_attachment_content) {
this.mail_attachment_content = mail_attachment_content;
}
public String getMail_eml_file() {
return mail_eml_file;
}
public void setMail_eml_file(String mail_eml_file) {
this.mail_eml_file = mail_eml_file;
}
public String getMail_snapshot() {
return mail_snapshot;
}
public void setMail_snapshot(String mail_snapshot) {
this.mail_snapshot = mail_snapshot;
}
public int getDns_message_id() {
return dns_message_id;
}
public void setDns_message_id(int dns_message_id) {
this.dns_message_id = dns_message_id;
}
public int getDns_qr() {
return dns_qr;
}
public void setDns_qr(int dns_qr) {
this.dns_qr = dns_qr;
}
public int getDns_opcode() {
return dns_opcode;
}
public void setDns_opcode(int dns_opcode) {
this.dns_opcode = dns_opcode;
}
public int getDns_aa() {
return dns_aa;
}
public void setDns_aa(int dns_aa) {
this.dns_aa = dns_aa;
}
public int getDns_tc() {
return dns_tc;
}
public void setDns_tc(int dns_tc) {
this.dns_tc = dns_tc;
}
public int getDns_rd() {
return dns_rd;
}
public void setDns_rd(int dns_rd) {
this.dns_rd = dns_rd;
}
public int getDns_ra() {
return dns_ra;
}
public void setDns_ra(int dns_ra) {
this.dns_ra = dns_ra;
}
public int getDns_rcode() {
return dns_rcode;
}
public void setDns_rcode(int dns_rcode) {
this.dns_rcode = dns_rcode;
}
public int getDns_qdcount() {
return dns_qdcount;
}
public void setDns_qdcount(int dns_qdcount) {
this.dns_qdcount = dns_qdcount;
}
public int getDns_ancount() {
return dns_ancount;
}
public void setDns_ancount(int dns_ancount) {
this.dns_ancount = dns_ancount;
}
public int getDns_nscount() {
return dns_nscount;
}
public void setDns_nscount(int dns_nscount) {
this.dns_nscount = dns_nscount;
}
public int getDns_arcount() {
return dns_arcount;
}
public void setDns_arcount(int dns_arcount) {
this.dns_arcount = dns_arcount;
}
public String getDns_qname() {
return dns_qname;
}
public void setDns_qname(String dns_qname) {
this.dns_qname = dns_qname;
}
public int getDns_qtype() {
return dns_qtype;
}
public void setDns_qtype(int dns_qtype) {
this.dns_qtype = dns_qtype;
}
public int getDns_qclass() {
return dns_qclass;
}
public void setDns_qclass(int dns_qclass) {
this.dns_qclass = dns_qclass;
}
public String getDns_cname() {
return dns_cname;
}
public void setDns_cname(String dns_cname) {
this.dns_cname = dns_cname;
}
public int getDns_sub() {
return dns_sub;
}
public void setDns_sub(int dns_sub) {
this.dns_sub = dns_sub;
}
public String getDns_rr() {
return dns_rr;
}
public void setDns_rr(String dns_rr) {
this.dns_rr = dns_rr;
}
public int getSsl_pinningst() {
return ssl_pinningst;
}
public void setSsl_pinningst(int ssl_pinningst) {
this.ssl_pinningst = ssl_pinningst;
}
public int getSsl_intercept_state() {
return ssl_intercept_state;
}
public void setSsl_intercept_state(int ssl_intercept_state) {
this.ssl_intercept_state = ssl_intercept_state;
}
public int getSsl_server_side_latency() {
return ssl_server_side_latency;
}
public void setSsl_server_side_latency(int ssl_server_side_latency) {
this.ssl_server_side_latency = ssl_server_side_latency;
}
public int getSsl_client_side_latency() {
return ssl_client_side_latency;
}
public void setSsl_client_side_latency(int ssl_client_side_latency) {
this.ssl_client_side_latency = ssl_client_side_latency;
}
public int getSsl_cert_verify() {
return ssl_cert_verify;
}
public void setSsl_cert_verify(int ssl_cert_verify) {
this.ssl_cert_verify = ssl_cert_verify;
}
public int getSsl_con_latency_ms() {
return ssl_con_latency_ms;
}
public void setSsl_con_latency_ms(int ssl_con_latency_ms) {
this.ssl_con_latency_ms = ssl_con_latency_ms;
}
public String getSsl_version() {
return ssl_version;
}
public void setSsl_version(String ssl_version) {
this.ssl_version = ssl_version;
}
public String getSsl_sni() {
return ssl_sni;
}
public void setSsl_sni(String ssl_sni) {
this.ssl_sni = ssl_sni;
}
public String getSsl_san() {
return ssl_san;
}
public void setSsl_san(String ssl_san) {
this.ssl_san = ssl_san;
}
public String getSsl_cn() {
return ssl_cn;
}
public void setSsl_cn(String ssl_cn) {
this.ssl_cn = ssl_cn;
}
public String getSsl_server_side_version() {
return ssl_server_side_version;
}
public void setSsl_server_side_version(String ssl_server_side_version) {
this.ssl_server_side_version = ssl_server_side_version;
}
public String getSsl_client_side_version() {
return ssl_client_side_version;
}
public void setSsl_client_side_version(String ssl_client_side_version) {
this.ssl_client_side_version = ssl_client_side_version;
}
public String getSsl_error() {
return ssl_error;
}
public void setSsl_error(String ssl_error) {
this.ssl_error = ssl_error;
}
public String getFtp_url() {
return ftp_url;
}
public void setFtp_url(String ftp_url) {
this.ftp_url = ftp_url;
}
public String getFtp_content() {
return ftp_content;
}
public void setFtp_content(String ftp_content) {
this.ftp_content = ftp_content;
}
public int getBgp_type() {
return bgp_type;
}
public void setBgp_type(int bgp_type) {
this.bgp_type = bgp_type;
}
public String getBgp_as_num() {
return bgp_as_num;
}
public void setBgp_as_num(String bgp_as_num) {
this.bgp_as_num = bgp_as_num;
}
public String getBgp_route() {
return bgp_route;
}
public void setBgp_route(String bgp_route) {
this.bgp_route = bgp_route;
}
public String getVoip_calling_account() {
return voip_calling_account;
}
public void setVoip_calling_account(String voip_calling_account) {
this.voip_calling_account = voip_calling_account;
}
public String getVoip_called_account() {
return voip_called_account;
}
public void setVoip_called_account(String voip_called_account) {
this.voip_called_account = voip_called_account;
}
public String getVoip_calling_number() {
return voip_calling_number;
}
public void setVoip_calling_number(String voip_calling_number) {
this.voip_calling_number = voip_calling_number;
}
public String getVoip_called_number() {
return voip_called_number;
}
public void setVoip_called_number(String voip_called_number) {
this.voip_called_number = voip_called_number;
}
public String getStreaming_media_url() {
return streaming_media_url;
}
public void setStreaming_media_url(String streaming_media_url) {
this.streaming_media_url = streaming_media_url;
}
public String getStreaming_media_protocol() {
return streaming_media_protocol;
}
public void setStreaming_media_protocol(String streaming_media_protocol) {
this.streaming_media_protocol = streaming_media_protocol;
}
}

5
src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java
View File

@@ -1,6 +1,7 @@
package cn.ac.iie.bolt.collect;
import cn.ac.iie.common.FlowWriteConfig;
import cn.ac.iie.utils.hbase.HBaseUtils;
import cn.ac.iie.utils.system.TupleUtils;
import com.zdjizhi.utils.StringUtil;
import org.apache.log4j.Logger;
@@ -16,21 +17,19 @@ import java.util.HashMap;
import java.util.Map;
import static cn.ac.iie.utils.general.TransFormUtils.getCollectProtocolMessage;
import static cn.ac.iie.utils.hbase.HbaseUtils.change;
import static cn.ac.iie.utils.hbase.HBaseUtils.change;
/**
* 通联关系日志补全
*
* @author qidaijie
*/
public class CollectCompletedBolt extends BaseBasicBolt {
private final static Logger logger = Logger.getLogger(CollectCompletedBolt.class);
private static final long serialVersionUID = 4682827168247333522L;
@Override
public void prepare(Map stormConf, TopologyContext context) {
}
@Override

2
src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java
View File

@@ -16,7 +16,7 @@ import java.util.HashMap;
import java.util.Map;
import static cn.ac.iie.utils.general.TransFormUtils.getProxyMessage;
import static cn.ac.iie.utils.hbase.HbaseUtils.change;
import static cn.ac.iie.utils.hbase.HBaseUtils.change;
/**
* 通联关系日志补全

4
src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java
View File

@@ -1,7 +1,7 @@
package cn.ac.iie.bolt.security;
import cn.ac.iie.common.FlowWriteConfig;
import cn.ac.iie.utils.hbase.HbaseUtils;
import cn.ac.iie.utils.hbase.HBaseUtils;
import cn.ac.iie.utils.system.TupleUtils;
import com.zdjizhi.utils.StringUtil;
import org.apache.log4j.Logger;
@@ -38,7 +38,7 @@ public class SecurityCompletionBolt extends BaseBasicBolt {
public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
try {
if (TupleUtils.isTick(tuple)) {
HbaseUtils.change();
HBaseUtils.change();
} else {
String message = tuple.getString(0);
if (StringUtil.isNotBlank(message)) {

22
src/main/java/cn/ac/iie/utils/general/TransFormUtils.java
View File

@@ -1,8 +1,11 @@
package cn.ac.iie.utils.general;
import cn.ac.iie.bean.*;
import cn.ac.iie.bean.connection.ConnectionRecordLog;
import cn.ac.iie.bean.proxy.ProxySessionRecordLog;
import cn.ac.iie.bean.radius.RadiusSessionRecordLog;
import cn.ac.iie.bean.security.SecurityPolicyLog;
import cn.ac.iie.common.FlowWriteConfig;
import cn.ac.iie.utils.hbase.HbaseUtils;
import cn.ac.iie.utils.hbase.HBaseUtils;
import cn.ac.iie.utils.system.SnowflakeId;
import com.alibaba.fastjson.JSONObject;
import com.zdjizhi.utils.IpLookup;
@@ -77,7 +80,7 @@ public class TransFormUtils {
proxySessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true));
proxySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true));
//TODO 集成AAA数据subscribe_id -数据端补全
proxySessionRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp));
proxySessionRecordLog.setCommon_subscriber_id(HBaseUtils.getAccount(clientIp));
proxySessionRecordLog.setHttp_domain(getTopDomain(null, proxySessionRecordLog.getHttp_host()));
return JSONObject.toJSONString(proxySessionRecordLog);
} catch (Exception e) {
@@ -107,7 +110,7 @@ public class TransFormUtils {
securitySessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true));
securitySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true));
//TODO 集成AAA数据subscribe_id -数据端补全
securitySessionRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp));
securitySessionRecordLog.setCommon_subscriber_id(HBaseUtils.getAccount(clientIp));
securitySessionRecordLog.setHttp_domain(getTopDomain(securitySessionRecordLog.getSsl_sni(), securitySessionRecordLog.getHttp_host()));
return JSONObject.toJSONString(securitySessionRecordLog);
} catch (Exception e) {
@@ -125,7 +128,7 @@ public class TransFormUtils {
* @return 补全后的日志
*/
public static String getCollectProtocolMessage(String message) {
CollectProtocolRecordLog collectProtocolRecordLog = JSONObject.parseObject(message, CollectProtocolRecordLog.class);
ConnectionRecordLog collectProtocolRecordLog = JSONObject.parseObject(message, ConnectionRecordLog.class);
String serverIp = collectProtocolRecordLog.getCommon_server_ip();
String clientIp = collectProtocolRecordLog.getCommon_client_ip();
try {
@@ -136,7 +139,7 @@ public class TransFormUtils {
collectProtocolRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true));
collectProtocolRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true));
//TODO 集成AAA数据subscribe_id -数据端补全
collectProtocolRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp));
collectProtocolRecordLog.setCommon_subscriber_id(HBaseUtils.getAccount(clientIp));
collectProtocolRecordLog.setHttp_domain(getTopDomain(collectProtocolRecordLog.getSsl_sni(), collectProtocolRecordLog.getHttp_host()));
return JSONObject.toJSONString(collectProtocolRecordLog);
} catch (Exception e) {
@@ -152,7 +155,7 @@ public class TransFormUtils {
* @param host host
* @return 顶级域名
*/
public static String getTopDomain(String sni, String host) {
private static String getTopDomain(String sni, String host) {
if (StringUtil.isNotBlank(sni)) {
return getDomain(sni);
} else if (StringUtil.isNotBlank(host)) {
@@ -181,10 +184,5 @@ public class TransFormUtils {
return "";
}
public static void main(String[] args) {
String s = ipLookup.cityLookupDetail("2001:470:19:790::24");
System.out.println(s);
}
}

36
src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java
View File

@@ -16,12 +16,14 @@ import java.util.HashMap;
import java.util.Map;
/**
* HBase 工具类
*
* @author qidaijie
*/
@SuppressWarnings("all")
public class HbaseUtils {
private final static Logger logger = Logger.getLogger(HbaseUtils.class);
private static Map<String, String> subIdMap = new HashMap<>(16);
public class HBaseUtils {
private final static Logger logger = Logger.getLogger(HBaseUtils.class);
private static Map<String, String> subIdMap = new HashMap<>(32);
private static Connection connection;
private static Long time;
@@ -55,29 +57,30 @@ public class HbaseUtils {
* @param endTime 结束时间
*/
private static void timestampsFilter(Long startTime, Long endTime) {
Long begin = System.currentTimeMillis();
Table table = null;
TableName tableName = TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME);
Admin admin = null;
ResultScanner scanner = null;
Scan scan2 = new Scan();
try {
table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME));
Scan scan2 = new Scan();
scan2.setTimeRange(startTime, endTime);
ResultScanner scanner = table.getScanner(scan2);
scanner = table.getScanner(scan2);
logger.warn("读取HBase变量耗时：" + (System.currentTimeMillis() - begin));
for (Result result : scanner) {
Cell[] cells = result.rawCells();
for (Cell cell : cells) {
subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell)));
}
}
admin = connection.getAdmin();
admin.flush(tableName);
logger.warn("当前集合长度" + subIdMap.keySet().size());
logger.warn("更新后集合keys：" + subIdMap.keySet());
logger.warn("更新缓存耗时：" + (System.currentTimeMillis() - begin));
time = endTime;
scanner.close();
} catch (IOException e) {
e.printStackTrace();
} finally {
if (scanner != null) {
scanner.close();
}
if (table != null) {
try {
table.close();
@@ -92,6 +95,7 @@ public class HbaseUtils {
* 获取所有的 key value
*/
private static void getAll() {
Long begin = System.currentTimeMillis();
try {
Table table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME));
Scan scan2 = new Scan();
@@ -102,18 +106,20 @@ public class HbaseUtils {
subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell)));
}
}
logger.warn("获取全量后集合keys：" + subIdMap.keySet());
logger.warn("获取全量后集合长度：" + subIdMap.size());
logger.warn("获取全量耗时：" + (System.currentTimeMillis() - begin));
scanner.close();
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* 获取 account
*
* @param ip
* @return
* @param ip client_ip
* @return account
*/
public static String getAccount(String ip) {
if (StringUtil.isNotBlank(ip)) {

2
src/main/java/log4j.properties
View File

@@ -12,7 +12,7 @@ log4j.appender.file.Threshold=error
log4j.appender.file.encoding=UTF-8
log4j.appender.file.Append=true
#路径请用相对路径，做好相关测试输出到应用目下
log4j.appender.file.file=galaxy-name.log
log4j.appender.file.file=storm-topology.log
log4j.appender.file.DatePattern='.'yyyy-MM-dd
log4j.appender.file.layout=org.apache.log4j.PatternLayout
#log4j.appender.file.layout.ConversionPattern=%d{HH:mm:ss} %X{ip} [%t] %5p %c{1} %m%n

140
src/test/java/cn/ac/iie/test/hbase/HBaseTest.java Normal file
View File

@@ -0,0 +1,140 @@
package cn.ac.iie.test.hbase;
import cn.ac.iie.common.FlowWriteConfig;
import cn.ac.iie.utils.hbase.HBaseUtils;
import com.zdjizhi.utils.StringUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.log4j.Logger;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
public class HBaseTest {
private final static Logger logger = Logger.getLogger(HBaseTest.class);
private static Map<String, String> subIdMap = new HashMap<>(16);
private static Connection connection;
private static Long time;
static {
// 管理Hbase的配置信息
Configuration configuration = HBaseConfiguration.create();
// 设置zookeeper节点
configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS);
// configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS);
// configuration.setInt("zookeeper.session.timeout", 2000);
try {
connection = ConnectionFactory.createConnection(configuration);
time = System.currentTimeMillis();
} catch (IOException e) {
logger.error("获取HBase连接失败" + e);
e.printStackTrace();
}
}
// @Before
// public void connHBase() {
//
// }
@Test
public void change() {
Long begin = System.currentTimeMillis();
System.gc();
Long start = Runtime.getRuntime().freeMemory();
System.out.println("开始内存"+start);
getAll();
System.gc();
Long end = Runtime.getRuntime().freeMemory();
System.out.println("结束内存"+end);
System.out.println( "一个HashMap对象占内存: " + (end - start));
System.out.println(System.currentTimeMillis() - begin);
}
/**
* 获取变更内容
*
* @param startTime 开始时间
* @param endTime 结束时间
*/
private static void timestampsFilter(Long startTime, Long endTime) {
Table table = null;
TableName tableName = TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME);
Admin admin = null;
try {
table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME));
Scan scan2 = new Scan();
// scan2.setCaching(500);
scan2.setTimeRange(startTime, endTime);
ResultScanner scanner = table.getScanner(scan2);
scanner.next().isEmpty();
for (Result result : scanner) {
Cell[] cells = result.rawCells();
for (Cell cell : cells) {
subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell)));
}
}
admin = connection.getAdmin();
admin.flush(tableName);
logger.warn("当前集合长度" + subIdMap.keySet().size());
logger.warn("更新后集合keys：" + subIdMap.keySet());
time = endTime;
scanner.close();
} catch (IOException e) {
e.printStackTrace();
} finally {
if (table != null) {
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
/**
* 获取所有的 key value
*/
public static void getAll() {
try {
Table table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME));
Scan scan2 = new Scan();
ResultScanner scanner = table.getScanner(scan2);
for (Result result : scanner) {
Cell[] cells = result.rawCells();
for (Cell cell : cells) {
subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell)));
}
}
logger.warn("获取全量后集合size：" + subIdMap.size());
scanner.close();
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* 获取 account
*
* @param ip client_ip
* @return account
*/
public static String getAccount(String ip) {
if (StringUtil.isNotBlank(ip)) {
return subIdMap.get(ip);
} else {
return "";
}
}
}